commit d2c0cd4d9acc7060320677cbdda761534308b7d4 Author: lakshit verma Date: Sat Aug 8 00:35:22 2026 +0530 runtimeclass-debugger: edge-node diagnostic tool for the KubeEdge RuntimeClass path diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..0a1d13a --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,32 @@ +name: ci + +on: + push: + branches: [main, master] + pull_request: + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: '1.25.x' + + - name: Build + run: go build ./... + + - name: Vet + run: go vet ./... + + - name: Test + run: go test ./... + + - name: Lint + uses: golangci/golangci-lint-action@v6 + with: + version: v1.64 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d106891 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +bin/ +*.db +*.db-journal +*.db-wal +*.db-shm diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..a1c4e32 --- /dev/null +++ b/Makefile @@ -0,0 +1,25 @@ +BINARY := runtimeclass-debugger +VERSION ?= $(shell git describe --tags --always --dirty 2>/dev/null || echo dev) +LDFLAGS := -X main.version=$(VERSION) + +.PHONY: all build test vet lint fmt clean + +all: build + +build: + go build -ldflags '$(LDFLAGS)' -o bin/$(BINARY) ./cmd/runtimeclass-debugger + +test: + go test ./... + +vet: + go vet ./... + +lint: + golangci-lint run + +fmt: + gofmt -l . + +clean: + rm -rf bin diff --git a/README.md b/README.md new file mode 100644 index 0000000..6630516 --- /dev/null +++ b/README.md @@ -0,0 +1,109 @@ +# runtimeclass-debugger + +runtimeclass-debugger is an edge-node diagnostic tool for the KubeEdge RuntimeClass path. The tool does not implement RuntimeClass support. It finds which layer of the path is broken on a given edge node. You run it on the node, and it answers four questions in dependency order. The report names the layer that failed. + +The project reference is kubeedge/kubeedge issue 7106, LFX 2026 Term 3. The cloud-to-edge RuntimeClass sync work is in pull request 7141. + +## Why it exists + +Edged bridges only three API groups from the kubelet client to the edge-local metaclient. They are CoreV1, StorageV1, and CoordinationV1. The bridge code lives in kubeclientbridge. NodeV1 is the group where RuntimeClass lives. Edged does not bridge NodeV1. The kubelet RuntimeClass manager therefore reads from an empty fake store. The manager always returns NotFound. + +The objects may still reach the edge. Containerd may still have the right handler. A pod that declares a runtime class still never starts. The failure looks like an infrastructure problem, not a KubeEdge bug. This tool walks the path top-down and shows where the gap is. + +## The four questions + +1. transport: do RuntimeClass objects exist on the edge at all? +2. bridge: can Edged resolve a class name to a handler? +3. cri: is the handler configured in the container runtime? +4. events: would the user see the failure? + +## Install and build + +The tool needs Go 1.23 or later and a C toolchain. The local store uses the +mattn/go-sqlite3 driver. + +make build creates the binary at bin/runtimeclass-debugger. You can also +build it directly: + +```sh +go build -o runtimeclass-debugger ./cmd/runtimeclass-debugger +``` + +## Usage + +```sh +runtimeclass-debugger diagnose [class-name] # run all four checks, print report +runtimeclass-debugger check transport # run only check 1 +runtimeclass-debugger check bridge # run only check 2 +runtimeclass-debugger check cri # run only check 3 +runtimeclass-debugger check events # run only check 4 +runtimeclass-debugger version +``` + +diagnose takes an optional class name. When you give a class name, checks 2 and 3 use that class only. When you omit it, the tool uses the classes that MetaServer serves. + +### Flags + +| Flag | Default | Meaning | +| --- | --- | --- | +| metaserver-address | 127.0.0.1:10550 | Address of the MetaServer. You may include a scheme. | +| containerd-config | /etc/containerd/config.toml | Path of the containerd config file. | +| edgecore-config | /etc/kubeedge/config/edgecore.yaml | Path of the edgecore config file. | +| db-file | /var/lib/kubeedge/edgecore.db | Path of the KubeEdge local SQLite store. | +| cert-file | | Client certificate for MetaServer auth. Optional. | +| key-file | | Client key for MetaServer auth. Optional. | +| ca-file | | CA certificate for MetaServer auth. Optional. | +| timeout | 5s | HTTP timeout for MetaServer queries. | +| output | table | Output format: table or json. The default is table. | + +### Example + +``` +$ runtimeclass-debugger diagnose kata +RuntimeClass Edge Diagnostics +============================= +[PASS] transport 1 RuntimeClass object(s) served by MetaServer (kata) +[FAIL] bridge class "kata" not resolvable: edge bridge gap (NodeV1 not wired to metaclient) +[PASS] cri handler "kata" configured in containerd +[WARN] events reportEvent is false: pod failures may not surface as events. Enable edgeCore.reportEvent + +Result: 2 pass, 1 fail, 1 warn +``` + +Pass --output json for machine-readable output. The exit code works in scripts and CI. A zero exit means all checks passed. One means at least one check failed. Two means warnings only. Three means the tool failed to run. + +## The four checks + +### 1. transport: do RuntimeClass objects exist on the edge at all? + +The tool sends a request to the local MetaServer at 127.0.0.1:10550. The request asks for RuntimeClass objects. MetaServer returns at least one object, and the check passes. MetaServer returns an empty list, and the check fails. The reason is a transport gap. The sync from cloud to edge has not reached the node. Pull request 7141 covers this sync. MetaServer is not reachable, and the check warns. MetaServer should always listen on an edge node. + +### 2. bridge: can Edged resolve a class name to a handler? + +The tool reproduces the path that Edged uses to resolve a class name. It runs two lookups. The naive lookup uses a client built the same way as kubeclientbridge. NodeV1 reads from an empty fake store. This is the current kubelet behavior. The lookup always returns NotFound. The wired lookup uses the classes from the local SQLite store. It shows what the kubelet would see once NodeV1 connects to the metaclient. The report shows both outcomes. You see the gap with evidence instead of an assumption. + +The resolution rules match upstream kubelet. No class name gives an empty handler, the default runtime. A known class gives its handler string. An unknown class gives NotFound. + +### 3. cri: is the handler configured in the runtime? + +The tool parses the containerd config file. It looks for CRI plugin runtime entries under the containerd runtimes section. It also accepts the legacy plugins.cri spelling. Every class handler from check 1 must have a matching entry. Kata uses handler names such as kata, kata-clh, kata-qemu, and kata-qemu-tdx. A handler with no entry keeps the pod Pending. The error is FailedCreatePodSandBox. + +### 4. events: would the user see the failure? + +The tool reads the value of edged.reportEvent from the edgecore config file. The default path is /etc/kubeedge/config/edgecore.yaml. When reportEvent is false, which is the default, edged creates no event client. Events never leave the node. A pod stuck in Pending may show no event at all. You would not see the failure. + +## Reading the report + +The tool exists to expose two root causes. + +- Objects never reach the edge. This is the transport gap. Pull request 7141 fixes this part. +- NodeV1 is not wired to the metaclient. This is the bridge gap. kubeclientbridge does not bridge NodeV1. + +When the transport and bridge checks both pass, check 3 confirms the runtime side. Check 4 confirms that you would see the failure. A failed bridge with a passed cri shows the classic KubeEdge symptom. The handler is ready. The class never resolves. + +## Development + +make fmt checks the formatting. make build builds the tool. make vet runs go vet. make lint runs golangci-lint. make test runs the unit tests. + +The tool reads local state only, it never writes to the node. It opens the local SQLite store read-only. It tolerates both store schemas: the meta_v2 table and the legacy meta table. + diff --git a/cmd/runtimeclass-debugger/check.go b/cmd/runtimeclass-debugger/check.go new file mode 100644 index 0000000..ffcde73 --- /dev/null +++ b/cmd/runtimeclass-debugger/check.go @@ -0,0 +1,79 @@ +package main + +import ( + "context" + "os" + + "github.com/spf13/cobra" + + "github.com/kubeedge/runtimeclass-debugger/pkg/diag" +) + +var checkCmd = &cobra.Command{ + Use: "check", + Short: "Run a single RuntimeClass path check", +} + +var checkTransportCmd = &cobra.Command{ + Use: "transport", + Short: "Check 1: does the edge have RuntimeClass objects at all?", + Args: cobra.NoArgs, + Run: func(cmd *cobra.Command, args []string) { + res, _ := buildDiag().CheckTransport(context.Background()) + os.Exit(printReport(singleReport(res))) + }, +} + +var checkBridgeCmd = &cobra.Command{ + Use: "bridge [class-name]", + Short: "Check 2: can Edged resolve the class name to a handler?", + Args: cobra.MaximumNArgs(1), + Run: func(cmd *cobra.Command, args []string) { + d := buildDiag() + os.Exit(printReport(singleReport(d.CheckBridge(scopeFor(d, args))))) + }, +} + +var checkCRICmd = &cobra.Command{ + Use: "cri [class-name]", + Short: "Check 3: is the handler configured in the container runtime?", + Args: cobra.MaximumNArgs(1), + Run: func(cmd *cobra.Command, args []string) { + d := buildDiag() + os.Exit(printReport(singleReport(d.CheckCRI(scopeFor(d, args))))) + }, +} + +var checkEventsCmd = &cobra.Command{ + Use: "events", + Short: "Check 4: would a failure be visible to the user?", + Args: cobra.NoArgs, + Run: func(cmd *cobra.Command, args []string) { + os.Exit(printReport(singleReport(buildDiag().CheckEvents()))) + }, +} + +func init() { + checkCmd.AddCommand(checkTransportCmd, checkBridgeCmd, checkCRICmd, checkEventsCmd) +} + +func singleReport(res diag.CheckResult) *diag.Report { + r := &diag.Report{Checks: []diag.CheckResult{res}} + r.ExitCode = r.ComputeExitCode() + return r +} + +func scopeFor(d *diag.Diag, args []string) []string { + if len(args) > 0 { + return args + } + classes, err := d.MetaServer.ListRuntimeClasses(context.Background()) + if err != nil { + return nil + } + names := make([]string, 0, len(classes)) + for _, c := range classes { + names = append(names, c.Name) + } + return names +} diff --git a/cmd/runtimeclass-debugger/diagnose.go b/cmd/runtimeclass-debugger/diagnose.go new file mode 100644 index 0000000..8aa71b2 --- /dev/null +++ b/cmd/runtimeclass-debugger/diagnose.go @@ -0,0 +1,21 @@ +package main + +import ( + "context" + "os" + + "github.com/spf13/cobra" +) + +var diagnoseCmd = &cobra.Command{ + Use: "diagnose [class-name]", + Short: "Run all four checks and print the diagnosis report", + Long: `Run all four RuntimeClass path checks in dependency order and print a single +report. An optional class name scopes the bridge and CRI checks to that +class; if omitted, the classes served by MetaServer are used.`, + Args: cobra.MaximumNArgs(1), + Run: func(cmd *cobra.Command, args []string) { + report := buildDiag().Run(context.Background(), args) + os.Exit(printReport(report)) + }, +} diff --git a/cmd/runtimeclass-debugger/main.go b/cmd/runtimeclass-debugger/main.go new file mode 100644 index 0000000..16f6edf --- /dev/null +++ b/cmd/runtimeclass-debugger/main.go @@ -0,0 +1,15 @@ +package main + +import ( + "fmt" + "os" + + "github.com/kubeedge/runtimeclass-debugger/pkg/diag" +) + +func main() { + if err := rootCmd.Execute(); err != nil { + fmt.Fprintf(os.Stderr, "runtimeclass-debugger: %v\n", err) + os.Exit(diag.ExitError) + } +} diff --git a/cmd/runtimeclass-debugger/root.go b/cmd/runtimeclass-debugger/root.go new file mode 100644 index 0000000..53e9e55 --- /dev/null +++ b/cmd/runtimeclass-debugger/root.go @@ -0,0 +1,89 @@ +package main + +import ( + "fmt" + "os" + "time" + + "github.com/spf13/cobra" + + "github.com/kubeedge/runtimeclass-debugger/pkg/diag" + "github.com/kubeedge/runtimeclass-debugger/pkg/metaserver" +) + +var ( + metaserverAddress string + containerdConfig string + edgecoreConfig string + certFile string + keyFile string + caFile string + requestTimeout time.Duration + outputFormat string + dbFile string +) + +var rootCmd = &cobra.Command{ + Use: "runtimeclass-debugger", + Short: "Diagnose the KubeEdge RuntimeClass path on an edge node", + Long: `runtimeclass-debugger answers four yes/no questions about the RuntimeClass +path on a KubeEdge edge node, in dependency order: + + 1. transport: does the edge have RuntimeClass objects at all? + 2. bridge: can Edged resolve a class name to a handler? + 3. cri: is the resolved handler configured in the container runtime? + 4. events: would a failure even be visible to the user? + +It reads only local state: the edge MetaServer, local config files, and the +local SQLite store. It does not depend on a working control plane. + +Reference: kubeedge/kubeedge#7106; related sync work: kubeedge/kubeedge#7141.`, + SilenceUsage: true, + SilenceErrors: true, + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { + if outputFormat != "table" && outputFormat != "json" { + return fmt.Errorf("invalid --output %q: must be table or json", outputFormat) + } + return nil + }, +} + +func init() { + rootCmd.PersistentFlags().StringVar(&metaserverAddress, "metaserver-address", "127.0.0.1:10550", "MetaServer address (host:port, may include a scheme)") + rootCmd.PersistentFlags().StringVar(&containerdConfig, "containerd-config", "/etc/containerd/config.toml", "containerd config path") + rootCmd.PersistentFlags().StringVar(&edgecoreConfig, "edgecore-config", "/etc/kubeedge/config/edgecore.yaml", "edgecore config path") + rootCmd.PersistentFlags().StringVar(&certFile, "cert-file", "", "client cert for MetaServer auth (optional)") + rootCmd.PersistentFlags().StringVar(&keyFile, "key-file", "", "client key for MetaServer auth (optional)") + rootCmd.PersistentFlags().StringVar(&caFile, "ca-file", "", "CA cert for MetaServer auth (optional)") + rootCmd.PersistentFlags().DurationVar(&requestTimeout, "timeout", 5*time.Second, "HTTP timeout for MetaServer queries") + rootCmd.PersistentFlags().StringVar(&outputFormat, "output", "table", "output format: table (default), json") + rootCmd.PersistentFlags().StringVar(&dbFile, "db-file", "/var/lib/kubeedge/edgecore.db", "KubeEdge local SQLite store path") + + rootCmd.AddCommand(diagnoseCmd, checkCmd, versionCmd) +} + +func buildDiag() *diag.Diag { + ms := &metaserver.Client{ + BaseURL: metaserverAddress, + Timeout: requestTimeout, + CertFile: certFile, + KeyFile: keyFile, + CAFile: caFile, + } + return diag.New(ms, dbFile, containerdConfig, edgecoreConfig) +} + +func printReport(report *diag.Report) int { + switch outputFormat { + case "json": + out, err := report.JSON() + if err != nil { + fmt.Fprintf(os.Stderr, "runtimeclass-debugger: %v\n", err) + return diag.ExitError + } + fmt.Println(out) + default: + fmt.Print(report.Table()) + } + return report.ExitCode +} diff --git a/cmd/runtimeclass-debugger/version.go b/cmd/runtimeclass-debugger/version.go new file mode 100644 index 0000000..936b68d --- /dev/null +++ b/cmd/runtimeclass-debugger/version.go @@ -0,0 +1,17 @@ +package main + +import ( + "fmt" + + "github.com/spf13/cobra" +) + +var version = "dev" + +var versionCmd = &cobra.Command{ + Use: "version", + Short: "Print the runtimeclass-debugger version", + Run: func(cmd *cobra.Command, args []string) { + fmt.Println("runtimeclass-debugger", version) + }, +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..93a5ae7 --- /dev/null +++ b/go.mod @@ -0,0 +1,55 @@ +module github.com/kubeedge/runtimeclass-debugger + +go 1.25.12 + +require ( + github.com/mattn/go-sqlite3 v1.14.22 + github.com/pelletier/go-toml/v2 v2.4.3 + github.com/spf13/cobra v1.10.2 + gopkg.in/yaml.v3 v3.0.1 + k8s.io/api v0.32.10 + k8s.io/apimachinery v0.32.10 + k8s.io/client-go v0.32.10 + k8s.io/kubernetes v1.32.10 +) + +require ( + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/emicklei/go-restful/v3 v3.11.0 // indirect + github.com/fxamacker/cbor/v2 v2.7.0 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.23.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/google/gnostic-models v0.6.8 // indirect + github.com/google/go-cmp v0.6.0 // indirect + github.com/google/gofuzz v1.2.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/spf13/pflag v1.0.9 // indirect + github.com/x448/float16 v0.8.4 // indirect + golang.org/x/net v0.30.0 // indirect + golang.org/x/oauth2 v0.23.0 // indirect + golang.org/x/sys v0.26.0 // indirect + golang.org/x/term v0.25.0 // indirect + golang.org/x/text v0.19.0 // indirect + golang.org/x/time v0.7.0 // indirect + google.golang.org/protobuf v1.35.1 // indirect + gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + k8s.io/klog/v2 v2.130.1 // indirect + k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f // indirect + k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 // indirect + sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.4.2 // indirect + sigs.k8s.io/yaml v1.4.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..c4160b6 --- /dev/null +++ b/go.sum @@ -0,0 +1,167 @@ +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= +github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= +github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= +github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= +github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= +github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/onsi/ginkgo/v2 v2.21.0 h1:7rg/4f3rB88pb5obDgNZrNHrQ4e6WpjonchcpuBRnZM= +github.com/onsi/ginkgo/v2 v2.21.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= +github.com/onsi/gomega v1.35.1 h1:Cwbd75ZBPxFSuZ6T+rN/WCb/gOc6YgFBXLlZLhC7Ds4= +github.com/onsi/gomega v1.35.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= +github.com/pelletier/go-toml/v2 v2.4.3 h1:GTRvJQutkOSftxIFD5xw9aepkYNuPWmVJpffdDPYVpY= +github.com/pelletier/go-toml/v2 v2.4.3/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= +github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= +github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= +golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= +golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs= +golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= +golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.25.0 h1:WtHI/ltw4NvSUig5KARz9h521QvRC8RmF/cuYqifU24= +golang.org/x/term v0.25.0/go.mod h1:RPyXicDX+6vLxogjjRxjgD2TKtmAO6NZBsBRfrOLu7M= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= +golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/time v0.7.0 h1:ntUhktv3OPE6TgYxXWv9vKvUSJyIFJlyohwbkEwPrKQ= +golang.org/x/time v0.7.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ= +golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= +google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= +gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/api v0.32.10 h1:ocp4turNfa1V40TuBW/LuA17TeXG9g/GI2ebg0KxBNk= +k8s.io/api v0.32.10/go.mod h1:AsMsc4b6TuampYqgMEGSv0HBFpRS4BlKTXAVCAa7oF4= +k8s.io/apimachinery v0.32.10 h1:SAg2kUPLYRcBJQj66oniP1BnXSqw+l1GvJFsJlBmVvQ= +k8s.io/apimachinery v0.32.10/go.mod h1:GpHVgxoKlTxClKcteaeuF1Ul/lDVb74KpZcxcmLDElE= +k8s.io/client-go v0.32.10 h1:MFmIjsKtcnn7mStjrJG1ZW2WzLsKKn6ZtL9hHM/W0xU= +k8s.io/client-go v0.32.10/go.mod h1:qJy/Ws3zSwnu/nD75D+/of1uxbwWHxrYT5P3FuobVLI= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f h1:GA7//TjRY9yWGy1poLzYYJJ4JRdzg3+O6e8I+e+8T5Y= +k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f/go.mod h1:R/HEjbvWI0qdfb8viZUeVZm0X6IZnxAydC7YU42CMw4= +k8s.io/kubernetes v1.32.10 h1:yiRa8DyKp4Yrbv028MP6kpp5N1N3eO8Hp/tSCbBGIPE= +k8s.io/kubernetes v1.32.10/go.mod h1:o2pRStsMR7Uq62zcugfUEQsxnuyFt9r8migMrbsVH00= +k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 h1:M3sRQVHv7vB20Xc2ybTt7ODCeFj6JSWYFzOFnYeS6Ro= +k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 h1:/Rv+M11QRah1itp8VhT6HoVx1Ray9eB4DBr+K+/sCJ8= +sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3/go.mod h1:18nIHnGi6636UCz6m8i4DhaJ65T6EruyzmoQqI2BVDo= +sigs.k8s.io/structured-merge-diff/v4 v4.4.2 h1:MdmvkGuXi/8io6ixD5wud3vOLwc1rj0aNqRlpuvjmwA= +sigs.k8s.io/structured-merge-diff/v4 v4.4.2/go.mod h1:N8f93tFZh9U6vpxwRArLiikrE5/2tiu1w1AGfACIGE4= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/pkg/bridge/localstore.go b/pkg/bridge/localstore.go new file mode 100644 index 0000000..1744b1e --- /dev/null +++ b/pkg/bridge/localstore.go @@ -0,0 +1,139 @@ +package bridge + +import ( + "database/sql" + "encoding/json" + "errors" + "fmt" + "os" + "sort" + + nodev1 "k8s.io/api/node/v1" + + _ "github.com/mattn/go-sqlite3" +) + +// ErrStoreNotFound is returned when the local SQLite store file does not +// exist. +var ErrStoreNotFound = errors.New("local store not found") + +// Store reads Kubernetes API objects from the KubeEdge local SQLite store +// (/var/lib/kubeedge/edgecore.db by default). +type Store struct { + path string + db *sql.DB +} + +// OpenStore opens the KubeEdge local SQLite store read-only. +func OpenStore(path string) (*Store, error) { + if _, err := os.Stat(path); err != nil { + if os.IsNotExist(err) { + return nil, fmt.Errorf("%w: %s", ErrStoreNotFound, path) + } + return nil, err + } + db, err := sql.Open("sqlite3", "file:"+path+"?mode=ro") + if err != nil { + return nil, err + } + if err := db.Ping(); err != nil { + db.Close() + return nil, err + } + return &Store{path: path, db: db}, nil +} + +// Close closes the underlying database handle. +func (s *Store) Close() error { + return s.db.Close() +} + +// ListRuntimeClasses returns the RuntimeClass objects stored locally. +// Both the meta_v2 table (key format /////) +// and the legacy meta table (key format //) are scanned, so +// the tool keeps working regardless of which store generation the node runs. +// Column naming varies between KubeEdge versions (group_version_resource vs +// groupversionresource), so the schema is introspected first. +func (s *Store) ListRuntimeClasses() ([]nodev1.RuntimeClass, error) { + classes := []nodev1.RuntimeClass{} + seen := map[string]bool{} + classes, seen = s.scanTable("meta_v2", classes, seen) + classes, _ = s.scanTable("meta", classes, seen) + sort.Slice(classes, func(i, j int) bool { return classes[i].Name < classes[j].Name }) + return classes, nil +} + +func (s *Store) scanTable(table string, classes []nodev1.RuntimeClass, seen map[string]bool) ([]nodev1.RuntimeClass, map[string]bool) { + columns, err := s.tableColumns(table) + if err != nil || !hasColumn(columns, "value") { + return classes, seen + } + where := "key LIKE '%/runtimeclass%'" + if hasColumn(columns, "group_version_resource") { + where += " OR group_version_resource LIKE '%runtimeclass%'" + } else if hasColumn(columns, "groupversionresource") { + where += " OR groupversionresource LIKE '%runtimeclass%'" + } else if hasColumn(columns, "type") { + where += " OR type LIKE '%runtimeclass%'" + } + rows, err := s.db.Query(fmt.Sprintf("SELECT value FROM %s WHERE %s", table, where)) + if err != nil { + return classes, seen + } + return appendClasses(rows, classes, seen) +} + +func (s *Store) tableColumns(table string) ([]string, error) { + rows, err := s.db.Query(fmt.Sprintf("SELECT name FROM pragma_table_info('%s')", table)) + if err != nil { + return nil, err + } + defer rows.Close() + var columns []string + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + return nil, err + } + columns = append(columns, name) + } + return columns, rows.Err() +} + +func hasColumn(columns []string, name string) bool { + for _, c := range columns { + if c == name { + return true + } + } + return false +} + +func appendClasses(rows *sql.Rows, classes []nodev1.RuntimeClass, seen map[string]bool) ([]nodev1.RuntimeClass, map[string]bool) { + defer rows.Close() + for rows.Next() { + var raw string + if err := rows.Scan(&raw); err != nil { + continue + } + var probe struct { + Kind string `json:"kind"` + } + if err := json.Unmarshal([]byte(raw), &probe); err != nil { + continue + } + if probe.Kind != "RuntimeClass" { + continue + } + var rc nodev1.RuntimeClass + if err := json.Unmarshal([]byte(raw), &rc); err != nil { + continue + } + if rc.Name == "" || seen[rc.Name] { + continue + } + seen[rc.Name] = true + classes = append(classes, rc) + } + return classes, seen +} diff --git a/pkg/bridge/localstore_test.go b/pkg/bridge/localstore_test.go new file mode 100644 index 0000000..28d3316 --- /dev/null +++ b/pkg/bridge/localstore_test.go @@ -0,0 +1,143 @@ +package bridge + +import ( + "database/sql" + "errors" + "fmt" + "path/filepath" + "testing" +) + +const createMetaV2 = `CREATE TABLE meta_v2 ( + key TEXT PRIMARY KEY, + group_version_resource TEXT, + namespace TEXT, + name TEXT, + resource_version INTEGER, + value TEXT +)` + +const createMeta = `CREATE TABLE meta (key TEXT PRIMARY KEY, type TEXT, value TEXT)` + +func classJSON(name, handler string) string { + return fmt.Sprintf(`{"apiVersion":"node.k8s.io/v1","kind":"RuntimeClass","metadata":{"name":%q},"handler":%q}`, name, handler) +} + +func seedDB(t *testing.T, statements []string) string { + t.Helper() + path := filepath.Join(t.TempDir(), "edgecore.db") + db, err := sql.Open("sqlite3", path) + if err != nil { + t.Fatalf("open seed db: %v", err) + } + defer db.Close() + for _, q := range statements { + if _, err := db.Exec(q); err != nil { + t.Fatalf("seed statement %q: %v", q, err) + } + } + return path +} + +func TestListFromMetaV2(t *testing.T) { + path := seedDB(t, []string{ + createMetaV2, + `INSERT INTO meta_v2 (key, group_version_resource, namespace, name, resource_version, value) VALUES ('/node.k8s.io/v1/runtimeclasses/null/kata', 'node.k8s.io/v1, Resource=runtimeclasses', 'null', 'kata', 7, '` + classJSON("kata", "kata") + `')`, + }) + store, err := OpenStore(path) + if err != nil { + t.Fatalf("OpenStore: %v", err) + } + defer store.Close() + classes, err := store.ListRuntimeClasses() + if err != nil { + t.Fatalf("ListRuntimeClasses: %v", err) + } + if len(classes) != 1 || classes[0].Name != "kata" || classes[0].Handler != "kata" { + t.Fatalf("unexpected classes: %+v", classes) + } +} + +func TestListFromLegacyMeta(t *testing.T) { + path := seedDB(t, []string{ + createMeta, + `INSERT INTO meta (key, type, value) VALUES ('null/runtimeclasses/runsc', 'runtimeclasses', '` + classJSON("runsc", "runsc") + `')`, + }) + store, err := OpenStore(path) + if err != nil { + t.Fatalf("OpenStore: %v", err) + } + defer store.Close() + classes, err := store.ListRuntimeClasses() + if err != nil { + t.Fatalf("ListRuntimeClasses: %v", err) + } + if len(classes) != 1 || classes[0].Name != "runsc" || classes[0].Handler != "runsc" { + t.Fatalf("unexpected classes: %+v", classes) + } +} + +func TestListBothTablesSortedAndDeduped(t *testing.T) { + path := seedDB(t, []string{ + createMetaV2, + createMeta, + `INSERT INTO meta_v2 (key, group_version_resource, namespace, name, resource_version, value) VALUES ('/node.k8s.io/v1/runtimeclasses/null/kata', 'node.k8s.io/v1, Resource=runtimeclasses', 'null', 'kata', 1, '` + classJSON("kata", "kata") + `')`, + `INSERT INTO meta_v2 (key, group_version_resource, namespace, name, resource_version, value) VALUES ('/node.k8s.io/v1/runtimeclasses/null/runsc', 'node.k8s.io/v1, Resource=runtimeclasses', 'null', 'runsc', 2, '` + classJSON("runsc", "runsc") + `')`, + `INSERT INTO meta (key, type, value) VALUES ('null/runtimeclasses/kata', 'runtimeclasses', '` + classJSON("kata", "kata") + `')`, + }) + store, err := OpenStore(path) + if err != nil { + t.Fatalf("OpenStore: %v", err) + } + defer store.Close() + classes, err := store.ListRuntimeClasses() + if err != nil { + t.Fatalf("ListRuntimeClasses: %v", err) + } + if len(classes) != 2 || classes[0].Name != "kata" || classes[1].Name != "runsc" { + t.Fatalf("unexpected classes: %+v", classes) + } +} + +func TestListIgnoresNonRuntimeClassObjects(t *testing.T) { + path := seedDB(t, []string{ + createMetaV2, + `INSERT INTO meta_v2 (key, group_version_resource, namespace, name, resource_version, value) VALUES ('/core/v1/secrets/default/mine', 'core/v1, Resource=secrets', 'default', 'mine', 3, '{"apiVersion":"v1","kind":"Secret","metadata":{"name":"mine"}}')`, + }) + store, err := OpenStore(path) + if err != nil { + t.Fatalf("OpenStore: %v", err) + } + defer store.Close() + classes, err := store.ListRuntimeClasses() + if err != nil { + t.Fatalf("ListRuntimeClasses: %v", err) + } + if len(classes) != 0 { + t.Fatalf("expected no classes, got %+v", classes) + } +} + +func TestListMissingTablesTreatedAsEmpty(t *testing.T) { + path := seedDB(t, []string{`CREATE TABLE unrelated (id INTEGER)`}) + store, err := OpenStore(path) + if err != nil { + t.Fatalf("OpenStore: %v", err) + } + defer store.Close() + classes, err := store.ListRuntimeClasses() + if err != nil { + t.Fatalf("ListRuntimeClasses: %v", err) + } + if len(classes) != 0 { + t.Fatalf("expected no classes, got %+v", classes) + } +} + +func TestOpenStoreMissingFile(t *testing.T) { + path := filepath.Join(t.TempDir(), "does-not-exist.db") + _, err := OpenStore(path) + if !errors.Is(err, ErrStoreNotFound) { + t.Fatalf("expected ErrStoreNotFound, got %v", err) + } +} diff --git a/pkg/bridge/lookup.go b/pkg/bridge/lookup.go new file mode 100644 index 0000000..a790aef --- /dev/null +++ b/pkg/bridge/lookup.go @@ -0,0 +1,78 @@ +// Package bridge mirrors the RuntimeClass resolution path used by the +// KubeEdge edged kubelet: a kubeclientbridge whose NodeV1 group is not +// wired to the metaclient, plus the kubelet's runtimeclass manager. +package bridge + +import ( + "errors" + "time" + + nodev1 "k8s.io/api/node/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes" + kubefake "k8s.io/client-go/kubernetes/fake" + "k8s.io/kubernetes/pkg/kubelet/runtimeclass" +) + +// bridgeClient mirrors kubeedge/edge/pkg/edged/kubeclientbridge: a fake +// clientset whose NodeV1 group is not overridden, so node.k8s.io resources +// are served from the empty in-memory fake store and never reach the +// metaclient or the local SQLite store. +type bridgeClient struct { + kubernetes.Interface +} + +// NewNaiveManager builds the resolution manager the same way edged builds +// it today: through the kubeclientbridge with NodeV1 backed by the fake +// store. Lookups against it always return NotFound. +func NewNaiveManager() (*runtimeclass.Manager, error) { + return NewManagerForClient(&bridgeClient{Interface: kubefake.NewSimpleClientset()}) +} + +// NewWiredManager builds the resolution manager as it would be built once +// NodeV1 is wired to the local store: a client seeded with the classes +// found locally. Lookups against it resolve class names to handlers. +func NewWiredManager(classes []nodev1.RuntimeClass) (*runtimeclass.Manager, error) { + objs := make([]runtime.Object, 0, len(classes)) + for i := range classes { + objs = append(objs, &classes[i]) + } + return NewManagerForClient(kubefake.NewSimpleClientset(objs...)) +} + +// NewManagerForClient builds a kubelet runtimeclass manager over the given +// client, starts its informer and waits for the cache to sync. +func NewManagerForClient(client kubernetes.Interface) (*runtimeclass.Manager, error) { + return NewManagerForClientWithTimeout(client, 10*time.Second) +} + +// NewManagerForClientWithTimeout is NewManagerForClient with a configurable +// cache-sync timeout. +func NewManagerForClientWithTimeout(client kubernetes.Interface, timeout time.Duration) (*runtimeclass.Manager, error) { + m := runtimeclass.NewManager(client) + stop := make(chan struct{}) + m.Start(stop) + synced := make(chan struct{}) + go func() { + m.WaitForCacheSync(stop) + close(synced) + }() + select { + case <-synced: + close(stop) + return m, nil + case <-time.After(timeout): + close(stop) + return nil, errors.New("timed out waiting for runtimeclass informer cache sync") + } +} + +// ResolveHandler applies the upstream kubelet resolution rules: +// no class name -> empty handler (default runtime); known class -> the +// class handler string; unknown class -> NotFound. +func ResolveHandler(m *runtimeclass.Manager, name string) (string, error) { + if name == "" { + return "", nil + } + return m.LookupRuntimeHandler(&name) +} diff --git a/pkg/bridge/lookup_test.go b/pkg/bridge/lookup_test.go new file mode 100644 index 0000000..a21b940 --- /dev/null +++ b/pkg/bridge/lookup_test.go @@ -0,0 +1,93 @@ +package bridge + +import ( + "errors" + "testing" + "time" + + nodev1 "k8s.io/api/node/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + kubefake "k8s.io/client-go/kubernetes/fake" + ktesting "k8s.io/client-go/testing" +) + +func runtimeClass(name, handler string) nodev1.RuntimeClass { + return nodev1.RuntimeClass{ + ObjectMeta: metav1.ObjectMeta{Name: name}, + Handler: handler, + } +} + +func TestNaiveLookupAlwaysNotFound(t *testing.T) { + m, err := NewNaiveManager() + if err != nil { + t.Fatalf("NewNaiveManager: %v", err) + } + _, err = ResolveHandler(m, "kata") + if !apierrors.IsNotFound(err) { + t.Fatalf("expected NotFound for naive lookup, got %v", err) + } +} + +func TestLookupRules(t *testing.T) { + m, err := NewWiredManager([]nodev1.RuntimeClass{ + runtimeClass("kata", "kata"), + runtimeClass("runsc", "runsc"), + }) + if err != nil { + t.Fatalf("NewWiredManager: %v", err) + } + + tests := []struct { + name string + class string + handler string + notFound bool + }{ + {name: "empty class name resolves to default handler", class: "", handler: ""}, + {name: "known class resolves to its handler", class: "kata", handler: "kata"}, + {name: "second known class resolves to its handler", class: "runsc", handler: "runsc"}, + {name: "unknown class returns NotFound", class: "nope", notFound: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + handler, err := ResolveHandler(m, tt.class) + if tt.notFound { + if !apierrors.IsNotFound(err) { + t.Fatalf("expected NotFound, got handler %q err %v", handler, err) + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if handler != tt.handler { + t.Fatalf("expected handler %q, got %q", tt.handler, handler) + } + }) + } +} + +func TestLookupWiredMissingClassNotFound(t *testing.T) { + m, err := NewWiredManager(nil) + if err != nil { + t.Fatalf("NewWiredManager: %v", err) + } + _, err = ResolveHandler(m, "kata") + if !apierrors.IsNotFound(err) { + t.Fatalf("expected NotFound, got %v", err) + } +} + +func TestNewManagerForClientSyncError(t *testing.T) { + fake := kubefake.NewSimpleClientset() + fake.PrependReactor("list", "runtimeclasses", func(action ktesting.Action) (bool, runtime.Object, error) { + return true, nil, errors.New("list exploded") + }) + _, err := NewManagerForClientWithTimeout(fake, time.Second) + if err == nil { + t.Fatal("expected sync error, got nil") + } +} diff --git a/pkg/config/edgecore.go b/pkg/config/edgecore.go new file mode 100644 index 0000000..54e149a --- /dev/null +++ b/pkg/config/edgecore.go @@ -0,0 +1,43 @@ +// Package config reads edge node configuration files. +package config + +import ( + "errors" + "fmt" + "os" + + "gopkg.in/yaml.v3" +) + +// ErrConfigNotFound is returned when the edgecore config file does not +// exist. +var ErrConfigNotFound = errors.New("edgecore config not found") + +// ReportEvent reads edged.reportEvent from the edgecore.yaml at the given +// path. It returns the value, whether the key was present, and an error. +func ReportEvent(path string) (bool, bool, error) { + data, err := os.ReadFile(path) + if err != nil { + if os.IsNotExist(err) { + return false, false, fmt.Errorf("%w: %s", ErrConfigNotFound, path) + } + return false, false, err + } + var tree map[string]interface{} + if err := yaml.Unmarshal(data, &tree); err != nil { + return false, false, fmt.Errorf("parse %s: %w", path, err) + } + edged, ok := tree["edged"].(map[string]interface{}) + if !ok { + return false, false, nil + } + value, ok := edged["reportEvent"] + if !ok { + return false, false, nil + } + report, ok := value.(bool) + if !ok { + return false, true, fmt.Errorf("edged.reportEvent is not a boolean in %s: %v", path, value) + } + return report, true, nil +} diff --git a/pkg/config/edgecore_test.go b/pkg/config/edgecore_test.go new file mode 100644 index 0000000..a00d688 --- /dev/null +++ b/pkg/config/edgecore_test.go @@ -0,0 +1,87 @@ +package config + +import ( + "errors" + "os" + "path/filepath" + "testing" +) + +const edgecoreWithReportEventTrue = `apiVersion: edgecore.config.kube.io/v1alpha2 +kind: EdgeCore +edged: + reportEvent: true + nodeStatusUpdateFrequency: 10 +` + +const edgecoreWithReportEventFalse = `apiVersion: edgecore.config.kube.io/v1alpha2 +kind: EdgeCore +edged: + reportEvent: false +` + +const edgecoreWithoutReportEvent = `apiVersion: edgecore.config.kube.io/v1alpha2 +kind: EdgeCore +edged: + nodeStatusUpdateFrequency: 10 +` + +func writeFixture(t *testing.T, content string) string { + t.Helper() + path := filepath.Join(t.TempDir(), "edgecore.yaml") + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + return path +} + +func TestReportEventTrue(t *testing.T) { + report, present, err := ReportEvent(writeFixture(t, edgecoreWithReportEventTrue)) + if err != nil { + t.Fatalf("ReportEvent: %v", err) + } + if !present || !report { + t.Fatalf("expected (true, true), got (%v, %v)", report, present) + } +} + +func TestReportEventFalse(t *testing.T) { + report, present, err := ReportEvent(writeFixture(t, edgecoreWithReportEventFalse)) + if err != nil { + t.Fatalf("ReportEvent: %v", err) + } + if !present || report { + t.Fatalf("expected (false, true), got (%v, %v)", report, present) + } +} + +func TestReportEventMissing(t *testing.T) { + report, present, err := ReportEvent(writeFixture(t, edgecoreWithoutReportEvent)) + if err != nil { + t.Fatalf("ReportEvent: %v", err) + } + if present || report { + t.Fatalf("expected (false, false), got (%v, %v)", report, present) + } +} + +func TestReportEventMissingFile(t *testing.T) { + _, _, err := ReportEvent(filepath.Join(t.TempDir(), "does-not-exist.yaml")) + if !errors.Is(err, ErrConfigNotFound) { + t.Fatalf("expected ErrConfigNotFound, got %v", err) + } +} + +func TestReportEventMalformed(t *testing.T) { + _, _, err := ReportEvent(writeFixture(t, "not: [valid")) + if err == nil { + t.Fatal("expected parse error, got nil") + } +} + +func TestReportEventNonBoolean(t *testing.T) { + _, _, err := ReportEvent(writeFixture(t, "edged:\n reportEvent: \"yes\"")) + if err == nil { + t.Fatal("expected type error, got nil") + } +} diff --git a/pkg/cri/containerd.go b/pkg/cri/containerd.go new file mode 100644 index 0000000..f5640b0 --- /dev/null +++ b/pkg/cri/containerd.go @@ -0,0 +1,70 @@ +// Package cri inspects the container runtime configuration on the node. +package cri + +import ( + "errors" + "fmt" + "os" + "sort" + + "github.com/pelletier/go-toml/v2" +) + +// ErrConfigNotFound is returned when the containerd config file does not +// exist. +var ErrConfigNotFound = errors.New("containerd config not found") + +// Containerd holds the CRI runtime handlers configured in containerd. +type Containerd struct { + path string + handlers map[string]struct{} +} + +// Load parses the containerd config at the given path and collects the CRI +// runtime handler names from +// [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.] (and the +// legacy [plugins.cri.containerd.runtimes.] spelling). +func Load(path string) (*Containerd, error) { + data, err := os.ReadFile(path) + if err != nil { + if os.IsNotExist(err) { + return nil, fmt.Errorf("%w: %s", ErrConfigNotFound, path) + } + return nil, err + } + var tree map[string]interface{} + if err := toml.Unmarshal(data, &tree); err != nil { + return nil, fmt.Errorf("parse %s: %w", path, err) + } + c := &Containerd{path: path, handlers: map[string]struct{}{}} + c.collectHandlers(tree) + return c, nil +} + +func (c *Containerd) collectHandlers(tree map[string]interface{}) { + plugins, _ := tree["plugins"].(map[string]interface{}) + for _, criName := range []string{"io.containerd.grpc.v1.cri", "cri"} { + criSection, _ := plugins[criName].(map[string]interface{}) + containerdSection, _ := criSection["containerd"].(map[string]interface{}) + runtimes, _ := containerdSection["runtimes"].(map[string]interface{}) + for handler := range runtimes { + c.handlers[handler] = struct{}{} + } + } +} + +// HandlerNames returns the configured runtime handler names, sorted. +func (c *Containerd) HandlerNames() []string { + names := make([]string, 0, len(c.handlers)) + for h := range c.handlers { + names = append(names, h) + } + sort.Strings(names) + return names +} + +// HasHandler reports whether the named runtime handler is configured. +func (c *Containerd) HasHandler(name string) bool { + _, ok := c.handlers[name] + return ok +} diff --git a/pkg/cri/containerd_test.go b/pkg/cri/containerd_test.go new file mode 100644 index 0000000..80a9f38 --- /dev/null +++ b/pkg/cri/containerd_test.go @@ -0,0 +1,83 @@ +package cri + +import ( + "errors" + "os" + "path/filepath" + "reflect" + "testing" +) + +func loadFixture(t *testing.T, content string) string { + t.Helper() + path := filepath.Join(t.TempDir(), "config.toml") + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + return path +} + +func TestLoadWithKata(t *testing.T) { + cc, err := Load(loadFixture(t, configWithKata)) + if err != nil { + t.Fatalf("Load: %v", err) + } + if !cc.HasHandler("kata") { + t.Fatalf("expected handler kata, got %v", cc.HandlerNames()) + } +} + +func TestLoadWithMultipleRuntimes(t *testing.T) { + cc, err := Load(loadFixture(t, configWithMultipleRuntimes)) + if err != nil { + t.Fatalf("Load: %v", err) + } + want := []string{"kata", "kata-qemu", "kata-qemu-tdx"} + if !reflect.DeepEqual(cc.HandlerNames(), want) { + t.Fatalf("expected %v, got %v", want, cc.HandlerNames()) + } +} + +func TestLoadWithoutRuntimes(t *testing.T) { + cc, err := Load(loadFixture(t, configWithoutRuntimes)) + if err != nil { + t.Fatalf("Load: %v", err) + } + if got := cc.HandlerNames(); len(got) != 0 { + t.Fatalf("expected no handlers, got %v", got) + } +} + +func TestLoadLegacyCRIName(t *testing.T) { + cc, err := Load(loadFixture(t, configWithLegacyCRIName)) + if err != nil { + t.Fatalf("Load: %v", err) + } + if !cc.HasHandler("runsc") { + t.Fatalf("expected handler runsc, got %v", cc.HandlerNames()) + } +} + +func TestLoadEmptyRuntimes(t *testing.T) { + cc, err := Load(loadFixture(t, configWithEmptyRuntimes)) + if err != nil { + t.Fatalf("Load: %v", err) + } + if got := cc.HandlerNames(); len(got) != 0 { + t.Fatalf("expected no handlers, got %v", got) + } +} + +func TestLoadMissingFile(t *testing.T) { + _, err := Load(filepath.Join(t.TempDir(), "does-not-exist.toml")) + if !errors.Is(err, ErrConfigNotFound) { + t.Fatalf("expected ErrConfigNotFound, got %v", err) + } +} + +func TestLoadMalformed(t *testing.T) { + _, err := Load(loadFixture(t, "this is not = toml [[")) + if err == nil { + t.Fatal("expected parse error, got nil") + } +} diff --git a/pkg/cri/containerd_testdata.go b/pkg/cri/containerd_testdata.go new file mode 100644 index 0000000..098c1ed --- /dev/null +++ b/pkg/cri/containerd_testdata.go @@ -0,0 +1,45 @@ +package cri + +// Fixture: a containerd config with the kata runtime handler registered. +const configWithKata = `version = 2 +root = "/var/lib/containerd" +state = "/run/containerd" + +[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata] +runtime_type = "io.containerd.kata.v2" +privileged_without_host_devices = true +` + +// Fixture: multiple kata handler names, including a quoted dotted name. +const configWithMultipleRuntimes = `version = 2 + +[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata] +runtime_type = "io.containerd.kata.v2" + +[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-qemu] +runtime_type = "io.containerd.kata.v2" + +[plugins."io.containerd.grpc.v1.cri".containerd.runtimes."kata-qemu-tdx"] +runtime_type = "io.containerd.kata.v2" +` + +// Fixture: a CRI plugin without any containerd.runtimes entries. +const configWithoutRuntimes = `version = 2 + +[plugins."io.containerd.grpc.v1.cri"] +sandbox_image = "registry.k8s.io/pause:3.10" +` + +// Fixture: the legacy "cri" plugin name spelling. +const configWithLegacyCRIName = `version = 2 + +[plugins.cri.containerd.runtimes.runsc] +runtime_type = "io.containerd.runsc.v1" +` + +// Fixture: an empty runtimes table. +const configWithEmptyRuntimes = `version = 2 + +[plugins."io.containerd.grpc.v1.cri".containerd] +runtimes = {} +` diff --git a/pkg/diag/diag.go b/pkg/diag/diag.go new file mode 100644 index 0000000..fc11a93 --- /dev/null +++ b/pkg/diag/diag.go @@ -0,0 +1,226 @@ +// Package diag orchestrates the four RuntimeClass path checks and renders +// the diagnosis report. +package diag + +import ( + "context" + "errors" + "fmt" + "strings" + "sync" + + nodev1 "k8s.io/api/node/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + + "github.com/kubeedge/runtimeclass-debugger/pkg/bridge" + "github.com/kubeedge/runtimeclass-debugger/pkg/config" + "github.com/kubeedge/runtimeclass-debugger/pkg/cri" + "github.com/kubeedge/runtimeclass-debugger/pkg/metaserver" +) + +// RuntimeClassLister is the MetaServer view of the transport check. +type RuntimeClassLister interface { + ListRuntimeClasses(ctx context.Context) ([]nodev1.RuntimeClass, error) +} + +// ClassSource is the local SQLite store view of the bridge check. +type ClassSource interface { + ListRuntimeClasses() ([]nodev1.RuntimeClass, error) +} + +// Diag runs the four checks. Its dependencies are interfaces so tests can +// substitute fakes. +type Diag struct { + MetaServer RuntimeClassLister + Store ClassSource + ContainerdPath string + EdgeCorePath string +} + +// New builds a Diag from a MetaServer client and the local file paths. +// The local store is opened lazily by the first check that needs it. +func New(meta RuntimeClassLister, dbPath, containerdPath, edgecorePath string) *Diag { + return &Diag{ + MetaServer: meta, + Store: &lazyStore{path: dbPath}, + ContainerdPath: containerdPath, + EdgeCorePath: edgecorePath, + } +} + +// Run executes all four checks in dependency order and returns the report. +// A non-empty scope limits the bridge and CRI checks to those class names; +// otherwise the classes served by MetaServer are used. +func (d *Diag) Run(ctx context.Context, scope []string) *Report { + transport, classes := d.CheckTransport(ctx) + checks := []CheckResult{transport} + if len(scope) == 0 { + for _, c := range classes { + scope = append(scope, c.Name) + } + } + checks = append(checks, d.CheckBridge(scope)) + checks = append(checks, d.CheckCRI(scope)) + checks = append(checks, d.CheckEvents()) + report := &Report{Checks: checks} + report.ExitCode = report.ComputeExitCode() + return report +} + +// CheckTransport: does the edge have RuntimeClass objects at all? +func (d *Diag) CheckTransport(ctx context.Context) (CheckResult, []nodev1.RuntimeClass) { + classes, err := d.MetaServer.ListRuntimeClasses(ctx) + if err != nil { + var authErr *metaserver.AuthRequiredError + var unreachErr *metaserver.UnreachableError + switch { + case errors.As(err, &authErr): + return CheckResult{ID: "transport", Status: StatusWarn, Detail: authErr.Reason}, nil + case errors.As(err, &unreachErr): + return CheckResult{ID: "transport", Status: StatusWarn, Detail: unreachErr.Reason + "; cannot inspect transport"}, nil + default: + return CheckResult{ID: "transport", Status: StatusWarn, Detail: err.Error() + "; cannot inspect transport"}, nil + } + } + if len(classes) == 0 { + return CheckResult{ID: "transport", Status: StatusFail, Detail: "MetaServer reachable but returns no RuntimeClass objects: objects never reached the edge (transport gap; matches the #7141 problem)"}, nil + } + names := make([]string, 0, len(classes)) + for _, c := range classes { + names = append(names, c.Name) + } + return CheckResult{ID: "transport", Status: StatusPass, Detail: fmt.Sprintf("%d RuntimeClass object(s) served by MetaServer (%s)", len(classes), strings.Join(names, ", "))}, classes +} + +// CheckBridge: can Edged resolve the class name to a handler? +func (d *Diag) CheckBridge(scope []string) CheckResult { + if len(scope) == 0 { + return CheckResult{ID: "bridge", Status: StatusSkip, Detail: "nothing to resolve; see the transport check"} + } + local, err := d.Store.ListRuntimeClasses() + if err != nil { + return CheckResult{ID: "bridge", Status: StatusWarn, Detail: fmt.Sprintf("cannot read the local store: %v", err)} + } + localByName := map[string]bool{} + for _, c := range local { + localByName[c.Name] = true + } + naive, err := bridge.NewNaiveManager() + if err != nil { + return CheckResult{ID: "bridge", Status: StatusWarn, Detail: fmt.Sprintf("cannot build the naive lookup: %v", err)} + } + wired, err := bridge.NewWiredManager(local) + if err != nil { + return CheckResult{ID: "bridge", Status: StatusWarn, Detail: fmt.Sprintf("cannot build the wired lookup: %v", err)} + } + lines := []string{} + status := StatusPass + for _, name := range scope { + _, naiveErr := bridge.ResolveHandler(naive, name) + handler, wiredErr := bridge.ResolveHandler(wired, name) + if wiredErr == nil { + lines = append(lines, fmt.Sprintf("class %q: naive lookup (kubelet bridge) -> %s; wired lookup -> handler %q", name, lookupOutcome("", naiveErr), handler)) + continue + } + status = StatusFail + if localByName[name] { + lines = append(lines, fmt.Sprintf("class %q exists in the local store but the resolution path fails: %v", name, wiredErr)) + } else { + lines = append(lines, fmt.Sprintf("class %q not resolvable: edge bridge gap (NodeV1 not wired to metaclient)", name)) + } + } + return CheckResult{ID: "bridge", Status: status, Detail: strings.Join(lines, "; ")} +} + +// CheckCRI: is the resolved handler configured in the container runtime? +func (d *Diag) CheckCRI(scope []string) CheckResult { + cc, err := cri.Load(d.ContainerdPath) + if err != nil { + if errors.Is(err, cri.ErrConfigNotFound) { + return CheckResult{ID: "cri", Status: StatusWarn, Detail: fmt.Sprintf("containerd config not found at %s; pass --containerd-config", d.ContainerdPath)} + } + return CheckResult{ID: "cri", Status: StatusWarn, Detail: fmt.Sprintf("cannot read containerd config: %v", err)} + } + if len(scope) == 0 { + return CheckResult{ID: "cri", Status: StatusPass, Detail: "no RuntimeClass handlers to verify"} + } + missing := []string{} + configured := []string{} + for _, name := range scope { + handler := d.handlerFor(name) + if cc.HasHandler(handler) { + configured = append(configured, handler) + } else { + missing = append(missing, handler) + } + } + if len(missing) > 0 { + return CheckResult{ID: "cri", Status: StatusFail, Detail: fmt.Sprintf("%s not configured in containerd; pod will stay Pending (FailedCreatePodSandBox)", quoteHandlers("handler", missing))} + } + return CheckResult{ID: "cri", Status: StatusPass, Detail: fmt.Sprintf("%s configured in containerd", quoteHandlers("handler", configured))} +} + +// CheckEvents: would a failure even be visible to the user? +func (d *Diag) CheckEvents() CheckResult { + report, present, err := config.ReportEvent(d.EdgeCorePath) + if err != nil { + if errors.Is(err, config.ErrConfigNotFound) { + return CheckResult{ID: "events", Status: StatusWarn, Detail: fmt.Sprintf("edgecore.yaml not found at %s", d.EdgeCorePath)} + } + return CheckResult{ID: "events", Status: StatusWarn, Detail: fmt.Sprintf("cannot read edgecore.yaml: %v", err)} + } + if report { + return CheckResult{ID: "events", Status: StatusPass, Detail: "reportEvent is true; pod events are reported to the cloud"} + } + if !present { + return CheckResult{ID: "events", Status: StatusWarn, Detail: "edged.reportEvent is absent (defaults to false): pod failures may not surface as events. Enable edgeCore.reportEvent"} + } + return CheckResult{ID: "events", Status: StatusWarn, Detail: "reportEvent is false: pod failures may not surface as events. Enable edgeCore.reportEvent"} +} + +func (d *Diag) handlerFor(name string) string { + classes, err := d.Store.ListRuntimeClasses() + if err != nil { + return name + } + for _, c := range classes { + if c.Name == name && c.Handler != "" { + return c.Handler + } + } + return name +} + +func lookupOutcome(handler string, err error) string { + if err == nil { + return fmt.Sprintf("handler %q", handler) + } + if apierrors.IsNotFound(err) { + return "NotFound" + } + return err.Error() +} + +func quoteHandlers(singular string, names []string) string { + if len(names) == 1 { + return fmt.Sprintf("%s %q", singular, names[0]) + } + return fmt.Sprintf("%ss %q", singular, strings.Join(names, "\", \"")) +} + +type lazyStore struct { + path string + once sync.Once + store *bridge.Store + err error +} + +func (l *lazyStore) ListRuntimeClasses() ([]nodev1.RuntimeClass, error) { + l.once.Do(func() { + l.store, l.err = bridge.OpenStore(l.path) + }) + if l.err != nil { + return nil, l.err + } + return l.store.ListRuntimeClasses() +} diff --git a/pkg/diag/diag_test.go b/pkg/diag/diag_test.go new file mode 100644 index 0000000..3dd9f17 --- /dev/null +++ b/pkg/diag/diag_test.go @@ -0,0 +1,291 @@ +package diag + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + + nodev1 "k8s.io/api/node/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/kubeedge/runtimeclass-debugger/pkg/metaserver" +) + +type fakeMeta struct { + classes []nodev1.RuntimeClass + err error +} + +func (f *fakeMeta) ListRuntimeClasses(context.Context) ([]nodev1.RuntimeClass, error) { + return f.classes, f.err +} + +type fakeStore struct { + classes []nodev1.RuntimeClass + err error +} + +func (f *fakeStore) ListRuntimeClasses() ([]nodev1.RuntimeClass, error) { + return f.classes, f.err +} + +func runtimeClass(name, handler string) nodev1.RuntimeClass { + return nodev1.RuntimeClass{ + ObjectMeta: metav1.ObjectMeta{Name: name}, + Handler: handler, + } +} + +func writeFile(t *testing.T, dir, name, content string) string { + t.Helper() + path := filepath.Join(dir, name) + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + return path +} + +func withPaths(t *testing.T, d *Diag) *Diag { + dir := t.TempDir() + d.ContainerdPath = writeFile(t, dir, "config.toml", `version = 2 + +[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata] +runtime_type = "io.containerd.kata.v2" +`) + d.EdgeCorePath = writeFile(t, dir, "edgecore.yaml", "edged:\n reportEvent: true\n") + return d +} + +func TestCheckTransport(t *testing.T) { + t.Run("pass with classes", func(t *testing.T) { + d := &Diag{MetaServer: &fakeMeta{classes: []nodev1.RuntimeClass{runtimeClass("kata", "kata")}}} + res, classes := d.CheckTransport(context.Background()) + if res.Status != StatusPass || len(classes) != 1 { + t.Fatalf("expected pass with 1 class, got %+v", res) + } + }) + t.Run("fail on empty list", func(t *testing.T) { + d := &Diag{MetaServer: &fakeMeta{}} + res, classes := d.CheckTransport(context.Background()) + if res.Status != StatusFail || len(classes) != 0 { + t.Fatalf("expected fail with 0 classes, got %+v", res) + } + if !strings.Contains(res.Detail, "transport gap") { + t.Fatalf("expected transport gap reason, got %q", res.Detail) + } + }) + t.Run("warn on auth required", func(t *testing.T) { + d := &Diag{MetaServer: &fakeMeta{err: &metaserver.AuthRequiredError{Reason: "requires certs"}}} + res, _ := d.CheckTransport(context.Background()) + if res.Status != StatusWarn { + t.Fatalf("expected warn, got %+v", res) + } + }) + t.Run("warn on unreachable", func(t *testing.T) { + d := &Diag{MetaServer: &fakeMeta{err: &metaserver.UnreachableError{Reason: "down"}}} + res, _ := d.CheckTransport(context.Background()) + if res.Status != StatusWarn { + t.Fatalf("expected warn, got %+v", res) + } + }) + t.Run("warn on unexpected error", func(t *testing.T) { + d := &Diag{MetaServer: &fakeMeta{err: errBoom}} + res, _ := d.CheckTransport(context.Background()) + if res.Status != StatusWarn { + t.Fatalf("expected warn, got %+v", res) + } + }) +} + +func TestCheckBridge(t *testing.T) { + t.Run("skip with no scope", func(t *testing.T) { + d := &Diag{Store: &fakeStore{}} + res := d.CheckBridge(nil) + if res.Status != StatusSkip { + t.Fatalf("expected skip, got %+v", res) + } + }) + t.Run("pass when class resolves via local store", func(t *testing.T) { + d := &Diag{Store: &fakeStore{classes: []nodev1.RuntimeClass{runtimeClass("kata", "kata")}}} + res := d.CheckBridge([]string{"kata"}) + if res.Status != StatusPass { + t.Fatalf("expected pass, got %+v", res) + } + if !strings.Contains(res.Detail, "wired lookup") || !strings.Contains(res.Detail, "NotFound") { + t.Fatalf("expected both lookup outcomes in detail, got %q", res.Detail) + } + }) + t.Run("fail when class is missing from local store", func(t *testing.T) { + d := &Diag{Store: &fakeStore{}} + res := d.CheckBridge([]string{"kata"}) + if res.Status != StatusFail { + t.Fatalf("expected fail, got %+v", res) + } + if !strings.Contains(res.Detail, "edge bridge gap") { + t.Fatalf("expected edge bridge gap reason, got %q", res.Detail) + } + }) + t.Run("warn when local store unreadable", func(t *testing.T) { + d := &Diag{Store: &fakeStore{err: errBoom}} + res := d.CheckBridge([]string{"kata"}) + if res.Status != StatusWarn { + t.Fatalf("expected warn, got %+v", res) + } + }) +} + +func TestCheckCRI(t *testing.T) { + t.Run("pass when handler configured", func(t *testing.T) { + d := withPaths(t, &Diag{Store: &fakeStore{classes: []nodev1.RuntimeClass{runtimeClass("kata", "kata")}}}) + res := d.CheckCRI([]string{"kata"}) + if res.Status != StatusPass { + t.Fatalf("expected pass, got %+v", res) + } + }) + t.Run("pass with no scope", func(t *testing.T) { + d := withPaths(t, &Diag{}) + res := d.CheckCRI(nil) + if res.Status != StatusPass { + t.Fatalf("expected pass, got %+v", res) + } + }) + t.Run("fail when handler missing", func(t *testing.T) { + d := withPaths(t, &Diag{Store: &fakeStore{classes: []nodev1.RuntimeClass{runtimeClass("kata", "kata")}}}) + res := d.CheckCRI([]string{"runsc"}) + if res.Status != StatusFail { + t.Fatalf("expected fail, got %+v", res) + } + if !strings.Contains(res.Detail, "not configured in containerd") { + t.Fatalf("expected not-configured reason, got %q", res.Detail) + } + }) + t.Run("warn when config missing", func(t *testing.T) { + d := &Diag{Store: &fakeStore{classes: []nodev1.RuntimeClass{runtimeClass("kata", "kata")}}, ContainerdPath: filepath.Join(t.TempDir(), "nope.toml")} + res := d.CheckCRI([]string{"kata"}) + if res.Status != StatusWarn { + t.Fatalf("expected warn, got %+v", res) + } + }) +} + +func TestCheckEvents(t *testing.T) { + t.Run("pass when reportEvent true", func(t *testing.T) { + d := withPaths(t, &Diag{}) + if res := d.CheckEvents(); res.Status != StatusPass { + t.Fatalf("expected pass, got %+v", res) + } + }) + t.Run("warn when reportEvent false", func(t *testing.T) { + dir := t.TempDir() + path := writeFile(t, dir, "edgecore.yaml", "edged:\n reportEvent: false\n") + d := &Diag{EdgeCorePath: path} + res := d.CheckEvents() + if res.Status != StatusWarn { + t.Fatalf("expected warn, got %+v", res) + } + }) + t.Run("warn when reportEvent absent", func(t *testing.T) { + dir := t.TempDir() + path := writeFile(t, dir, "edgecore.yaml", "edged:\n nodeStatusUpdateFrequency: 10\n") + d := &Diag{EdgeCorePath: path} + res := d.CheckEvents() + if res.Status != StatusWarn { + t.Fatalf("expected warn, got %+v", res) + } + }) + t.Run("warn when config missing", func(t *testing.T) { + d := &Diag{EdgeCorePath: filepath.Join(t.TempDir(), "nope.yaml")} + res := d.CheckEvents() + if res.Status != StatusWarn { + t.Fatalf("expected warn, got %+v", res) + } + }) +} + +func TestRunHealthy(t *testing.T) { + d := &Diag{ + MetaServer: &fakeMeta{classes: []nodev1.RuntimeClass{runtimeClass("kata", "kata")}}, + Store: &fakeStore{classes: []nodev1.RuntimeClass{runtimeClass("kata", "kata")}}, + } + d = withPaths(t, d) + report := d.Run(context.Background(), nil) + want := map[string]Status{"transport": StatusPass, "bridge": StatusPass, "cri": StatusPass, "events": StatusPass} + if len(report.Checks) != 4 { + t.Fatalf("expected 4 checks, got %d", len(report.Checks)) + } + for _, c := range report.Checks { + if c.Status != want[c.ID] { + t.Fatalf("check %s: expected %s, got %+v", c.ID, want[c.ID], c) + } + } + if report.ExitCode != ExitOK { + t.Fatalf("expected exit 0, got %d", report.ExitCode) + } +} + +func TestRunStockNode(t *testing.T) { + dir := t.TempDir() + d := &Diag{ + MetaServer: &fakeMeta{}, + Store: &fakeStore{}, + ContainerdPath: filepath.Join(dir, "missing.toml"), + EdgeCorePath: writeFile(t, dir, "edgecore.yaml", "edged:\n reportEvent: false\n"), + } + report := d.Run(context.Background(), nil) + want := map[string]Status{"transport": StatusFail, "bridge": StatusSkip, "cri": StatusWarn, "events": StatusWarn} + for _, c := range report.Checks { + if c.Status != want[c.ID] { + t.Fatalf("check %s: expected %s, got %+v", c.ID, want[c.ID], c) + } + } + if report.ExitCode != ExitFail { + t.Fatalf("expected exit 1, got %d", report.ExitCode) + } +} + +func TestRunWarnOnly(t *testing.T) { + dir := t.TempDir() + d := &Diag{ + MetaServer: &fakeMeta{err: &metaserver.UnreachableError{Reason: "down"}}, + Store: &fakeStore{}, + ContainerdPath: filepath.Join(dir, "missing.toml"), + EdgeCorePath: writeFile(t, dir, "edgecore.yaml", "edged:\n reportEvent: false\n"), + } + report := d.Run(context.Background(), nil) + if report.ExitCode != ExitWarn { + t.Fatalf("expected exit 2, got %d", report.ExitCode) + } +} + +func TestReportRendering(t *testing.T) { + r := &Report{ + Checks: []CheckResult{ + {ID: "transport", Status: StatusPass, Detail: "1 RuntimeClass object(s) served by MetaServer (kata)"}, + {ID: "bridge", Status: StatusFail, Detail: "class \"kata\" not resolvable: edge bridge gap (NodeV1 not wired to metaclient)"}, + {ID: "events", Status: StatusWarn, Detail: "reportEvent is false"}, + }, + } + r.ExitCode = r.ComputeExitCode() + if r.ExitCode != ExitFail { + t.Fatalf("expected exit 1, got %d", r.ExitCode) + } + table := r.Table() + if !strings.Contains(table, "[PASS] transport ") || !strings.Contains(table, "Result: 1 pass, 1 fail, 1 warn") { + t.Fatalf("unexpected table:\n%s", table) + } + jsonOut, err := r.JSON() + if err != nil { + t.Fatalf("JSON: %v", err) + } + if !strings.Contains(jsonOut, "\"exitCode\": 1") || !strings.Contains(jsonOut, "\"status\": \"fail\"") { + t.Fatalf("unexpected json:\n%s", jsonOut) + } +} + +var errBoom = contextError("boom") + +type contextError string + +func (e contextError) Error() string { return string(e) } diff --git a/pkg/diag/report.go b/pkg/diag/report.go new file mode 100644 index 0000000..f58c266 --- /dev/null +++ b/pkg/diag/report.go @@ -0,0 +1,57 @@ +package diag + +import ( + "encoding/json" + "fmt" + "strings" +) + +const reportTitle = "RuntimeClass Edge Diagnostics" + +// Table renders the human-readable report. +func (r *Report) Table() string { + var b strings.Builder + b.WriteString(reportTitle + "\n") + b.WriteString(strings.Repeat("=", len(reportTitle)) + "\n") + for _, c := range r.Checks { + fmt.Fprintf(&b, "[%s] %-10s %s\n", strings.ToUpper(string(c.Status)), c.ID, c.Detail) + } + b.WriteString("\nResult: " + r.summary() + "\n") + return b.String() +} + +func (r *Report) summary() string { + counts := map[Status]int{} + for _, c := range r.Checks { + counts[c.Status]++ + } + var parts []string + for _, s := range []Status{StatusPass, StatusFail, StatusWarn, StatusSkip} { + if n := counts[s]; n > 0 { + parts = append(parts, fmt.Sprintf("%d %s", n, s)) + } + } + return strings.Join(parts, ", ") +} + +// JSON renders the machine-readable report. +func (r *Report) JSON() (string, error) { + type checkJSON struct { + ID string `json:"id"` + Status string `json:"status"` + Detail string `json:"detail"` + } + type reportJSON struct { + Checks []checkJSON `json:"checks"` + ExitCode int `json:"exitCode"` + } + out := reportJSON{ExitCode: r.ExitCode} + for _, c := range r.Checks { + out.Checks = append(out.Checks, checkJSON{ID: c.ID, Status: string(c.Status), Detail: c.Detail}) + } + b, err := json.MarshalIndent(out, "", " ") + if err != nil { + return "", err + } + return string(b), nil +} diff --git a/pkg/diag/result.go b/pkg/diag/result.go new file mode 100644 index 0000000..5896527 --- /dev/null +++ b/pkg/diag/result.go @@ -0,0 +1,53 @@ +package diag + +// Status is the outcome of a single check. +type Status string + +const ( + StatusPass Status = "pass" + StatusFail Status = "fail" + StatusWarn Status = "warn" + StatusSkip Status = "skip" +) + +const ( + ExitOK = 0 + ExitFail = 1 + ExitWarn = 2 + ExitError = 3 +) + +// CheckResult is the outcome of a single check. +type CheckResult struct { + ID string + Status Status + Detail string +} + +// Report is the full set of check results for one run. +type Report struct { + Checks []CheckResult + ExitCode int +} + +// ComputeExitCode derives the exit code from the checks: +// 0 = all pass; 1 = any fail; 2 = any warn and no fail. +func (r *Report) ComputeExitCode() int { + hasFail, hasWarn := false, false + for _, c := range r.Checks { + switch c.Status { + case StatusFail: + hasFail = true + case StatusWarn: + hasWarn = true + } + } + switch { + case hasFail: + return ExitFail + case hasWarn: + return ExitWarn + default: + return ExitOK + } +} diff --git a/pkg/metaserver/client.go b/pkg/metaserver/client.go new file mode 100644 index 0000000..59e5302 --- /dev/null +++ b/pkg/metaserver/client.go @@ -0,0 +1,174 @@ +// Package metaserver queries the edge node MetaServer HTTP API. +package metaserver + +import ( + "context" + "crypto/tls" + "crypto/x509" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "net/url" + "os" + "sort" + "strings" + "time" + + nodev1 "k8s.io/api/node/v1" +) + +const runtimeClassPath = "/apis/node.k8s.io/v1/runtimeclasses" + +// AuthRequiredError indicates the MetaServer requires authentication the +// caller did not provide. +type AuthRequiredError struct { + Reason string +} + +func (e *AuthRequiredError) Error() string { return e.Reason } + +// UnreachableError indicates the MetaServer could not be reached at all. +type UnreachableError struct { + Reason string +} + +func (e *UnreachableError) Error() string { return e.Reason } + +// Client queries a MetaServer. +type Client struct { + BaseURL string + Timeout time.Duration + CertFile string + KeyFile string + CAFile string +} + +// ListRuntimeClasses fetches the RuntimeClass objects served by the +// MetaServer. Auth is detected and handled in three modes: plain HTTP when +// no auth is configured, mTLS retry when client certificates are provided, +// and an AuthRequiredError when the server demands certificates that were +// not provided. +func (c *Client) ListRuntimeClasses(ctx context.Context) ([]nodev1.RuntimeClass, error) { + endpoint := c.endpoint() + useTLS := strings.HasPrefix(c.BaseURL, "https://") + + body, code, err := c.get(ctx, c.httpClient(useTLS), endpoint) + if err == nil { + return c.handleStatus(body, code) + } + + if c.hasCerts() { + body, code, retryErr := c.get(ctx, c.httpClient(true), endpoint) + if retryErr == nil { + return c.handleStatus(body, code) + } + err = retryErr + } + + if c.speaksTLS(endpoint) { + if c.hasCerts() { + return nil, &AuthRequiredError{Reason: fmt.Sprintf("MetaServer rejected client certificates: %v", err)} + } + return nil, &AuthRequiredError{Reason: "MetaServer requires TLS with client certificates; pass --cert-file/--key-file/--ca-file"} + } + return nil, &UnreachableError{Reason: fmt.Sprintf("MetaServer not reachable at %s: %v", c.BaseURL, err)} +} + +func (c *Client) endpoint() string { + base := strings.TrimRight(c.BaseURL, "/") + if !strings.HasPrefix(base, "http://") && !strings.HasPrefix(base, "https://") { + base = "http://" + base + } + return base + runtimeClassPath +} + +func (c *Client) get(ctx context.Context, client *http.Client, endpoint string) ([]byte, int, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + return nil, 0, err + } + req.Header.Set("Accept", "application/json") + resp, err := client.Do(req) + if err != nil { + return nil, 0, err + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, resp.StatusCode, err + } + return body, resp.StatusCode, nil +} + +func (c *Client) handleStatus(body []byte, code int) ([]nodev1.RuntimeClass, error) { + switch code { + case http.StatusOK: + return parseList(body) + case http.StatusUnauthorized, http.StatusForbidden: + return nil, &AuthRequiredError{Reason: fmt.Sprintf("MetaServer requires authentication (HTTP %d); pass --cert-file/--key-file/--ca-file", code)} + default: + return nil, fmt.Errorf("MetaServer returned HTTP %d: %.200s", code, strings.TrimSpace(string(body))) + } +} + +func parseList(body []byte) ([]nodev1.RuntimeClass, error) { + var list nodev1.RuntimeClassList + if err := json.Unmarshal(body, &list); err != nil { + return nil, fmt.Errorf("unexpected MetaServer response: %v", err) + } + sort.Slice(list.Items, func(i, j int) bool { return list.Items[i].Name < list.Items[j].Name }) + return list.Items, nil +} + +func (c *Client) hasCerts() bool { + return c.CertFile != "" && c.KeyFile != "" +} + +func (c *Client) httpClient(useTLS bool) *http.Client { + if !useTLS { + return &http.Client{Timeout: c.Timeout} + } + tlsConfig := &tls.Config{MinVersion: tls.VersionTLS12} + if c.CAFile != "" { + pool, err := x509.SystemCertPool() + if err != nil { + pool = x509.NewCertPool() + } + if ca, err := os.ReadFile(c.CAFile); err == nil { + pool.AppendCertsFromPEM(ca) + } + tlsConfig.RootCAs = pool + } + if c.hasCerts() { + if cert, err := tls.LoadX509KeyPair(c.CertFile, c.KeyFile); err == nil { + tlsConfig.Certificates = []tls.Certificate{cert} + } + } + return &http.Client{ + Timeout: c.Timeout, + Transport: &http.Transport{ + TLSClientConfig: tlsConfig, + }, + } +} + +// speaksTLS probes whether the endpoint answers TLS handshakes, which +// indicates a MetaServer that requires client certificates. +func (c *Client) speaksTLS(endpoint string) bool { + u, err := url.Parse(endpoint) + if err != nil { + return false + } + dialer := &net.Dialer{Timeout: c.Timeout} + conn, err := tls.DialWithDialer(dialer, "tcp", u.Host, &tls.Config{ + InsecureSkipVerify: true, + MinVersion: tls.VersionTLS12, + }) + if err != nil { + return false + } + conn.Close() + return true +} diff --git a/pkg/metaserver/client_test.go b/pkg/metaserver/client_test.go new file mode 100644 index 0000000..21b6c0b --- /dev/null +++ b/pkg/metaserver/client_test.go @@ -0,0 +1,278 @@ +package metaserver + +import ( + "context" + "crypto/rand" + "crypto/rsa" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/json" + "encoding/pem" + "errors" + "math/big" + "net" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + "time" +) + +func runtimeClassList(names ...string) []byte { + items := []map[string]interface{}{} + for _, n := range names { + items = append(items, map[string]interface{}{ + "apiVersion": "node.k8s.io/v1", + "kind": "RuntimeClass", + "metadata": map[string]interface{}{"name": n}, + "handler": n, + }) + } + b, _ := json.Marshal(map[string]interface{}{ + "apiVersion": "node.k8s.io/v1", + "kind": "RuntimeClassList", + "metadata": map[string]interface{}{}, + "items": items, + }) + return b +} + +func newClient(serverURL string) *Client { + return &Client{BaseURL: serverURL, Timeout: 5 * time.Second} +} + +func TestListRuntimeClasses(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/apis/node.k8s.io/v1/runtimeclasses" { + w.WriteHeader(http.StatusNotFound) + return + } + _, _ = w.Write(runtimeClassList("runsc", "kata")) + })) + defer server.Close() + + classes, err := newClient(server.URL).ListRuntimeClasses(context.Background()) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(classes) != 2 || classes[0].Name != "kata" || classes[1].Name != "runsc" { + t.Fatalf("expected sorted [kata runsc], got %+v", classes) + } +} + +func TestListEmptyList(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write(runtimeClassList()) + })) + defer server.Close() + + classes, err := newClient(server.URL).ListRuntimeClasses(context.Background()) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(classes) != 0 { + t.Fatalf("expected no classes, got %+v", classes) + } +} + +func TestServerErrorNotAuthNotUnreachable(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte("boom")) + })) + defer server.Close() + + _, err := newClient(server.URL).ListRuntimeClasses(context.Background()) + if err == nil { + t.Fatal("expected error, got nil") + } + var authErr *AuthRequiredError + var unreachErr *UnreachableError + if errors.As(err, &authErr) || errors.As(err, &unreachErr) { + t.Fatalf("expected plain error, got %T: %v", err, err) + } +} + +func TestAuthRequiredOnHTTPStatus(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusForbidden) + })) + defer server.Close() + + _, err := newClient(server.URL).ListRuntimeClasses(context.Background()) + var authErr *AuthRequiredError + if !errors.As(err, &authErr) { + t.Fatalf("expected AuthRequiredError, got %v", err) + } +} + +func TestUnreachable(t *testing.T) { + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + addr := l.Addr().String() + l.Close() + + _, err = newClient("http://" + addr).ListRuntimeClasses(context.Background()) + var unreachErr *UnreachableError + if !errors.As(err, &unreachErr) { + t.Fatalf("expected UnreachableError, got %v", err) + } +} + +func TestMalformedResponse(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("not json")) + })) + defer server.Close() + + _, err := newClient(server.URL).ListRuntimeClasses(context.Background()) + if err == nil { + t.Fatal("expected error, got nil") + } +} + +type testPKI struct { + caPath string + clientCertPath string + clientKeyPath string + serverCert tls.Certificate +} + +func generateTestPKI(t *testing.T) *testPKI { + t.Helper() + dir := t.TempDir() + now := time.Now() + + caKey, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + t.Fatal(err) + } + caTmpl := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "runtimeclass-debugger-test-ca"}, + NotBefore: now.Add(-time.Hour), + NotAfter: now.Add(24 * time.Hour), + IsCA: true, + KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageDigitalSignature, + BasicConstraintsValid: true, + } + caDER, err := x509.CreateCertificate(rand.Reader, caTmpl, caTmpl, &caKey.PublicKey, caKey) + if err != nil { + t.Fatal(err) + } + caCert, err := x509.ParseCertificate(caDER) + if err != nil { + t.Fatal(err) + } + caPath := filepath.Join(dir, "ca.crt") + if err := os.WriteFile(caPath, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: caDER}), 0o600); err != nil { + t.Fatal(err) + } + + signLeaf := func(cn string, ip net.IP, usage []x509.ExtKeyUsage) (tls.Certificate, string, string) { + key, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + t.Fatal(err) + } + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(2), + Subject: pkix.Name{CommonName: cn}, + NotBefore: now.Add(-time.Hour), + NotAfter: now.Add(24 * time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment, + ExtKeyUsage: usage, + } + if ip != nil { + tmpl.IPAddresses = []net.IP{ip} + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, caCert, &key.PublicKey, caKey) + if err != nil { + t.Fatal(err) + } + certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}) + keyDER, err := x509.MarshalPKCS8PrivateKey(key) + if err != nil { + t.Fatal(err) + } + keyPEM := pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyDER}) + certPath := filepath.Join(dir, cn+".crt") + keyPath := filepath.Join(dir, cn+".key") + if err := os.WriteFile(certPath, certPEM, 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(keyPath, keyPEM, 0o600); err != nil { + t.Fatal(err) + } + cert, err := tls.X509KeyPair(certPEM, keyPEM) + if err != nil { + t.Fatal(err) + } + return cert, certPath, keyPath + } + + serverCert, _, _ := signLeaf("metaserver", net.ParseIP("127.0.0.1"), []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}) + _, clientCertPath, clientKeyPath := signLeaf("client", nil, []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}) + return &testPKI{caPath: caPath, clientCertPath: clientCertPath, clientKeyPath: clientKeyPath, serverCert: serverCert} +} + +func newTLSMetaServer(t *testing.T, pki *testPKI) *httptest.Server { + t.Helper() + caPEM, err := os.ReadFile(pki.caPath) + if err != nil { + t.Fatal(err) + } + pool := x509.NewCertPool() + pool.AppendCertsFromPEM(caPEM) + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if len(r.TLS.PeerCertificates) == 0 { + w.WriteHeader(http.StatusUnauthorized) + return + } + _, _ = w.Write(runtimeClassList("kata")) + }) + server := httptest.NewUnstartedServer(handler) + server.TLS = &tls.Config{ + Certificates: []tls.Certificate{pki.serverCert}, + ClientCAs: pool, + ClientAuth: tls.VerifyClientCertIfGiven, + MinVersion: tls.VersionTLS12, + } + server.StartTLS() + return server +} + +func TestAuthTLSWithoutCertsWarns(t *testing.T) { + server := newTLSMetaServer(t, generateTestPKI(t)) + defer server.Close() + + _, err := newClient(server.URL).ListRuntimeClasses(context.Background()) + var authErr *AuthRequiredError + if !errors.As(err, &authErr) { + t.Fatalf("expected AuthRequiredError, got %v", err) + } +} + +func TestAuthTLSWithCertsSucceeds(t *testing.T) { + pki := generateTestPKI(t) + server := newTLSMetaServer(t, pki) + defer server.Close() + + client := &Client{ + BaseURL: server.URL, + Timeout: 5 * time.Second, + CertFile: pki.clientCertPath, + KeyFile: pki.clientKeyPath, + CAFile: pki.caPath, + } + classes, err := client.ListRuntimeClasses(context.Background()) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(classes) != 1 || classes[0].Name != "kata" { + t.Fatalf("expected [kata], got %+v", classes) + } +}