From 445b71cc4c90058c858d4c9f64c0ef5d23487506 Mon Sep 17 00:00:00 2001 From: Dominik Richter Date: Mon, 16 Oct 2023 23:40:29 -0700 Subject: [PATCH 1/2] =?UTF-8?q?=F0=9F=A7=B9=20make=20providers/defaults?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Dominik Richter --- .github/actions/spelling/expect.txt | 1 + Makefile | 4 + providers-sdk/v1/util/defaults/defaults.go | 170 +++++++++++++++++++++ providers/defaults.go | 147 +++++++++++++----- providers/defaults_shared.go | 30 ++++ 5 files changed, 313 insertions(+), 39 deletions(-) create mode 100644 providers-sdk/v1/util/defaults/defaults.go create mode 100644 providers/defaults_shared.go diff --git a/.github/actions/spelling/expect.txt b/.github/actions/spelling/expect.txt index 0c8890cb75..ffc746db45 100644 --- a/.github/actions/spelling/expect.txt +++ b/.github/actions/spelling/expect.txt @@ -15,6 +15,7 @@ customresources datapath Ddos dfw +DIRECTORYID dlq ekm elbv diff --git a/Makefile b/Makefile index 9db9273d66..fefd45e24a 100644 --- a/Makefile +++ b/Makefile @@ -158,6 +158,10 @@ providers/proto: providers/config: go run ./providers-sdk/v1/util/configure/configure.go -f providers.yaml -o providers/builtin_dev.go +.PHONY: providers/defaults +providers/defaults: + go run ./providers-sdk/v1/util/defaults/defaults.go -o providers/defaults.go + .PHONY: providers/lr providers/lr: go build -o lr ./providers-sdk/v1/lr/cli/main.go diff --git a/providers-sdk/v1/util/defaults/defaults.go b/providers-sdk/v1/util/defaults/defaults.go new file mode 100644 index 0000000000..9052f0cf34 --- /dev/null +++ b/providers-sdk/v1/util/defaults/defaults.go @@ -0,0 +1,170 @@ +// Copyright (c) Mondoo, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package main + +import ( + "encoding/json" + "errors" + "fmt" + "go/format" + "os" + "os/exec" + "path/filepath" + "sort" + "strings" + + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" + "github.com/spf13/cobra" + "go.mondoo.com/cnquery/v9/logger" + "go.mondoo.com/cnquery/v9/providers-sdk/v1/plugin" +) + +var rootCmd = &cobra.Command{ + Use: "defaults [-o file]", + Short: "configure provider defaults", + Run: func(cmd *cobra.Command, args []string) { + outPath, err := cmd.Flags().GetString("output") + if err != nil { + log.Fatal().Err(err).Msg("Can't get --output") + } + + providers, err := getProviders() + if err != nil { + log.Fatal().Err(err).Msg("failed to get providers") + } + + var configs []*plugin.Provider + for name, path := range providers { + provider, err := getConfig(name, path) + if err != nil { + log.Error().Err(err).Str("provider", name).Msg("failed to get provider config") + } else { + configs = append(configs, provider) + } + } + + sort.Slice(configs, func(i, j int) bool { + return configs[i].Name < configs[j].Name + }) + + res, err := goGen(configs) + if err != nil { + log.Fatal().Err(err).Msg("failed to generate go code for provideer defaults") + } + + if err = os.WriteFile(outPath, res, 0o644); err != nil { + log.Fatal().Err(err).Str("path", outPath).Msg("failed to save file") + } else { + log.Info().Str("path", outPath).Msg("updated") + } + }, +} + +func goGen(configs []*plugin.Provider) ([]byte, error) { + var body strings.Builder + + for i := range configs { + conf := configs[i] + + var conns strings.Builder + for j := range conf.Connectors { + conn := conf.Connectors[j] + conns.WriteString(fmt.Sprintf(connectorTemplate, conn.Name, conn.Use, conn.Short)) + } + body.WriteString(fmt.Sprintf(providerTemplate, conf.Name, conf.Name, conf.ID, conf.ConnectionTypes, conns.String())) + } + + res := fmt.Sprintf(template, body.String()) + return format.Source([]byte(res)) +} + +const template = `// Copyright (c) Mondoo, Inc. +// SPDX-License-Identifier: BUSL-1.1 +// +// This file is auto-generated by 'make providers/defaults' + +package providers + +import "go.mondoo.com/cnquery/v9/providers-sdk/v1/plugin" + +// DefaultProviders are useful when working in air-gapped environments +// to tell users what providers are used for common connections, when there +// is no other way to find out. +var DefaultProviders Providers = map[string]*Provider{ + %s +} +` + +const providerTemplate = ` + %#v: { + Provider: &plugin.Provider{ + Name: %#v, + ID: %#v, + ConnectionTypes: %#v, + Connectors: []plugin.Connector{ + %s + }, + }, + }, +` + +const connectorTemplate = ` + { + Name: %#v, + Use: %#v, + Short: %#v, + }, +` + +func getProviders() (map[string]string, error) { + dir, err := os.ReadDir("./providers") + if err != nil { + return nil, err + } + + res := map[string]string{} + for i := range dir { + entry := dir[i] + if !entry.IsDir() { + continue + } + name := entry.Name() + res[name] = filepath.Join("./providers", name) + } + return res, nil +} + +func getConfig(providerName string, providerPath string) (*plugin.Provider, error) { + log.Info().Str("provider", providerName).Msg("generate provider config") + cmd := exec.Command("go", "run", "./gen/main.go", ".") + cmd.Dir = providerPath + _, err := cmd.Output() + if err != nil { + return nil, errors.New("failed to run:" + cmd.String() + " in " + providerPath) + } + + raw, err := os.ReadFile(filepath.Join(providerPath, "dist", providerName+".json")) + if err != nil { + return nil, err + } + + var res plugin.Provider + err = json.Unmarshal(raw, &res) + return &res, err +} + +func init() { + rootCmd.Flags().StringP("output", "o", "providers/defaults.go", "output go-file for defaults of providers") +} + +func main() { + logger.CliCompactLogger(logger.LogOutputWriter) + zerolog.SetGlobalLevel(zerolog.DebugLevel) + + if err := rootCmd.Execute(); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} diff --git a/providers/defaults.go b/providers/defaults.go index dde32183cb..38ca867258 100644 --- a/providers/defaults.go +++ b/providers/defaults.go @@ -1,323 +1,392 @@ // Copyright (c) Mondoo, Inc. // SPDX-License-Identifier: BUSL-1.1 +// +// This file is auto-generated by 'make providers/defaults' package providers -import ( - "github.com/cockroachdb/errors" - "go.mondoo.com/cnquery/v9/providers-sdk/v1/plugin" -) - -const ( - DefaultOsID = "go.mondoo.com/cnquery/v9/providers/os" - DeprecatedDefaultOsID = "go.mondoo.com/cnquery/providers/os" // temp to migrate v9 beta users -) - -var defaultRuntime *Runtime - -func DefaultRuntime() *Runtime { - if defaultRuntime == nil { - defaultRuntime = Coordinator.NewRuntime() - } - return defaultRuntime -} - -func SetDefaultRuntime(rt *Runtime) error { - if rt == nil { - return errors.New("attempted to set default runtime to null") - } - defaultRuntime = rt - return nil -} +import "go.mondoo.com/cnquery/v9/providers-sdk/v1/plugin" // DefaultProviders are useful when working in air-gapped environments // to tell users what providers are used for common connections, when there // is no other way to find out. var DefaultProviders Providers = map[string]*Provider{ + "arista": { Provider: &plugin.Provider{ Name: "arista", + ID: "go.mondoo.com/cnquery/v9/providers/arista", ConnectionTypes: []string{"arista"}, Connectors: []plugin.Connector{ + { Name: "arista", + Use: "arista user@host", Short: "an Arista EOS device", }, }, }, }, + "atlassian": { Provider: &plugin.Provider{ - Name: "atlassian", - ConnectionTypes: []string{ - "atlassian", - "jira", - "admin", - "confluence", - "scim", - }, + Name: "atlassian", + ID: "go.mondoo.com/cnquery/providers/atlassian", + ConnectionTypes: []string{"atlassian", "jira", "admin", "confluence", "scim"}, Connectors: []plugin.Connector{ + { Name: "atlassian", + Use: "atlassian", Short: "Atlassian", }, }, }, }, + "aws": { Provider: &plugin.Provider{ Name: "aws", + ID: "go.mondoo.com/cnquery/v9/providers/aws", ConnectionTypes: []string{"aws", "ebs"}, Connectors: []plugin.Connector{ + { Name: "aws", + Use: "aws", Short: "an AWS account", }, }, }, }, + "azure": { Provider: &plugin.Provider{ Name: "azure", + ID: "go.mondoo.com/cnquery/v9/providers/azure", ConnectionTypes: []string{"azure"}, Connectors: []plugin.Connector{ + { Name: "azure", + Use: "azure", Short: "an Azure subscription", }, }, }, }, + "core": { Provider: &plugin.Provider{ Name: "core", + ID: "go.mondoo.com/cnquery/v9/providers/core", ConnectionTypes: []string(nil), Connectors: []plugin.Connector{}, }, }, + "equinix": { Provider: &plugin.Provider{ Name: "equinix", + ID: "go.mondoo.com/cnquery/v9/providers/equinix", ConnectionTypes: []string{"equinix"}, Connectors: []plugin.Connector{ + { Name: "equinix", + Use: "equinix [org ] [project ] [--token ]", Short: "an Equinix Metal organization", }, }, }, }, + "gcp": { Provider: &plugin.Provider{ Name: "gcp", + ID: "go.mondoo.com/cnquery/v9/providers/gcp", ConnectionTypes: []string{"gcp", "gcp-snapshot"}, Connectors: []plugin.Connector{ + { Name: "gcp", - Short: "a GCP project", + Use: "gcp", + Short: "a Google Cloud project", }, }, }, }, + "github": { Provider: &plugin.Provider{ Name: "github", + ID: "go.mondoo.com/cnquery/v9/providers/github", ConnectionTypes: []string{"github"}, Connectors: []plugin.Connector{ + { Name: "github", + Use: "github", Short: "a GitHub organization or repository", }, }, }, }, + "gitlab": { Provider: &plugin.Provider{ Name: "gitlab", + ID: "go.mondoo.com/cnquery/v9/providers/gitlab", ConnectionTypes: []string{"gitlab", "gitlab-group", "gitlab-project"}, Connectors: []plugin.Connector{ + { Name: "gitlab", + Use: "gitlab", Short: "a GitLab group or project", }, }, }, }, + "google-workspace": { Provider: &plugin.Provider{ Name: "google-workspace", + ID: "go.mondoo.com/cnquery/v9/providers/google-workspace", ConnectionTypes: []string{"google-workspace"}, Connectors: []plugin.Connector{ + { Name: "google-workspace", + Use: "google-workspace [--credentials-path ] [--customer-id ] [--impersonated-user-email ]", Short: "a Google Workspace account", }, }, }, }, + "ipmi": { Provider: &plugin.Provider{ Name: "ipmi", + ID: "go.mondoo.com/cnquery/v9/providers/ipmi", ConnectionTypes: []string{"ipmi"}, Connectors: []plugin.Connector{ + { Name: "ipmi", + Use: "ipmi user@host", Short: "an IPMI interface", }, }, }, }, + "k8s": { Provider: &plugin.Provider{ Name: "k8s", + ID: "go.mondoo.com/cnquery/v9/providers/k8s", ConnectionTypes: []string{"k8s"}, Connectors: []plugin.Connector{ + { Name: "k8s", - Short: "a Kubernetes cluster or local manifest file(s).", + Use: "k8s (optional MANIFEST path)", + Short: "a Kubernetes cluster or local manifest file(s)", }, }, }, }, + "ms365": { Provider: &plugin.Provider{ Name: "ms365", + ID: "go.mondoo.com/cnquery/v9/providers/ms365", ConnectionTypes: []string{"ms365"}, Connectors: []plugin.Connector{ + { Name: "ms365", + Use: "ms365", Short: "a Microsoft 365 account", }, }, }, }, + "network": { Provider: &plugin.Provider{ Name: "network", + ID: "go.mondoo.com/cnquery/v9/providers/network", ConnectionTypes: []string{"host"}, Connectors: []plugin.Connector{ + { Name: "host", + Use: "host HOST", Short: "a remote host", }, }, }, }, + "oci": { Provider: &plugin.Provider{ Name: "oci", + ID: "go.mondoo.com/cnquery/v9/providers/oci", ConnectionTypes: []string{"oci"}, Connectors: []plugin.Connector{ + { Name: "oci", + Use: "oci", Short: "an Oracle Cloud Infrastructure tenancy", }, }, }, }, + "okta": { Provider: &plugin.Provider{ Name: "okta", + ID: "go.mondoo.com/cnquery/v9/providers/okta", ConnectionTypes: []string{"okta"}, Connectors: []plugin.Connector{ + { Name: "okta", + Use: "okta", Short: "Okta", }, }, }, }, + "opcua": { Provider: &plugin.Provider{ Name: "opcua", + ID: "go.mondoo.com/cnquery/v9/providers/opcua", ConnectionTypes: []string{"opcua"}, Connectors: []plugin.Connector{ + { Name: "opcua", + Use: "opcua [--endpoint ]", Short: "an OPC UA device", }, }, }, }, + "os": { Provider: &plugin.Provider{ Name: "os", + ID: "go.mondoo.com/cnquery/v9/providers/os", ConnectionTypes: []string{"local", "ssh", "tar", "docker-snapshot", "vagrant", "docker-image", "docker-container", "docker-registry", "container-registry", "registry-image", "filesystem"}, Connectors: []plugin.Connector{ + { Name: "local", + Use: "local", Short: "your local system", }, + { Name: "ssh", + Use: "ssh user@host", Short: "a remote system via SSH", }, + { Name: "winrm", + Use: "winrm user@host", Short: "a remote system via WinRM", }, + { Name: "vagrant", + Use: "vagrant host", Short: "a Vagrant host", }, + { Name: "container", + Use: "container", Short: "a running container or container image", }, + { Name: "docker", - Short: "a running Docker or Docker image", + Use: "docker", + Short: "a running Docker container or Docker image", }, + { Name: "filesystem", - Short: "a mounted file system target.", + Use: "filesystem [flags]", + Short: "a mounted file system target", }, }, }, }, + "slack": { Provider: &plugin.Provider{ Name: "slack", + ID: "go.mondoo.com/cnquery/v9/providers/slack", ConnectionTypes: []string{"slack"}, Connectors: []plugin.Connector{ + { Name: "slack", + Use: "slack", Short: "a Slack team", }, }, }, }, + "terraform": { Provider: &plugin.Provider{ Name: "terraform", + ID: "go.mondoo.com/cnquery/v9/providers/terraform", ConnectionTypes: []string{"terraform-state", "terraform-plan", "terraform-hcl", "terraform-hcl-git"}, Connectors: []plugin.Connector{ + { Name: "terraform", - Short: "a Terraform HCL file or directory.", + Use: "terraform PATH", + Short: "a Terraform HCL file or directory", }, }, }, }, + "vcd": { Provider: &plugin.Provider{ Name: "vcd", + ID: "go.mondoo.com/cnquery/v9/providers/vcd", ConnectionTypes: []string{"vcd"}, Connectors: []plugin.Connector{ + { Name: "vcd", + Use: "vcd [--user ] [--host ] [--organization ] [--ask-pass] [--password ]", Short: "a VMware Cloud Director installation", }, }, }, }, + "vsphere": { Provider: &plugin.Provider{ Name: "vsphere", + ID: "go.mondoo.com/cnquery/v9/providers/vsphere", ConnectionTypes: []string{"vsphere"}, Connectors: []plugin.Connector{ + { Name: "vsphere", + Use: "vsphere user@host", Short: "a VMware vSphere installation", }, }, diff --git a/providers/defaults_shared.go b/providers/defaults_shared.go new file mode 100644 index 0000000000..29ec602568 --- /dev/null +++ b/providers/defaults_shared.go @@ -0,0 +1,30 @@ +// Copyright (c) Mondoo, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package providers + +import ( + "github.com/cockroachdb/errors" +) + +const ( + DefaultOsID = "go.mondoo.com/cnquery/v9/providers/os" + DeprecatedDefaultOsID = "go.mondoo.com/cnquery/providers/os" // temp to migrate v9 beta users +) + +var defaultRuntime *Runtime + +func DefaultRuntime() *Runtime { + if defaultRuntime == nil { + defaultRuntime = Coordinator.NewRuntime() + } + return defaultRuntime +} + +func SetDefaultRuntime(rt *Runtime) error { + if rt == nil { + return errors.New("attempted to set default runtime to null") + } + defaultRuntime = rt + return nil +} From 3646c4b9c343d7b8950ab16aa3fcea507c119776 Mon Sep 17 00:00:00 2001 From: Dominik Richter Date: Tue, 17 Oct 2023 00:20:09 -0700 Subject: [PATCH 2/2] =?UTF-8?q?=E2=9C=A8=20Show=20all=20provider=20subcomm?= =?UTF-8?q?ands?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Usually we only show subcommands for providers that are installed. This offers an alternative where we show all provider subcommands in the help menu. eg: ```bash ~ $> cnquery shell --help x1 Allows the interactive exploration of MQL queries. Usage: cnquery shell [flags] cnquery shell [command] Available Commands: arista Interactive shell with an Arista EOS device atlassian Interactive shell with Atlassian aws Interactive shell with an AWS account azure Interactive shell with an Azure subscription container Interactive shell with a running container or container image docker Interactive shell with a running docker or docker image equinix Interactive shell with an Equinix Metal organization filesystem Interactive shell with a mounted file system target. gcp Interactive shell with a Google Cloud project github Interactive shell with a GitHub organization or repository ... ``` I still wonder if we should create a custom help for these provider plugins that is shorter than the current list of defaults. They are getting a bit out of hand... Signed-off-by: Dominik Richter --- cli/providers/providers.go | 4 ++++ providers/providers.go | 15 ++++++++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/cli/providers/providers.go b/cli/providers/providers.go index 77cdab2c36..fdb5863e4b 100644 --- a/cli/providers/providers.go +++ b/cli/providers/providers.go @@ -41,6 +41,10 @@ func AttachCLIs(rootCmd *cobra.Command, commands ...*Command) error { } } + // We want to offer as many commands as possible, + // useful for e.g. help commands + providers.AddMissingDefaults(existing) + // Now that we know we have all providers, it's time to load them to build // the remaining CLI. Probably an opportunity to optimize in the future, // but fine for now to do it all. diff --git a/providers/providers.go b/providers/providers.go index 7644f25e35..680aaaa7db 100644 --- a/providers/providers.go +++ b/providers/providers.go @@ -6,7 +6,6 @@ package providers import ( "archive/tar" "encoding/json" - "go.mondoo.com/ranger-rpc" "io" "net/http" "os" @@ -23,6 +22,7 @@ import ( "go.mondoo.com/cnquery/v9/cli/config" "go.mondoo.com/cnquery/v9/providers-sdk/v1/plugin" "go.mondoo.com/cnquery/v9/providers-sdk/v1/resources" + "go.mondoo.com/ranger-rpc" "golang.org/x/exp/slices" ) @@ -160,6 +160,19 @@ func ListAll() ([]*Provider, error) { return res, nil } +// AddMissingDefaults to a list of existing providers. A system may have +// a set of providers already installed. This command cycles through +// the list of default providers and adds them to the list of existing +// providers to create a full list of possible providers (existing + potential). +// Useful when generating the CLI without having all providers installed. +func AddMissingDefaults(existing Providers) { + for _, v := range DefaultProviders { + if _, ok := existing[v.ID]; !ok { + existing[v.ID] = v + } + } +} + // EnsureProvider makes sure that a given provider exists and returns it. // You can supply providers either via: // 1. connectorName, which is what you see in the CLI e.g. "local", "ssh", ...