Skip to content

Commit

Permalink
Merge pull request #150 from jtaleric/virt
Browse files Browse the repository at this point in the history 
Virt
  • Loading branch information
@vishnuchalla
vishnuchalla committed Sep 11, 2024
2 parents 6065e03 + a4a6ae1 commit 22f69b9
Show file tree
Hide file tree
Showing 36 changed files with 3,784 additions and 234 deletions.
13 changes: 12 additions & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ $ cd k8s-netperf
$ make docker-build
```

## Running
## Running with Pods
Ensure your `kubeconfig` is properly set to the cluster you would like to run `k8s-netperf` against.

also be sure to create a `netperf` namespace. (Not over-writable yet)
Expand Down Expand Up @@ -95,6 +95,17 @@ Flags:
> *Note: With OpenShift, we attempt to discover the OpenShift route. If that route is not reachable, it might be required to `port-forward` the service and pass that via the `--prom` option.*
## Running with VMs
Running k8s-netperf against Virtual Machines (OpenShift CNV) requires
- OpenShift CNV must be deployed and users should be able to define VMIs
- SSH keys to be present in the home directory `(~/.ssh/id_rsa.pub)`
- OpenShift Routes - k8s-netperf uses this to reach the VMs (k8s-netperf will create the route for the user, but we need Routes)
If the two above are in place, users can orhestrate k8s-netperf to launch VMs by running
`k8s-netperf --vm`
### Config file
#### Config File v2
The v2 config file will be executed in the order the tests are presented in the config file.
Expand Down
125 changes: 97 additions & 28 deletions cmd/k8s-netperf/k8s-netperf.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,13 +16,15 @@ import (
"github.com/cloud-bulldozer/k8s-netperf/pkg/config"
"github.com/cloud-bulldozer/k8s-netperf/pkg/drivers"
"github.com/cloud-bulldozer/k8s-netperf/pkg/k8s"
kubevirtv1 "github.com/cloud-bulldozer/k8s-netperf/pkg/kubevirt/client-go/clientset/versioned/typed/core/v1"
log "github.com/cloud-bulldozer/k8s-netperf/pkg/logging"
"github.com/cloud-bulldozer/k8s-netperf/pkg/metrics"
result "github.com/cloud-bulldozer/k8s-netperf/pkg/results"
"github.com/cloud-bulldozer/k8s-netperf/pkg/sample"
"github.com/google/uuid"
"github.com/spf13/cobra"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/dynamic"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/clientcmd"
)
Expand All @@ -39,6 +41,7 @@ var (
uperf bool
acrossAZ bool
full bool
vm bool
debug bool
promURL string
id string
Expand All @@ -48,6 +51,7 @@ var (
json bool
version bool
csvArchive bool
searchIndex string
)

var rootCmd = &cobra.Command{
Expand Down Expand Up @@ -147,6 +151,21 @@ var rootCmd = &cobra.Command{
os.Exit(1)
}

if vm {
s.VM = true
// Create a dynamic client
dynClient, err := dynamic.NewForConfig(rconfig)
if err != nil {
log.Error(err)
}
kclient, err := kubevirtv1.NewForConfig(rconfig)
if err != nil {
log.Error(err)
}
s.KClient = kclient
s.DClient = dynClient
}

// Build the SUT (Deployments)
err = k8s.BuildSUT(client, &s)
if err != nil {
Expand Down Expand Up @@ -174,37 +193,73 @@ var rootCmd = &cobra.Command{
if iperf3 {
requestedDrivers = append(requestedDrivers, "iperf3")
}

// Run through each test
for _, nc := range s.Configs {
// Determine the metric for the test
metric := string("OP/s")
if strings.Contains(nc.Profile, "STREAM") {
metric = "Mb/s"
}
nc.Metric = metric
nc.AcrossAZ = acrossAZ
// No need to run hostNetwork through Service.
var pr result.Data
for _, driver := range requestedDrivers {
if s.HostNetwork && !nc.Service {
pr = executeWorkload(nc, s, true, driver)
if !s.VM {
for _, nc := range s.Configs {
// Determine the metric for the test
metric := string("OP/s")
if strings.Contains(nc.Profile, "STREAM") {
metric = "Mb/s"
}
nc.Metric = metric
nc.AcrossAZ = acrossAZ
// No need to run hostNetwork through Service.
var pr result.Data
for _, driver := range requestedDrivers {
if s.HostNetwork && !nc.Service {
pr = executeWorkload(nc, s, true, driver, false)
if len(pr.Profile) > 1 {
sr.Results = append(sr.Results, pr)
}
}
pr = executeWorkload(nc, s, false, driver, false)
if len(pr.Profile) > 1 {
sr.Results = append(sr.Results, pr)
}
}
pr = executeWorkload(nc, s, false, driver)
if len(pr.Profile) > 1 {
sr.Results = append(sr.Results, pr)
}
} else {
sr.Virt = true
log.Info("Connecting via ssh to the VMI")
client, err := k8s.SSHConnect(&s)
if err != nil {
log.Fatal(err)
}
s.SSHClient = client
for _, nc := range s.Configs {
// Determine the metric for the test
metric := string("OP/s")
if strings.Contains(nc.Profile, "STREAM") {
metric = "Mb/s"
}
nc.Metric = metric
nc.AcrossAZ = acrossAZ
// No need to run hostNetwork through Service.
var pr result.Data
for _, driver := range requestedDrivers {
if s.HostNetwork && !nc.Service {
pr = executeWorkload(nc, s, true, driver, true)
if len(pr.Profile) > 1 {
sr.Results = append(sr.Results, pr)
}
}
pr = executeWorkload(nc, s, false, driver, true)
if len(pr.Profile) > 1 {
sr.Results = append(sr.Results, pr)
}
}
}
}

if pavail {
for i, npr := range sr.Results {
sr.Results[i].ClientMetrics, _ = metrics.QueryNodeCPU(npr.ClientNodeInfo, pcon, npr.StartTime, npr.EndTime)
sr.Results[i].ServerMetrics, _ = metrics.QueryNodeCPU(npr.ServerNodeInfo, pcon, npr.StartTime, npr.EndTime)
sr.Results[i].ClientPodCPU, _ = metrics.TopPodCPU(npr.ClientNodeInfo, pcon, npr.StartTime, npr.EndTime)
sr.Results[i].ServerPodCPU, _ = metrics.TopPodCPU(npr.ServerNodeInfo, pcon, npr.StartTime, npr.EndTime)
if len(npr.ClientNodeInfo.Hostname) > 0 && len(npr.ServerNodeInfo.Hostname) > 0 {
sr.Results[i].ClientMetrics, _ = metrics.QueryNodeCPU(npr.ClientNodeInfo, pcon, npr.StartTime, npr.EndTime)
sr.Results[i].ServerMetrics, _ = metrics.QueryNodeCPU(npr.ServerNodeInfo, pcon, npr.StartTime, npr.EndTime)
sr.Results[i].ClientPodCPU, _ = metrics.TopPodCPU(npr.ClientNodeInfo, pcon, npr.StartTime, npr.EndTime)
sr.Results[i].ServerPodCPU, _ = metrics.TopPodCPU(npr.ServerNodeInfo, pcon, npr.StartTime, npr.EndTime)
}
}
}

Expand All @@ -230,13 +285,22 @@ var rootCmd = &cobra.Command{
}

if len(searchURL) > 1 {
var esClient *indexers.Indexer
jdocs, err := archive.BuildDocs(sr, uid)
if err != nil {
log.Fatal(err)
}
esClient, err := archive.Connect(searchURL, index, true)
if err != nil {
log.Fatal(err)
if len(searchIndex) > 1 {
esClient, err = archive.Connect(searchURL, searchIndex, true)
if err != nil {
log.Fatal(err)
}
} else {
esClient, err = archive.Connect(searchURL, index, true)
if err != nil {
log.Fatal(err)
}

}
log.Infof("Indexing [%d] documents in %s with UUID %s", len(jdocs), index, uid)
resp, err := (*esClient).Index(jdocs, indexers.IndexingOpts{})
Expand Down Expand Up @@ -305,14 +369,17 @@ func cleanup(client *kubernetes.Clientset) {
}

// executeWorkload executes the workload and returns the result data.
func executeWorkload(nc config.Config, s config.PerfScenarios, hostNet bool, driverName string) result.Data {
func executeWorkload(nc config.Config,
s config.PerfScenarios,
hostNet bool,
driverName string, virt bool) result.Data {
serverIP := ""
Client := s.Client
var driver drivers.Driver
if nc.Service {
if iperf3 {
if driverName == "iperf3" {
serverIP = s.IperfService.Spec.ClusterIP
} else if uperf {
} else if driverName == "uperf" {
serverIP = s.UperfService.Spec.ClusterIP
} else {
serverIP = s.NetperfService.Spec.ClusterIP
Expand Down Expand Up @@ -360,7 +427,7 @@ func executeWorkload(nc config.Config, s config.PerfScenarios, hostNet bool, dri
log.Warnf("Test %s is not supported with driver %s. Skipping.", nc.Profile, npr.Driver)
return npr
}
r, err := driver.Run(s.ClientSet, s.RestConfig, nc, Client, serverIP)
r, err := driver.Run(s.ClientSet, s.RestConfig, nc, Client, serverIP, &s)
if err != nil {
log.Fatal(err)
}
Expand All @@ -372,7 +439,7 @@ func executeWorkload(nc config.Config, s config.PerfScenarios, hostNet bool, dri
// Retry the current test.
for try < retry {
log.Warn("Rerunning test.")
r, err := driver.Run(s.ClientSet, s.RestConfig, nc, Client, serverIP)
r, err := driver.Run(s.ClientSet, s.RestConfig, nc, Client, serverIP, &s)
if err != nil {
log.Error(err)
continue
Expand Down Expand Up @@ -412,12 +479,14 @@ func main() {
rootCmd.Flags().BoolVar(&clean, "clean", true, "Clean-up resources created by k8s-netperf")
rootCmd.Flags().BoolVar(&json, "json", false, "Instead of human-readable output, return JSON to stdout")
rootCmd.Flags().BoolVar(&nl, "local", false, "Run network performance tests with Server-Pods/Client-Pods on the same Node")
rootCmd.Flags().BoolVar(&vm, "vm", false, "Launch Virtual Machines instead of pods for client/servers")
rootCmd.Flags().BoolVar(&acrossAZ, "across", false, "Place the client and server across availability zones")
rootCmd.Flags().BoolVar(&full, "all", false, "Run all tests scenarios - hostNet and podNetwork (if possible)")
rootCmd.Flags().BoolVar(&debug, "debug", false, "Enable debug log")
rootCmd.Flags().StringVar(&promURL, "prom", "", "Prometheus URL")
rootCmd.Flags().StringVar(&id, "uuid", "", "User provided UUID")
rootCmd.Flags().StringVar(&searchURL, "search", "", "OpenSearch URL, if you have auth, pass in the format of https://user:pass@url:port")
rootCmd.Flags().StringVar(&searchIndex, "index", "", "OpenSearch Index to save the results to, defaults to k8s-netperf")
rootCmd.Flags().BoolVar(&showMetrics, "metrics", false, "Show all system metrics retrieved from prom")
rootCmd.Flags().Float64Var(&tcpt, "tcp-tolerance", 10, "Allowed %diff from hostNetwork to podNetwork, anything above tolerance will result in k8s-netperf exiting 1.")
rootCmd.Flags().BoolVar(&version, "version", false, "k8s-netperf version")
Expand Down
16 changes: 14 additions & 2 deletions go.mod
View file
Original file line number Diff line number Diff line change
@@ -1,28 +1,32 @@
module github.com/cloud-bulldozer/k8s-netperf

go 1.18
go 1.19

require (
github.com/aclements/go-moremath v0.0.0-20210112150236-f10218a38794
github.com/cloud-bulldozer/go-commons v1.0.16
github.com/google/uuid v1.3.0
github.com/melbahja/goph v1.4.0
github.com/montanaflynn/stats v0.6.6
github.com/olekukonko/tablewriter v0.0.5
github.com/prometheus/common v0.44.0
github.com/sirupsen/logrus v1.9.0
github.com/spf13/cobra v1.6.1
golang.org/x/crypto v0.14.0
golang.org/x/text v0.13.0
gopkg.in/yaml.v3 v3.0.1
k8s.io/api v0.28.4
k8s.io/apimachinery v0.28.4
k8s.io/client-go v0.28.4
k8s.io/utils v0.0.0-20230406110748-d93618cff8a2
k8s.io/utils v0.0.0-20230505201702-9f6742963106
kubevirt.io/api v1.2.2
)

require (
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/elastic/go-elasticsearch/v7 v7.13.1 // indirect
github.com/emicklei/go-restful/v3 v3.9.0 // indirect
github.com/evanphx/json-patch v4.12.0+incompatible // indirect
github.com/go-logr/logr v1.2.4 // indirect
github.com/go-openapi/jsonpointer v0.19.6 // indirect
github.com/go-openapi/jsonreference v0.20.2 // indirect
Expand All @@ -36,13 +40,18 @@ require (
github.com/inconshreveable/mousetrap v1.0.1 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/kr/fs v0.1.0 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mattn/go-runewidth v0.0.9 // indirect
github.com/moby/spdystream v0.2.0 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/opensearch-project/opensearch-go v1.1.0 // indirect
github.com/openshift/api v0.0.0-20230503133300-8bbcb7ca7183 // indirect
github.com/openshift/custom-resource-status v1.1.2 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pkg/sftp v1.13.5 // indirect
github.com/prometheus/client_golang v1.15.1 // indirect
github.com/spf13/pflag v1.0.5 // indirect
golang.org/x/net v0.17.0 // indirect
Expand All @@ -54,8 +63,11 @@ require (
google.golang.org/protobuf v1.31.0 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
k8s.io/apiextensions-apiserver v0.26.3 // indirect
k8s.io/klog/v2 v2.100.1 // indirect
k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect
kubevirt.io/containerized-data-importer-api v1.57.0-alpha1 // indirect
kubevirt.io/controller-lifecycle-operator-sdk/api v0.0.0-20220329064328-f3cc58c6ed90 // indirect
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
sigs.k8s.io/yaml v1.3.0 // indirect
Expand Down
Loading

0 comments on commit 22f69b9

Please sign in to comment.