diff --git a/cmd/csi-bizflycloud/main.go b/cmd/csi-bizflycloud/main.go index ef85c65..ef9e138 100644 --- a/cmd/csi-bizflycloud/main.go +++ b/cmd/csi-bizflycloud/main.go @@ -35,15 +35,17 @@ import ( ) var ( - endpoint string - nodeID string - authMethod string - username string - password string - appCredID string - appCredSecret string - cluster string - apiUrl string + endpoint string + nodeID string + authMethod string + username string + password string + tenantID string + appCredID string + appCredSecret string + cluster string + apiUrl string + isControlPlane bool ) func init() { @@ -88,6 +90,8 @@ func main() { cmd.PersistentFlags().StringVar(&endpoint, "endpoint", "", "CSI endpoint") cmd.MarkPersistentFlagRequired("endpoint") + cmd.PersistentFlags().BoolVar(&isControlPlane, "is_control_plane", false, "Is Control Plane node") + cmd.PersistentFlags().StringVar(&authMethod, "auth_method", "password", "Authentication method") cmd.PersistentFlags().StringVar(&username, "username", "", "BizFly Cloud username") @@ -98,6 +102,8 @@ func main() { cmd.PersistentFlags().StringVar(&appCredSecret, "application_credential_secret", "", "BizFly Cloud Application Credential Secret") + cmd.PersistentFlags().StringVar(&tenantID, "tenant_id", "", "BizFly Cloud Tenant ID") + cmd.PersistentFlags().StringVar(&apiUrl, "api_url", "https://manage.bizflycloud.vn", "BizFly Cloud API URL") cmd.PersistentFlags().StringVar(&cluster, "cluster", "", "The identifier of the cluster that the plugin is running in.") @@ -129,29 +135,33 @@ func handle() { klog.V(3).Infof("Failed to GetMetadataProvider: %v", err) } - client, err := gobizfly.NewClient(gobizfly.WithTenantName(username), gobizfly.WithAPIUrl(apiUrl)) - if err != nil { - klog.Errorf("failed to create bizfly client: %v", err) - return + if isControlPlane { + client, err := gobizfly.NewClient(gobizfly.WithTenantName(username), gobizfly.WithAPIUrl(apiUrl), gobizfly.WithTenantID(tenantID)) + if err != nil { + klog.Errorf("failed to create bizfly client: %v", err) + return + } + ctx, cancelFunc := context.WithTimeout(context.Background(), time.Second*10) + defer cancelFunc() + + tok, err := client.Token.Create(ctx, &gobizfly.TokenCreateRequest{ + AuthMethod: authMethod, + Username: username, + Password: password, + AppCredID: appCredID, + AppCredSecret: appCredSecret}) + + client.SetKeystoneToken(tok.KeystoneToken) + + if err != nil { + klog.Errorf("Failed to get bizfly client token: %v", err) + return + } + client.SetKeystoneToken(tok.KeystoneToken) + d.SetupControlDriver(client, iMount, metadatda) + d.Run() + } else { + d.SetupNodeDriver(iMount, metadatda) + d.Run() } - ctx, cancelFunc := context.WithTimeout(context.Background(), time.Second*10) - defer cancelFunc() - - tok, err := client.Token.Create(ctx, &gobizfly.TokenCreateRequest{ - AuthMethod: authMethod, - Username: username, - Password: password, - AppCredID: appCredID, - AppCredSecret: appCredSecret}) - - client.SetKeystoneToken(tok.KeystoneToken) - - if err != nil { - klog.Errorf("Failed to get bizfly client token: %v", err) - return - } - client.SetKeystoneToken(tok.KeystoneToken) - - d.SetupDriver(client, iMount, metadatda) - d.Run() } diff --git a/driver/driver.go b/driver/driver.go index e7892d7..a544431 100644 --- a/driver/driver.go +++ b/driver/driver.go @@ -115,11 +115,16 @@ func (d *VolumeDriver) AddNodeServiceCapabilities(nl []csi.NodeServiceCapability return nil } -// SetupDriver setups driver for volume driver -func (d *VolumeDriver) SetupDriver(client *gobizfly.Client, mount mount.IMount, metadata openstack.IMetadata) { +// SetupControlDriver setups driver for control plane +func (d *VolumeDriver) SetupControlDriver(client *gobizfly.Client, mount mount.IMount, metadata openstack.IMetadata) { d.ids = NewIdentityServer(d) d.cs = NewControllerServer(d, client) - d.ns = NewNodeServer(d, mount, metadata, client) +} + +// SetupControlDriver setups driver for control plane +func (d *VolumeDriver) SetupNodeDriver(mount mount.IMount, metadata openstack.IMetadata) { + d.ids = NewIdentityServer(d) + d.ns = NewNodeServer(d, mount, metadata) } // Run run driver diff --git a/driver/nodeserver.go b/driver/nodeserver.go index d828d63..3c770b1 100644 --- a/driver/nodeserver.go +++ b/driver/nodeserver.go @@ -23,14 +23,13 @@ import ( "path/filepath" "strings" - "github.com/bizflycloud/gobizfly" + //"github.com/bizflycloud/gobizfly" "github.com/container-storage-interface/spec/lib/go/csi" "golang.org/x/net/context" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "k8s.io/cloud-provider-openstack/pkg/csi/cinder/openstack" "k8s.io/cloud-provider-openstack/pkg/util/blockdevice" - cpoerrors "k8s.io/cloud-provider-openstack/pkg/util/errors" "k8s.io/cloud-provider-openstack/pkg/util/metadata" "k8s.io/cloud-provider-openstack/pkg/util/mount" "k8s.io/klog" @@ -42,7 +41,7 @@ type nodeServer struct { Driver *VolumeDriver Mount mount.IMount Metadata openstack.IMetadata - Client *gobizfly.Client + //Client *gobizfly.Client } func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) { @@ -68,13 +67,13 @@ func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis if len(source) == 0 { return nil, status.Error(codes.InvalidArgument, "NodePublishVolume Staging Target Path must be provided") } - _, err := ns.Client.Volume.Get(ctx, volumeID) - if err != nil { - if cpoerrors.IsNotFound(err) { - return nil, status.Error(codes.NotFound, "Volume not found") - } - return nil, status.Error(codes.Internal, fmt.Sprintf("GetVolume failed with error %v", err)) - } + //_, err := ns.Client.Volume.Get(ctx, volumeID) + //if err != nil { + // if cpoerrors.IsNotFound(err) { + // return nil, status.Error(codes.NotFound, "Volume not found") + // } + // return nil, status.Error(codes.Internal, fmt.Sprintf("GetVolume failed with error %v", err)) + //} mountOptions := []string{"bind"} if req.GetReadonly() { @@ -163,24 +162,24 @@ func (ns *nodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpu return nil, status.Error(codes.InvalidArgument, "NodeUnpublishVolume volumeID must be provided") } - if _, err := ns.Client.Volume.Get(ctx, volumeID); err != nil { - if !cpoerrors.IsNotFound(err) { - return nil, status.Error(codes.Internal, fmt.Sprintf("GetVolume failed with error %v", err)) - } - - // if not found by id, try to search by name - volName := fmt.Sprintf("ephemeral-%s", volumeID) - - vol, err := GetVolumesByName(ctx, ns.Client, volName) - - //if volume not found then GetVolumesByName returns empty list - if err != nil { - return nil, status.Error(codes.Internal, fmt.Sprintf("GetVolume failed with error %v", err)) - } - if vol == nil { - return nil, status.Error(codes.NotFound, fmt.Sprintf("Volume not found %s", volName)) - } - } + //if _, err := ns.Client.Volume.Get(ctx, volumeID); err != nil { + // if !cpoerrors.IsNotFound(err) { + // return nil, status.Error(codes.Internal, fmt.Sprintf("GetVolume failed with error %v", err)) + // } + // + // // if not found by id, try to search by name + // volName := fmt.Sprintf("ephemeral-%s", volumeID) + // + // vol, err := GetVolumesByName(ctx, ns.Client, volName) + // + // //if volume not found then GetVolumesByName returns empty list + // if err != nil { + // return nil, status.Error(codes.Internal, fmt.Sprintf("GetVolume failed with error %v", err)) + // } + // if vol == nil { + // return nil, status.Error(codes.NotFound, fmt.Sprintf("Volume not found %s", volName)) + // } + //} m := ns.Mount notMnt, err := m.IsLikelyNotMountPointDetach(targetPath) @@ -220,13 +219,13 @@ func (ns *nodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVol return nil, status.Error(codes.InvalidArgument, "NodeStageVolume Volume Capability must be provided") } - _, err := ns.Client.Volume.Get(ctx, volumeID) - if err != nil { - if cpoerrors.IsNotFound(err) { - return nil, status.Error(codes.NotFound, "Volume not found") - } - return nil, status.Error(codes.Internal, fmt.Sprintf("GetVolume failed with error %v", err)) - } + //_, err := ns.Client.Volume.Get(ctx, volumeID) + //if err != nil { + // if cpoerrors.IsNotFound(err) { + // return nil, status.Error(codes.NotFound, "Volume not found") + // } + // return nil, status.Error(codes.Internal, fmt.Sprintf("GetVolume failed with error %v", err)) + //} m := ns.Mount // Do not trust the path provided by cinder, get the real path on node @@ -281,14 +280,14 @@ func (ns *nodeServer) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstag return nil, status.Error(codes.InvalidArgument, "NodeUnstageVolume Staging Target Path must be provided") } - _, err := ns.Client.Volume.Get(ctx, volumeID) - if err != nil { - if cpoerrors.IsNotFound(err) { - klog.V(4).Infof("NodeUnstageVolume: Unable to find volume: %v", err) - return nil, status.Error(codes.NotFound, "Volume not found") - } - return nil, status.Error(codes.Internal, fmt.Sprintf("GetVolume failed with error %v", err)) - } + //_, err := ns.Client.Volume.Get(ctx, volumeID) + //if err != nil { + // if cpoerrors.IsNotFound(err) { + // klog.V(4).Infof("NodeUnstageVolume: Unable to find volume: %v", err) + // return nil, status.Error(codes.NotFound, "Volume not found") + // } + // return nil, status.Error(codes.Internal, fmt.Sprintf("GetVolume failed with error %v", err)) + //} m := ns.Mount diff --git a/driver/util.go b/driver/util.go index 7fa6dd9..1e1c8ba 100644 --- a/driver/util.go +++ b/driver/util.go @@ -67,12 +67,11 @@ func NewIdentityServer(d *VolumeDriver) *identityServer { } } -func NewNodeServer(d *VolumeDriver, mount mount.IMount, metadata openstack.IMetadata, client *gobizfly.Client) *nodeServer { +func NewNodeServer(d *VolumeDriver, mount mount.IMount, metadata openstack.IMetadata) *nodeServer { return &nodeServer{ Driver: d, Mount: mount, Metadata: metadata, - Client: client, } } diff --git a/go.mod b/go.mod index e0ad59e..b57096b 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/bizflycloud/csi-bizflycloud go 1.14 require ( - github.com/bizflycloud/gobizfly v0.0.0-20200514081735-03c430a0585b + github.com/bizflycloud/gobizfly v0.0.0-20200610135315-1b3e23893213 github.com/container-storage-interface/spec v1.2.0 github.com/golang/protobuf v1.4.1 github.com/spf13/cobra v1.0.0 diff --git a/go.sum b/go.sum index cdccf6d..bd82d43 100644 --- a/go.sum +++ b/go.sum @@ -61,6 +61,8 @@ github.com/bizflycloud/gobizfly v0.0.0-20200509022858-1dd705ff35b3 h1:3195YbHKzP github.com/bizflycloud/gobizfly v0.0.0-20200509022858-1dd705ff35b3/go.mod h1:2otUgZd6PrKezjMrdAw9gy7byEXMrigX9Sm4ujCaFuk= github.com/bizflycloud/gobizfly v0.0.0-20200514081735-03c430a0585b h1:9DaY0hAkK/PGs12481LXTyFmuGKfkW6u7VPsyRXuPHk= github.com/bizflycloud/gobizfly v0.0.0-20200514081735-03c430a0585b/go.mod h1:2otUgZd6PrKezjMrdAw9gy7byEXMrigX9Sm4ujCaFuk= +github.com/bizflycloud/gobizfly v0.0.0-20200610135315-1b3e23893213 h1:Ihr811tiEnUKB0QaDJglGI+fCwFrcXy8Q17vpFTEvEc= +github.com/bizflycloud/gobizfly v0.0.0-20200610135315-1b3e23893213/go.mod h1:2otUgZd6PrKezjMrdAw9gy7byEXMrigX9Sm4ujCaFuk= github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g= diff --git a/releases/v0.1/bizfly-csi-controllerplugin-rbac.yaml b/releases/v0.1.0/bizfly-csi-controllerplugin-rbac.yaml similarity index 100% rename from releases/v0.1/bizfly-csi-controllerplugin-rbac.yaml rename to releases/v0.1.0/bizfly-csi-controllerplugin-rbac.yaml diff --git a/releases/v0.1/bizfly-csi-controllerplugin.yaml b/releases/v0.1.0/bizfly-csi-controllerplugin.yaml similarity index 92% rename from releases/v0.1/bizfly-csi-controllerplugin.yaml rename to releases/v0.1.0/bizfly-csi-controllerplugin.yaml index 7a647e5..e0faae9 100644 --- a/releases/v0.1/bizfly-csi-controllerplugin.yaml +++ b/releases/v0.1.0/bizfly-csi-controllerplugin.yaml @@ -82,7 +82,7 @@ spec: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ - name: bizflycloud-csi-plugin - image: cr-hn-1.vccloud.vn/31ff9581861a4d0ea4df5e7dda0f665d/csi-bizflycloud:v0.1 + image: cr-hn-1.vccloud.vn/31ff9581861a4d0ea4df5e7dda0f665d/csi-bizflycloud:v0.1.0 args : - /bin/csi-bizflycloud - "--nodeid=$(NODE_ID)" @@ -91,6 +91,8 @@ spec: - "--auth_method=application_credential" - "--application_credential_id=$(BIZFLYCLOUD_APPLICATION_CREDENTIAL_ID)" - "--application_credential_secret=$(BIZFLYCLOUD_APPLICATION_CREDENTIAL_SECRET)" + - "--tenant_id=$(TENANT_ID)" + - "--is_control_plane=true" env: - name: NODE_ID valueFrom: @@ -110,7 +112,12 @@ spec: secretKeyRef: name: bizflycloud key: application_credential_secret - imagePullPolicy: "IfNotPresent" + - name: TENANT_ID + valueFrom: + secretKeyRef: + name: bizflycloud + key: tenant_id + imagePullPolicy: "Always" volumeMounts: - name: socket-dir mountPath: /csi diff --git a/releases/v0.1/bizfly-csi-nodeplugin-rbac.yaml b/releases/v0.1.0/bizfly-csi-nodeplugin-rbac.yaml similarity index 100% rename from releases/v0.1/bizfly-csi-nodeplugin-rbac.yaml rename to releases/v0.1.0/bizfly-csi-nodeplugin-rbac.yaml diff --git a/releases/v0.1/bizfly-csi-nodeplugin.yaml b/releases/v0.1.0/bizfly-csi-nodeplugin.yaml similarity index 82% rename from releases/v0.1/bizfly-csi-nodeplugin.yaml rename to releases/v0.1.0/bizfly-csi-nodeplugin.yaml index 52ffcef..d821bef 100644 --- a/releases/v0.1/bizfly-csi-nodeplugin.yaml +++ b/releases/v0.1.0/bizfly-csi-nodeplugin.yaml @@ -48,14 +48,11 @@ spec: capabilities: add: ["SYS_ADMIN"] allowPrivilegeEscalation: true - image: cr-hn-1.vccloud.vn/31ff9581861a4d0ea4df5e7dda0f665d/csi-bizflycloud:v0.1 + image: cr-hn-1.vccloud.vn/31ff9581861a4d0ea4df5e7dda0f665d/csi-bizflycloud:v0.1.0 args : - /bin/csi-bizflycloud - "--nodeid=$(NODE_ID)" - "--endpoint=$(CSI_ENDPOINT)" - - "--auth_method=application_credential" - - "--application_credential_id=$(BIZFLYCLOUD_APPLICATION_CREDENTIAL_ID)" - - "--application_credential_secret=$(BIZFLYCLOUD_APPLICATION_CREDENTIAL_SECRET)" env: - name: NODE_ID valueFrom: @@ -63,16 +60,6 @@ spec: fieldPath: spec.nodeName - name: CSI_ENDPOINT value: unix://csi/csi.sock - - name: BIZFLYCLOUD_APPLICATION_CREDENTIAL_ID - valueFrom: - secretKeyRef: - name: bizflycloud - key: application_credential_id - - name: BIZFLYCLOUD_APPLICATION_CREDENTIAL_SECRET - valueFrom: - secretKeyRef: - name: bizflycloud - key: application_credential_secret imagePullPolicy: "IfNotPresent" volumeMounts: - name: socket-dir diff --git a/releases/v0.1/csi-bizfly-driver.yaml b/releases/v0.1.0/csi-bizfly-driver.yaml similarity index 76% rename from releases/v0.1/csi-bizfly-driver.yaml rename to releases/v0.1.0/csi-bizfly-driver.yaml index 5a2a9c8..11f5c92 100644 --- a/releases/v0.1/csi-bizfly-driver.yaml +++ b/releases/v0.1.0/csi-bizfly-driver.yaml @@ -5,5 +5,5 @@ metadata: spec: attachRequired: true podInfoOnMount: true - volumeLifecycleModes: - - Persistent + # volumeLifecycleModes: + # - Persistent diff --git a/releases/v0.1.0/secret.yaml b/releases/v0.1.0/secret.yaml new file mode 100644 index 0000000..aecc36c --- /dev/null +++ b/releases/v0.1.0/secret.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Secret +metadata: + name: bizflycloud + namespace: kube-system +stringData: + application_credential_id: "your_application_credential_id" + application_credential_secret: "your_application_credential_secret" + tenant_id: "your_tenant_id" diff --git a/releases/v0.1.yml b/releases/v0.1.yml deleted file mode 100644 index 30a6c36..0000000 --- a/releases/v0.1.yml +++ /dev/null @@ -1,109 +0,0 @@ -# This YAML file contains driver-registrar & csi driver nodeplugin API objects, -# which are necessary to run csi nodeplugin for bizflycloud. - -kind: DaemonSet -apiVersion: apps/v1 -metadata: - name: csi-bizflycloud-nodeplugin - namespace: kube-system -spec: - selector: - matchLabels: - app: csi-bizflycloud-nodeplugin - template: - metadata: - labels: - app: csi-bizflycloud-nodeplugin - spec: - serviceAccount: csi-bizflycloud-node-sa - hostNetwork: true - containers: - - name: node-driver-registrar - image: quay.io/k8scsi/csi-node-driver-registrar:v1.2.0 - args: - - "--csi-address=$(ADDRESS)" - - "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)" - lifecycle: - preStop: - exec: - command: ["/bin/sh", "-c", "rm -rf /registration/volume.csi.bizflycloud.vn /registration/volume.csi.bizflycloud.vn-reg.sock"] - env: - - name: ADDRESS - value: /csi/csi.sock - - name: DRIVER_REG_SOCK_PATH - value: /var/lib/kubelet/plugins/volume.csi.bizflycloud.vn/csi.sock - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - imagePullPolicy: "IfNotPresent" - volumeMounts: - - name: socket-dir - mountPath: /csi - - name: registration-dir - mountPath: /registration - - name: bizflycloud-csi-plugin - securityContext: - privileged: true - capabilities: - add: ["SYS_ADMIN"] - allowPrivilegeEscalation: true - image: cr-hn-1.vccloud.vn/31ff9581861a4d0ea4df5e7dda0f665d/csi-bizflycloud:latest - args : - - /bin/csi-bizflycloud - - "--nodeid=$(NODE_ID)" - - "--endpoint=$(CSI_ENDPOINT)" - - "--auth_method=application_credential" - - "--application_credential_id=$(BIZFLYCLOUD_APPLICATION_CREDENTIAL_ID)" - - "--application_credential_secret=$(BIZFLYCLOUD_APPLICATION_CREDENTIAL_SECRET)" - env: - - name: NODE_ID - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: CSI_ENDPOINT - value: unix://csi/csi.sock - - name: BIZFLYCLOUD_APPLICATION_CREDENTIAL_ID - valueFrom: - secretKeyRef: - name: bizflycloud - key: application_credential_id - - name: BIZFLYCLOUD_APPLICATION_CREDENTIAL_SECRET - valueFrom: - secretKeyRef: - name: bizflycloud - key: application_credential_secret - imagePullPolicy: "IfNotPresent" - volumeMounts: - - name: socket-dir - mountPath: /csi - - name: kubelet-dir - mountPath: /var/lib/kubelet - mountPropagation: "Bidirectional" - - name: pods-cloud-data - mountPath: /var/lib/cloud/data - readOnly: true - - name: pods-probe-dir - mountPath: /dev - mountPropagation: "HostToContainer" - volumes: - - name: socket-dir - hostPath: - path: /var/lib/kubelet/plugins/volume.csi.bizflycloud.vn - type: DirectoryOrCreate - - name: registration-dir - hostPath: - path: /var/lib/kubelet/plugins_registry/ - type: Directory - - name: kubelet-dir - hostPath: - path: /var/lib/kubelet - type: Directory - - name: pods-cloud-data - hostPath: - path: /var/lib/cloud/data - type: Directory - - name: pods-probe-dir - hostPath: - path: /dev - type: Directory diff --git a/releases/v0.1/secret.yaml b/releases/v0.1/secret.yaml deleted file mode 100644 index 2434f7d..0000000 --- a/releases/v0.1/secret.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: bizflycloud - namespace: kube-system -stringData: - application_credential_id: "youremail@example.com" - application_credential_secret: "yourPassWORD" \ No newline at end of file