diff --git a/cmd/cli/file/file.go b/cmd/cli/file/file.go index 52b2ef90..61c6817b 100644 --- a/cmd/cli/file/file.go +++ b/cmd/cli/file/file.go @@ -14,7 +14,6 @@ import ( var hostOpt option.HostOption var fileOpt option.FileOption -var kubeOpt option.KubeOption var inventory string var verbose string @@ -32,7 +31,7 @@ var FileCmd = &cobra.Command{ if inventoryType == constants.InventoryTypeHosts { HostFile(ctx, logger, fileOpt, hostOpt, inventory) } else if inventoryType == constants.InventoryTypeKubernetes { - KubeFile(ctx, logger, fileOpt, kubeOpt, inventory) + KubeFile(ctx, logger, fileOpt, inventory) } }, } @@ -51,13 +50,13 @@ func HostFile(ctx context.Context, logger *log.Logger, fileOpt option.FileOption return } -func KubeFile(ctx context.Context, logger *log.Logger, fileOpt option.FileOption, kubeOpt option.KubeOption, inventory string) (err error) { +func KubeFile(ctx context.Context, logger *log.Logger, fileOpt option.FileOption, inventory string) (err error) { client, err := utils.NewKubernetesClient(inventory) if err != nil { logger.Error.Println(err) return } - nodeList, err := kube.GetNodes(ctx, logger, client, kubeOpt) + nodeList, err := kube.GetNodes(ctx, logger, client, fileOpt.KubeOption) if err != nil { logger.Error.Println(err) } @@ -93,6 +92,7 @@ func init() { FileCmd.Flags().StringVarP(&hostOpt.PrivateKeyPath, "privatekeypath", "", constants.GetCurrentUserPrivateKeyPath(), "") FileCmd.Flags().IntVar(&hostOpt.Port, "port", 22, "") - FileCmd.Flags().StringVarP(&kubeOpt.NodeName, "nodename", "", "", "") - FileCmd.Flags().StringVarP(&kubeOpt.OpsNamespace, "opsnamespace", "", constants.DefaultOpsNamespace, "ops work namespace") + FileCmd.Flags().StringVarP(&fileOpt.NodeName, "nodename", "", "", "") + FileCmd.Flags().StringVarP(&fileOpt.RuntimeImage, "runtimeimage", "", constants.OpsCliRuntimeImage, "") + FileCmd.Flags().StringVarP(&fileOpt.OpsNamespace, "opsnamespace", "", constants.DefaultOpsNamespace, "ops work namespace") } diff --git a/docs/content/opscli-file.md b/docs/content/opscli-file.md index 5278f4d8..6f979d20 100644 --- a/docs/content/opscli-file.md +++ b/docs/content/opscli-file.md @@ -1,99 +1,86 @@ ## opscli file command -### `-i` 指定操作目标清单 +### 主机 - 本地与对象存储互传文件 -- 指定主机 - -`-i 1.1.1.1` - -通过 `--username` 指定用户名，`--password` 指定密码。 +- 设置 AK\SK -- 批量主机 +```bash +export ak= +export sk= +``` -`-i hosts.txt` +- 上传本地文件 `./tmp.log` 到对象存储 `s3://logs/tmp.log` ```bash -cat hosts.txt - -1.1.1.1 -2.2.2.2 +/usr/local/bin/opscli file --direction upload --localfile ./tmp.log --remotefile s3://logs/tmp.log --bucket obs-test ``` -opscli 会从每行中正则匹配 ip 地址，作为目标地址。 +`--bucket` 为 S3 bucket 名称，`--region` 为 S3 bucket 所在区域，`--endpoint` 为 S3 bucket 的 endpoint，`--direction` 为上传方向，`--localfile` 为本地文件，`--remotefile` 为远程文件。 -- 集群全部节点 +- 下载 S3 `s3://logs/tmp.log` 到本地文件 `./tmp1.log` ```bash --i ~/.kube/config --all +/usr/local/bin/opscli file --direction download --localfile ./tmp1.log --remotefile s3://logs/tmp.log --bucket obs-test ``` -`-i` 默认值为 `~/.kube/config`。 - -- 集群指定节点 +- 清理 AK\SK ```bash --i ~/.kube/config --nodename node1 +unset ak +unset sk ``` -node1 为节点名称。 +### 主机 - 本地与 API Server 互传文件 -### 本地文件与对象存储文件互传 +> 提供本地加解密，与服务器端进行文件传输 -- 设置 AK\SK +- 上传 ```bash -export ak= -export sk= +/usr/local/bin/opscli file --direction upload --api https://uploadapi.vinqi.com/api/v1/files --aeskey "" --localfile ./tmp.log + +Please use the following command to download the file: +opscli file --api https://uploadapi.vinqi.com/api/v1/files --aeskey xxx --direction download --remotefile https://download_url_link.com.aes ``` -- 上传本地文件 `./tmp.log` 到对象存储 `s3://logs/tmp.log` +这里的 api 提供上传服务，aeskey 为空字符串时自动生成一个随机秘钥，如果不设置 aeskey 默认为 unset 将不会进行文件加密。 + +- 下载 ```bash -/usr/local/bin/opscli file --direction upload --localfile ./tmp.log --remotefile s3://logs/tmp.log --bucket obs-test +/usr/local/bin/opscli file --api https://uploadapi.vinqi.com/api/v1/files --aeskey xxx --direction download --remotefile https://download_url_link.com.aes ``` -`--bucket` 为 S3 bucket 名称，`--region` 为 S3 bucket 所在区域，`--endpoint` 为 S3 bucket 的 endpoint，`--direction` 为上传方向，`--localfile` 为本地文件，`--remotefile` 为远程文件。 +### 集群 - 本地与 API Server 互传文件 -- 下载 S3 `s3://logs/tmp.log` 到本地文件 `./tmp1.log` +- 上传 ```bash -/usr/local/bin/opscli file --direction download --localfile ./tmp1.log --remotefile s3://logs/tmp.log --bucket obs-test +/usr/local/bin/opscli file -i ~/.kube/config --nodename node1 --direction upload --api https://uploadapi.vinqi.com/api/v1/files --aeskey "" --localfile /root/tmp.log --runtimeimage shaowenchen/ops-cli ``` -- 清理 AK\SK +- 下载 ```bash -unset ak -unset sk +/usr/local/bin/opscli file -i ~/.kube/config --nodename xxx --direction download --api https://uploadapi.vinqi.com/api/v1/files --aeskey xxx --localfile /root/tmp1.log --remotefile https://uploadapi.vinqi.com/uploadbases/cdn0/raw/1721621949-tmp.log.aes --runtimeimage shaowenchen/ops-cli ``` -### 本地文件分发到远程主机上 +### 集群 - 本地与对象存储互传文件 -- 上传本地文件 `./tmp.log` 到远程主机 `/tmp/tmp.log` +- 上传 ```bash -/usr/local/bin/opscli file --direction upload --localfile ./tmp.log --remotefile /tmp/tmp.log -i 1.2.3.4 --port 2222 --username root +/usr/local/bin/opscli file -i ~/.kube/config --nodename xxx --direction upload --ak xxx --sk xxx --region beijing --endpoint ks3-cn-beijing.ksyun.com --bucket multimodal --localfile /root/tmp.log --remotefile s3://logs/tmp.log --runtimeimage shaowenchen/ops-cli ``` -- 下载远程主机 `/tmp/tmp.log` 到本地文件 `./tmp1.log` +- 下载 ```bash -/usr/local/bin/opscli file --direction download --localfile ./tmp1.log --remotefile /tmp/tmp.log -i 1.2.3.4 --port 2222 --username root +/usr/local/bin/opscli file -i ~/.kube/config --nodename xxx --direction download --ak xxx --sk xxx --region beijing --endpoint ks3-cn-beijing.ksyun.com --bucket multimodal --localfile /root/tmp2.log --remotefile s3://logs/tmp.log --runtimeimage shaowenchen/ops-cli ``` -### 本地文件上传到 API Server，可加密 - -> 提供本地加解密，与服务器端进行文件传输 - -- 上传本地文件 `./tmp.log` 到 API Server +### 集群 - 镜像文件拷贝到本地 ```bash -/usr/local/bin/opscli file --direction upload --api https://uploadapi.vinqi.com/api/v1/files --aeskey "" --localfile ./tmp.log - -Please use the following command to download the file: -opscli file --api https://uploadapi.vinqi.com/api/v1/files --aeskey xxxxxxxxxxx --direction download --remotefile https://download_url_link.com.aes +/usr/local/bin/opscli file -i ~/.kube/config --nodename xxx --direction download --localfile /root/opscli-copy --remotefile shaowenchen/ops-cli:latest:///usr/local/bin/opscli ``` - -这里的 api 提供上传服务，aeskey 为空字符串时自动生成一个随机秘钥，如果不设置 aeskey 默认为 unset 将不会进行文件加密。 - -### 从镜像中提取文件到集群主机 diff --git a/docs/content/opscli-task.md b/docs/content/opscli-task.md index d9bee505..181b9375 100644 --- a/docs/content/opscli-task.md +++ b/docs/content/opscli-task.md @@ -75,4 +75,45 @@ node1 为节点名称。 ```bash /usr/local/bin/opscli task -f ~/.ops/tasks/app-istio.yaml --version 1.13.7 --kubeconfig /etc/kubernetes/admin.conf --action delete -``` \ No newline at end of file +``` + +### 上传文件 + +- 上传到 Server + +```bash +/usr/local/bin/opscli task -f tasks/file-upload.yaml --api https://uploadapi.vinqi.com/api/v1/files --localfile dockerfile + +> Run Task ops-system/file-upload on 127.0.0.1 +(1/1) upload file +Please use the following command to download the file: +opscli file --api https://uploadapi.vinqi.com/api/v1/files --aeskey a9f891afe71fda777b05a7063068360a914e83848d7da46d7513aee86c053f6c --direction download --remotefile https://uploadapi.vinqi.com/uploadbases/cdn0/raw/1721615659-dockerfile.aes +``` + +- 上传到 S3 + +```bash +/usr/local/bin/opscli task -f tasks/file-upload.yaml --ak xxx --sk xxx --region beijing --endpoint ks3-cn-beijing.ksyun.com --bucket xxx --localfile dockerfile --remotefile s3://dockerfile +``` + +### 下载文件 + +- 从 Server 下载 + +```bash +/usr/local/bin/opscli task -f task -f tasks/file-download.yaml --api https://uploadapi.vinqi.com/api/v1/files --aeskey a9f891afe71fda777b05a7063068360a914e83848d7da46d7513aee86c053f6c --remotefile https://uploadapi.vinqi.com/uploadbases/cdn0/raw/1721615659-dockerfile.aes --localfile dockerfile1 + +> Run Task ops-system/file-download on 127.0.0.1 +(1/1) download file +success download https://uploadapi.vinqi.com/uploadbases/cdn0/raw/1721615659-dockerfile.aes to dockerfile1 +``` + +- 从 S3 下载 + +```bash +/usr/local/bin/opscli task -f tasks/file-download.yaml --ak xxx --sk xxx --region beijing --endpoint ks3-cn-beijing.ksyun.com --bucket xxx --localfile dockerfile2 --remotefile s3://dockerfile + +> Run Task ops-system/file-download on 127.0.0.1 +(1/1) download file +success download s3 dockerfile to dockerfile2 +``` diff --git a/pkg/constants/kube.go b/pkg/constants/kube.go index 17953b57..09b34d45 100644 --- a/pkg/constants/kube.go +++ b/pkg/constants/kube.go @@ -22,6 +22,7 @@ const LabelOpsTaskValue = "task" const KubeAdminConfigPath = "/etc/kubernetes/admin.conf" const DefaultRuntimeImage = "docker.io/library/ubuntu:20.04" +const OpsCliRuntimeImage = "shaowenchen/opscli" const SyncResourceStatusHeatSeconds = 60 * 5 const SyncResourceRandomBiasSeconds = 60 * 2 diff --git a/pkg/kube/connect.go b/pkg/kube/connect.go index 668dc7ce..74dccfd4 100644 --- a/pkg/kube/connect.go +++ b/pkg/kube/connect.go @@ -212,7 +212,7 @@ func (kc *KubeConnection) FileNode(logger *opslog.Logger, node *corev1.Node, fil pod := &corev1.Pod{} if fileOpt.GetStorageType() == constants.RemoteStorageTypeS3 { if fileOpt.IsUploadDirection() { - pod, err = UploadS3FileOnNode(kc.Client, node, namespacedName, fileOpt) + pod, err = RunFileOnNode(kc.Client, node, namespacedName, fileOpt) if err != nil { return } diff --git a/pkg/kube/kube.go b/pkg/kube/kube.go index 75a4dfaa..a90ea39e 100644 --- a/pkg/kube/kube.go +++ b/pkg/kube/kube.go @@ -44,17 +44,13 @@ func File(logger *log.Logger, client *kubernetes.Clientset, node v1.Node, fileOp if err != nil { logger.Error.Println(err) } - if fileOpt.GetStorageType() == constants.RemoteStorageTypeS3 { - if fileOpt.IsUploadDirection() { - pod, err := UploadS3FileOnNode(client, &node, namespacedName, fileOpt) - if err != nil { - logger.Error.Println(err) - } - stdout, err = GetPodLog(logger, context.TODO(), false, client, pod) - logger.Info.Println(stdout) - } + pod := &v1.Pod{} + pod, err = RunFileOnNode(client, &node, namespacedName, fileOpt) + if err != nil { + logger.Error.Println(err) } - + stdout, err = GetPodLog(logger, context.TODO(), false, client, pod) + logger.Info.Println(stdout) return } diff --git a/pkg/kube/run.go b/pkg/kube/run.go index 3940f9ea..87e81e0d 100644 --- a/pkg/kube/run.go +++ b/pkg/kube/run.go @@ -2,7 +2,10 @@ package kube import ( "context" + "errors" "fmt" + "strings" + "github.com/shaowenchen/ops/pkg/constants" "github.com/shaowenchen/ops/pkg/option" "github.com/shaowenchen/ops/pkg/utils" @@ -11,7 +14,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" - "strings" ) func RunShellOnNode(client *kubernetes.Clientset, node *v1.Node, namespacedName types.NamespacedName, image string, shell string) (pod *corev1.Pod, err error) { @@ -96,7 +98,34 @@ func RunShellOnNode(client *kubernetes.Clientset, node *v1.Node, namespacedName return } -func UploadS3FileOnNode(client *kubernetes.Clientset, node *v1.Node, namespacedName types.NamespacedName, fileOpt option.FileOption) (pod *corev1.Pod, err error) { +func RunFileOnNode(client *kubernetes.Clientset, node *v1.Node, namespacedName types.NamespacedName, fileOpt option.FileOption) (pod *corev1.Pod, err error) { + hostLocalfile := "/host" + fileOpt.LocalFile + cmd := "" + switch fileOpt.GetStorageType() { + case constants.RemoteStorageTypeS3: + if fileOpt.IsDownloadDirection() { + cmd = utils.ShellOpscliDownS3(fileOpt.Region, fileOpt.Endpoint, fileOpt.Bucket, + fileOpt.AK, fileOpt.SK, hostLocalfile, fileOpt.RemoteFile) + } else if fileOpt.IsUploadDirection() { + cmd = utils.ShellOpscliUploadS3(fileOpt.Region, fileOpt.Endpoint, fileOpt.Bucket, + fileOpt.AK, fileOpt.SK, hostLocalfile, fileOpt.RemoteFile) + } + case constants.RemoteStorageTypeServer: + if fileOpt.IsDownloadDirection() { + cmd = utils.ShellOpscliDownServer(fileOpt.Api, fileOpt.AesKey, hostLocalfile, fileOpt.RemoteFile) + } else if fileOpt.IsUploadDirection() { + cmd = utils.ShellOpscliUploadServer(fileOpt.Api, fileOpt.AesKey, hostLocalfile, fileOpt.RemoteFile) + } + case constants.RemoteStorageTypeImage: + if fileOpt.IsDownloadDirection() { + cmd = fmt.Sprintf("cp -rbf %s %s", fileOpt.RemoteFile, hostLocalfile) + } + } + if cmd == "" { + err = errors.New("empty cmd") + return + } + tolerations := []v1.Toleration{} for _, taint := range node.Spec.Taints { tolerations = append(tolerations, v1.Toleration{ @@ -122,13 +151,11 @@ func UploadS3FileOnNode(client *kubernetes.Clientset, node *v1.Node, namespacedN NodeName: node.Name, Containers: []corev1.Container{ { - Name: "file", - Image: fileOpt.RuntimeImage, - Command: []string{"bash"}, - Args: []string{"-c", fmt.Sprintf("opscli file --direction upload"+ - " --endpoint %s --ak %s --sk %s --region %s --bucket %s --localfile /host%s --remotefile s3://%s", - fileOpt.Endpoint, fileOpt.AK, fileOpt.SK, fileOpt.Region, fileOpt.Bucket, fileOpt.LocalFile, fileOpt.Endpoint)}, - ImagePullPolicy: corev1.PullAlways, + Name: "file", + Image: fileOpt.RuntimeImage, + Command: []string{"bash"}, + Args: []string{"-c", cmd}, + ImagePullPolicy: corev1.PullIfNotPresent, VolumeMounts: []v1.VolumeMount{ { Name: "data", @@ -188,7 +215,7 @@ func DownloadS3FileOnNode(client *kubernetes.Clientset, node *v1.Node, namespace Args: []string{"-c", fmt.Sprintf("opscli file --direction upload"+ " --endpoint %s --ak %s --sk %s --region %s --bucket %s --localfile /host%s --remotefile s3://%s", fileOpt.Endpoint, fileOpt.AK, fileOpt.SK, fileOpt.Region, fileOpt.Bucket, fileOpt.LocalFile, fileOpt.LocalFile)}, - ImagePullPolicy: corev1.PullAlways, + ImagePullPolicy: corev1.PullIfNotPresent, VolumeMounts: []v1.VolumeMount{ { Name: "data", diff --git a/pkg/option/option.go b/pkg/option/option.go index 7a28cf5c..490fb24d 100644 --- a/pkg/option/option.go +++ b/pkg/option/option.go @@ -77,6 +77,7 @@ func (f *FileOption) GetStorageType() string { f.RemoteFile = remoteSplit[1] } else { f.StorageType = constants.RemoteStorageTypeImage + f.RuntimeImage = remoteSplit[0] f.RemoteFile = remoteSplit[1] } return f.StorageType diff --git a/pkg/storage/crypto.go b/pkg/storage/crypto.go index 35d289e0..df9af1d5 100644 --- a/pkg/storage/crypto.go +++ b/pkg/storage/crypto.go @@ -54,7 +54,6 @@ func decrypt(ciphertext []byte, key []byte) ([]byte, error) { iv := ciphertext[:aes.BlockSize] ciphertext = ciphertext[aes.BlockSize:] - // 使用AES加密算法中的CTR模式 stream := cipher.NewCTR(block, iv) stream.XORKeyStream(ciphertext, ciphertext) diff --git a/pkg/storage/s3.go b/pkg/storage/s3.go index feb4dc93..544963d2 100644 --- a/pkg/storage/s3.go +++ b/pkg/storage/s3.go @@ -2,13 +2,13 @@ package storage import ( "errors" - "fmt" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/s3/s3manager" "github.com/shaowenchen/ops/pkg/option" + "github.com/shaowenchen/ops/pkg/utils" "os" ) @@ -23,19 +23,19 @@ func S3File(fileOpt option.FileOption) (output string, err error) { err = errors.New("Please provide ak sk in params or env") return } - fmt.Println(fileOpt.RemoteFile) if fileOpt.IsUploadDirection() { - _, err = S3Upload(fileOpt.AK, fileOpt.SK, fileOpt.Region, fileOpt.Endpoint, fileOpt.Bucket, fileOpt.LocalFile, fileOpt.RemoteFile) + output, err = S3Upload(fileOpt.AK, fileOpt.SK, fileOpt.Region, fileOpt.Endpoint, fileOpt.Bucket, fileOpt.LocalFile, fileOpt.RemoteFile) } else if fileOpt.IsDownloadDirection() { - err = S3Download(fileOpt.AK, fileOpt.SK, fileOpt.Region, fileOpt.Endpoint, fileOpt.Bucket, fileOpt.LocalFile, fileOpt.RemoteFile) + output, err = S3Download(fileOpt.AK, fileOpt.SK, fileOpt.Region, fileOpt.Endpoint, fileOpt.Bucket, fileOpt.LocalFile, fileOpt.RemoteFile) } else { - err = errors.New("Please provide a valid direction") + output = "invalid direction" + err = errors.New(output) return } return } -func S3Upload(ak, sk, region, endpoint, bucket, localFilePath, remoteFile string) (location string, err error) { +func S3Upload(ak, sk, region, endpoint, bucket, localFilePath, remoteFile string) (output string, err error) { sess, _ := session.NewSession(&aws.Config{ Credentials: credentials.NewStaticCredentials(ak, sk, ""), Endpoint: aws.String(endpoint), @@ -62,10 +62,11 @@ func S3Upload(ak, sk, region, endpoint, bucket, localFilePath, remoteFile string if err != nil { return } - return result.Location, err + output = "success upload " + localFilePath + " to s3 " + result.Location + return } -func S3Download(ak, sk, region, endpoint, bucket, localFilePath, remoteFile string) (err error) { +func S3Download(ak, sk, region, endpoint, bucket, localFilePath, remoteFile string) (output string, err error) { sess, _ := session.NewSession(&aws.Config{ Credentials: credentials.NewStaticCredentials(ak, sk, ""), Endpoint: aws.String(endpoint), @@ -74,7 +75,7 @@ func S3Download(ak, sk, region, endpoint, bucket, localFilePath, remoteFile stri S3ForcePathStyle: aws.Bool(false), }) downloader := s3manager.NewDownloader(sess) - file, err := os.Create(localFilePath) + file, err := utils.CreateFile(localFilePath) if err != nil { return } @@ -86,5 +87,6 @@ func S3Download(ak, sk, region, endpoint, bucket, localFilePath, remoteFile stri if err != nil { return } + output = "success download s3 " + remoteFile + " to " + localFilePath return } diff --git a/pkg/storage/server.go b/pkg/storage/server.go index d38df656..5d81f527 100644 --- a/pkg/storage/server.go +++ b/pkg/storage/server.go @@ -5,7 +5,6 @@ import ( "encoding/hex" "errors" "fmt" - "github.com/shaowenchen/ops/pkg/option" "io" "mime/multipart" "net/http" @@ -13,6 +12,9 @@ import ( "path/filepath" "regexp" "strings" + + "github.com/shaowenchen/ops/pkg/option" + "github.com/shaowenchen/ops/pkg/utils" ) func ServerFile(fileOpt option.FileOption) (stdout string, err error) { @@ -29,9 +31,17 @@ func ServerFile(fileOpt option.FileOption) (stdout string, err error) { return } fileOpt.AesKey = string(aesKey) + } else { + aeskeyBytes, err1 := hex.DecodeString(fileOpt.AesKey) + if err1 != nil { + stdout = err1.Error() + return stdout, err + } + fileOpt.AesKey = string(aeskeyBytes) } tartgetFile := fileOpt.LocalFile + ".aes" err = EncryptFile(fileOpt.AesKey, fileOpt.LocalFile, tartgetFile) + defer os.Remove(tartgetFile) if err != nil { return } @@ -43,7 +53,7 @@ func ServerFile(fileOpt option.FileOption) (stdout string, err error) { err = err1 return } - stdout = "Please use the following command to download the file:" + + stdout = "Please use the following command to download the file: \n" + buildDowloadOpscliCmd(fileOpt.Api, resp, hex.EncodeToString([]byte(fileOpt.AesKey))) return } else if fileOpt.IsDownloadDirection() { @@ -54,15 +64,20 @@ func ServerFile(fileOpt option.FileOption) (stdout string, err error) { if err != nil { return } + targetFile := fileOpt.LocalFile if fileOpt.AesKey != UnSetFlag && strings.HasSuffix(fileOpt.LocalFile, ".aes") { - tartgetFile := strings.TrimSuffix(fileOpt.LocalFile, ".aes") - err = DecryptFile(fileOpt.AesKey, fileOpt.LocalFile, tartgetFile) + targetFile = strings.TrimSuffix(fileOpt.LocalFile, ".aes") + err = DecryptFile(fileOpt.AesKey, fileOpt.LocalFile, targetFile) + defer os.Remove(fileOpt.LocalFile) if err != nil { return } } + stdout = fmt.Sprintf("success download %s to %s", fileOpt.RemoteFile, targetFile) } else { - err = errors.New("Please provide a valid direction") + stdout = fmt.Sprintf("Unknown direction: %s", fileOpt.Direction) + err = errors.New(stdout) + } return } @@ -80,7 +95,7 @@ func buildDowloadOpscliCmd(api, resp, aesKey string) string { } func getFileToLocal(downloadUrl, localFilePath string) (err error) { - file, err := os.Create(localFilePath) + file, err := utils.CreateFile(localFilePath) if err != nil { return err } diff --git a/pkg/task/render.go b/pkg/task/render.go index 649c91ee..e9e80b2e 100644 --- a/pkg/task/render.go +++ b/pkg/task/render.go @@ -22,9 +22,9 @@ func GetRealVariables(t *opsv1.Task, taskOpt option.TaskOption) (map[string]stri globalVariables = RenderVarsVariables(globalVariables) // check variable in task is not empty - for key := range t.Spec.Variables { - if len(globalVariables[key]) == 0 { - return nil, errors.New("please set variable: " + key) + for k, v := range t.Spec.Variables { + if len(globalVariables[k]) == 0 && v.Required { + return nil, errors.New("please set variable: " + k) } } return globalVariables, nil diff --git a/pkg/task/task.go b/pkg/task/task.go index b23595b0..3288c1a1 100644 --- a/pkg/task/task.go +++ b/pkg/task/task.go @@ -130,10 +130,11 @@ func runStepFileOnHost(t *opsv1.Task, c *host.HostConnection, step opsv1.Step, t Direction: step.Direction, LocalFile: step.LocalFile, RemoteFile: step.RemoteFile, - AesKey: taskOpt.Variables["aeskey"], Api: taskOpt.Variables["api"], + AesKey: taskOpt.Variables["aeskey"], Region: taskOpt.Variables["region"], Endpoint: taskOpt.Variables["endpoint"], + Bucket: taskOpt.Variables["bucket"], AK: taskOpt.Variables["ak"], SK: taskOpt.Variables["sk"], } @@ -174,6 +175,8 @@ func runStepFileOnKube(logger *opslog.Logger, t *opsv1.Task, kc *kube.KubeConnec Direction: step.Direction, LocalFile: step.LocalFile, RemoteFile: step.RemoteFile, + Api: taskOpt.Variables["api"], + AesKey: taskOpt.Variables["aeskey"], AK: taskOpt.Variables["ak"], SK: taskOpt.Variables["sk"], Region: taskOpt.Variables["region"], diff --git a/pkg/utils/host.go b/pkg/utils/host.go index 01d8ed7d..96989ea4 100644 --- a/pkg/utils/host.go +++ b/pkg/utils/host.go @@ -10,6 +10,7 @@ import ( "path/filepath" "regexp" "strings" + "time" "github.com/shaowenchen/ops/pkg/constants" ) @@ -61,6 +62,28 @@ func CreateDir(dirpath string) error { return os.MkdirAll(dirpath, os.ModePerm) } +func CreateFile(localfile string) (file *os.File, err error) { + dir := filepath.Dir(localfile) + + err = os.MkdirAll(dir, os.ModePerm) + if err != nil { + return + } + + if _, err = os.Stat(localfile); err == nil { + newName := localfile + "." + time.Now().Format("2006-01-02-15-04-05-backup") + err = os.Rename(localfile, newName) + if err != nil { + return + } + } + file, err = os.Create(localfile) + if err != nil { + return + } + return +} + func FileMD5(path string) (string, error) { path = GetAbsoluteFilePath(path) file, err := os.Open(path) diff --git a/pkg/utils/script.go b/pkg/utils/script.go index ea19ae93..c2e566b6 100644 --- a/pkg/utils/script.go +++ b/pkg/utils/script.go @@ -8,19 +8,19 @@ import ( ) func ShellOpscliDownServer(api, aeskey, localfile, remotefile string) string { - return fmt.Sprintf(`opscli file --direction down --api %s --aeskey %s --localfile %s --remotefile %s`, api, aeskey, localfile, remotefile) + return fmt.Sprintf(`opscli file --direction "down" --api "%s" --aeskey "%s" --localfile "%s" --remotefile "%s"`, api, aeskey, localfile, remotefile) } func ShellOpscliUploadServer(api, aeskey, localfile, remotefile string) string { - return fmt.Sprintf(`opscli file --direction upload --api %s --aeskey %s --localfile %s --remotefile %s`, api, aeskey, localfile, remotefile) + return fmt.Sprintf(`opscli file --direction "upload" --api "%s" --aeskey "%s" --localfile "%s" --remotefile "%s"`, api, aeskey, localfile, remotefile) } func ShellOpscliDownS3(region, endpoint, bucket, ak, sk, localfile, remotefile string) string { - return fmt.Sprintf(`opscli file --direction down --region %s --endpoint %s --bucket %s --ak %s --sk %s --localfile %s --remotefile s3://%s`, region, endpoint, bucket, ak, sk, localfile, remotefile) + return fmt.Sprintf(`opscli file --direction "down" --region "%s" --endpoint "%s" --bucket "%s" --ak "%s" --sk "%s" --localfile "%s" --remotefile "s3://%s"`, region, endpoint, bucket, ak, sk, localfile, remotefile) } func ShellOpscliUploadS3(region, endpoint, bucket, ak, sk, localfile, remotefile string) string { - return fmt.Sprintf(`opscli file --direction upload --region %s --endpoint %s --bucket %s --ak %s --sk %s --localfile %s --remotefile s3://%s`, region, endpoint, bucket, ak, sk, localfile, remotefile) + return fmt.Sprintf(`opscli file --direction "upload" --region "%s" --endpoint "%s" --bucket "%s" --ak "%s" --sk "%s" --localfile "%s" --remotefile "s3://%s"`, region, endpoint, bucket, ak, sk, localfile, remotefile) } func ShellDownloadFile(proxy, sourceUrl, distPath string) string { diff --git a/tasks/file-download.yaml b/tasks/file-download.yaml new file mode 100644 index 00000000..3f3820e2 --- /dev/null +++ b/tasks/file-download.yaml @@ -0,0 +1,33 @@ +apiVersion: crd.chenshaowen.com/v1 +kind: Task +metadata: + name: file-download + namespace: ops-system +spec: + typeRef: cluster + desc: download file + runtimeImage: shaowenchen/ops-cli + variables: + api: + display: api server + aeskey: + display: aes key + ak: + display: access key + sk: + display: secret key + region: + display: region + endpoint: + display: endpoint + bucket: + display: bucket + remotefile: + display: remotefile + localfile: + display: localfile + steps: + - name: download file + remotefile: ${remotefile} + localfile: ${localfile} + direction: download diff --git a/tasks/file-upload.yaml b/tasks/file-upload.yaml new file mode 100644 index 00000000..77acb1db --- /dev/null +++ b/tasks/file-upload.yaml @@ -0,0 +1,33 @@ +apiVersion: crd.chenshaowen.com/v1 +kind: Task +metadata: + name: file-upload + namespace: ops-system +spec: + typeRef: cluster + desc: upload file + runtimeImage: shaowenchen/ops-cli + variables: + api: + display: api server + aeskey: + display: aes key + ak: + display: access key + sk: + display: secret key + region: + display: region + endpoint: + display: endpoint + bucket: + display: bucket + remotefile: + display: remotefile + localfile: + display: localfile + steps: + - name: upload file + remotefile: ${remotefile} + localfile: ${localfile} + direction: upload