From e0e88ef032b86f3445e86a28aa9648aa4429c730 Mon Sep 17 00:00:00 2001 From: Ketan Umare Date: Thu, 17 Oct 2019 23:29:35 -0700 Subject: [PATCH] Support signed URL's from minio. (#17) * Sign on minio * missing --- Gopkg.lock | 34 +++++++++++++++------- Gopkg.toml | 3 ++ flyteadmin_config.yaml | 2 +- pkg/data/factory.go | 33 ++++++++++++++++++++- pkg/data/implementations/aws_remote_url.go | 1 + 5 files changed, 60 insertions(+), 13 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index 6158e8cf8..b7df52dfe 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -18,12 +18,13 @@ version = "v10.2.1-beta" [[projects]] - digest = "1:e3d8dd629da696c66c2275f911078ce51a420520346c73330a84b938401585b7" + digest = "1:0f857a863c24bb1c277a5f3f8cb8a30e65b841405f69590132ee23ed9e7e6fbe" name = "github.com/Azure/go-autorest" packages = [ "autorest", "autorest/adal", "autorest/azure", + "autorest/date", "logger", "tracing", ] @@ -53,7 +54,7 @@ version = "v1.0.7" [[projects]] - digest = "1:8242149eca9920ef3ac114d07791f8d7cc05b347ffc0b40141b1627a5b378011" + digest = "1:c068593b2d873e3d100eb6fd1ea1715070ad588e8dcdb356b7c0d9530063e256" name = "github.com/aws/aws-sdk-go" packages = [ "aws", @@ -106,8 +107,8 @@ "service/sts/stsiface", ] pruneopts = "UT" - revision = "e40068abd62ea29db38cb7ca59bf6f58489343bf" - version = "v1.25.13" + revision = "adf0d7dcf139d718ab18e46f6055735a27be867a" + version = "v1.25.15" [[projects]] branch = "master" @@ -165,6 +166,14 @@ revision = "8991bc29aa16c548c550c7ff78260e27b9ab7c73" version = "v1.1.1" +[[projects]] + digest = "1:76dc72490af7174349349838f2fe118996381b31ea83243812a97e5a0fd5ed55" + name = "github.com/dgrijalva/jwt-go" + packages = ["."] + pruneopts = "UT" + revision = "06ea1031745cb8b3dab3f6a236daf2b0aa468b7e" + version = "v3.2.0" + [[projects]] digest = "1:ac425d784b13d49b37a5bbed3ce022677f8f3073b216f05d6adcb9303e27fa0f" name = "github.com/evanphx/json-patch" @@ -374,12 +383,12 @@ revision = "c2b33e84" [[projects]] - digest = "1:709cd2a2c29cc9b89732f6c24846bbb9d6270f28ef5ef2128cc73bd0d6d7bff9" + digest = "1:beb5b4f42a25056f0aa291b5eadd21e2f2903a05d15dfe7caf7eaee7e12fa972" name = "github.com/json-iterator/go" packages = ["."] pruneopts = "UT" - revision = "27518f6661eba504be5a7a9a9f6d9460d892ade3" - version = "v1.1.7" + revision = "03217c3e97663914aec3faafde50d081f197a0a2" + version = "1.1.8" [[projects]] digest = "1:fd9bea48bbc5bba66d9891c72af7255fbebecdff845c37c679406174ece5ca1b" @@ -480,9 +489,9 @@ "version", ] pruneopts = "UT" - revision = "2a121b0e3bc7fb85e36a5088e23ef59d401a78be" + revision = "9d0448cd08805c3ae39938d8fcaec34d11db366c" source = "https://github.com/lyft/flytestdlib" - version = "v0.2.22" + version = "v0.2.24" [[projects]] digest = "1:2a0da3440db3f2892609d99cd0389c2776a3fef24435f7b7b58bfc9030aa86ca" @@ -814,7 +823,7 @@ [[projects]] branch = "master" - digest = "1:4b234cf90d8b57ad0794e6c9f3d76ce806aaa851adb658d25d9f78db458b2800" + digest = "1:2b7fccff5bf6caff757f5e468d580c8d3b763cc7b9354f127adf14fe54451428" name = "google.golang.org/api" packages = [ "googleapi", @@ -828,7 +837,7 @@ "transport/http/internal/propagation", ] pruneopts = "UT" - revision = "4f26c6409ffa7b94f89d06aac760b63875d9b42c" + revision = "1c098db1e8d0c17c2819910afd08052313af7a44" [[projects]] digest = "1:3c03b58f57452764a4499c55c582346c0ee78c8a5033affe5bdfd9efd3da5bd1" @@ -1097,6 +1106,7 @@ "github.com/Selvatico/go-mocket", "github.com/aws/aws-sdk-go/aws", "github.com/aws/aws-sdk-go/aws/awserr", + "github.com/aws/aws-sdk-go/aws/credentials", "github.com/aws/aws-sdk-go/aws/request", "github.com/aws/aws-sdk-go/aws/session", "github.com/aws/aws-sdk-go/service/cloudwatchevents", @@ -1112,6 +1122,8 @@ "github.com/golang/protobuf/ptypes/duration", "github.com/golang/protobuf/ptypes/struct", "github.com/golang/protobuf/ptypes/timestamp", + "github.com/graymeta/stow", + "github.com/graymeta/stow/s3", "github.com/grpc-ecosystem/go-grpc-prometheus", "github.com/grpc-ecosystem/grpc-gateway/runtime", "github.com/jinzhu/gorm", diff --git a/Gopkg.toml b/Gopkg.toml index 27aec45fe..3b177b92e 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -116,3 +116,6 @@ go-tests = true unused-packages = true +[[constraint]] + name = "github.com/graymeta/stow" + revision = "903027f87de7054953efcdb8ba70d5dc02df38c7" diff --git a/flyteadmin_config.yaml b/flyteadmin_config.yaml index 377126c61..2da678978 100644 --- a/flyteadmin_config.yaml +++ b/flyteadmin_config.yaml @@ -57,7 +57,7 @@ notifications: http://example.com/projects/{{ project }}/domains/{{ domain }}/executions/{{ name }}. {{ error }} Logger: show-source: true - level: 5 + level: 6 storage: type: minio connection: diff --git a/pkg/data/factory.go b/pkg/data/factory.go index abf263721..b259f3ca1 100644 --- a/pkg/data/factory.go +++ b/pkg/data/factory.go @@ -4,6 +4,9 @@ import ( "context" "time" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/graymeta/stow" + "github.com/graymeta/stow/s3" "github.com/lyft/flytestdlib/logger" "github.com/lyft/flytestdlib/storage" @@ -43,7 +46,35 @@ func GetRemoteDataHandler(cfg RemoteDataHandlerConfig) RemoteDataHandler { remoteURL: implementations.NewAWSRemoteURL(awsConfig, presignedURLDuration), } case common.Local: - fallthrough + logger.Infof(context.TODO(), "setting up local signer ----- ") + // Since minio = aws s3, we are creating the same client but using the config primitives from aws + storageCfg := storage.GetConfig() + accessKeyID := "" + secret := "" + endpoint := "" + if storageCfg.Stow != nil { + stowCfg := stow.ConfigMap(storageCfg.Stow.Config) + accessKeyID, _ = stowCfg.Config(s3.ConfigAccessKeyID) + secret, _ = stowCfg.Config(s3.ConfigSecretKey) + endpoint, _ = stowCfg.Config(s3.ConfigEndpoint) + } else { + accessKeyID = storageCfg.Connection.AccessKey + secret = storageCfg.Connection.SecretKey + endpoint = storageCfg.Connection.Endpoint.String() + } + logger.Infof(context.TODO(), "setting up local signer - %s, %s, %s", accessKeyID, secret, endpoint) + creds := credentials.NewStaticCredentials(accessKeyID, secret, "") + awsConfig := aws.NewConfig(). + WithRegion(cfg.Region). + WithMaxRetries(cfg.Retries). + WithCredentials(creds). + WithEndpoint(endpoint). + WithDisableSSL(true). + WithS3ForcePathStyle(true) + presignedURLDuration := time.Minute * time.Duration(cfg.SignedURLDurationMinutes) + return &remoteDataHandler{ + remoteURL: implementations.NewAWSRemoteURL(awsConfig, presignedURLDuration), + } default: logger.Infof(context.Background(), "Using default noop remote url implementation for cloud provider type [%s]", cfg.CloudProvider) diff --git a/pkg/data/implementations/aws_remote_url.go b/pkg/data/implementations/aws_remote_url.go index 2e2df74c4..7301e2f6f 100644 --- a/pkg/data/implementations/aws_remote_url.go +++ b/pkg/data/implementations/aws_remote_url.go @@ -53,6 +53,7 @@ func (a *AWSRemoteURL) splitURI(ctx context.Context, uri string) (AWSS3Object, e } func (a *AWSRemoteURL) Get(ctx context.Context, uri string) (admin.UrlBlob, error) { + logger.Debugf(ctx, "Getting signed url for - %s", uri) s3URI, err := a.splitURI(ctx, uri) if err != nil { logger.Debugf(ctx, "failed to extract s3 bucket and key from uri: %s", uri)