diff --git a/crates/y-sweet-core/src/store/s3.rs b/crates/y-sweet-core/src/store/s3.rs index 8cd210e5..13ac0801 100644 --- a/crates/y-sweet-core/src/store/s3.rs +++ b/crates/y-sweet-core/src/store/s3.rs @@ -18,6 +18,9 @@ pub struct S3Config { pub bucket: String, pub region: String, pub bucket_prefix: Option, + + // Use old path-style URLs, needed to support some S3-compatible APIs (including some minio setups) + pub path_style: bool, } const PRESIGNED_URL_DURATION: Duration = Duration::from_secs(60 * 60); @@ -38,14 +41,18 @@ impl S3Store { Credentials::new(config.key, config.secret) }; let endpoint: Url = config.endpoint.parse().expect("endpoint is a valid url"); - let path_style = - // if endpoint is localhost then bucket url must be of form http://localhost:/ - // instead of :://. - if endpoint.host_str().expect("endpoint Url should have host") == "localhost" { - rusty_s3::UrlStyle::Path - } else { - rusty_s3::UrlStyle::VirtualHost - }; + + let path_style = if config.path_style { + rusty_s3::UrlStyle::Path + } else if endpoint.host_str() == Some("localhost") { + // Since this was the old behavior before we added AWS_S3_USE_PATH_STYLE, + // we continue to support it, but complain a bit. + tracing::warn!("Inferring path-style URLs for localhost for backwards-compatibility. This behavior may change in the future. Set AWS_S3_USE_PATH_STYLE=true to ensure that path-style URLs are used."); + rusty_s3::UrlStyle::Path + } else { + rusty_s3::UrlStyle::VirtualHost + }; + let bucket = Bucket::new(endpoint, path_style, config.bucket, config.region) .expect("Url has a valid scheme and host"); let client = Client::new(); diff --git a/crates/y-sweet-worker/Cargo.lock b/crates/y-sweet-worker/Cargo.lock index 9d494fdc..8229655c 100644 --- a/crates/y-sweet-worker/Cargo.lock +++ b/crates/y-sweet-worker/Cargo.lock @@ -1533,7 +1533,7 @@ dependencies = [ [[package]] name = "y-sweet-core" -version = "0.2.2" +version = "0.3.0" dependencies = [ "anyhow", "async-trait", diff --git a/crates/y-sweet-worker/src/config.rs b/crates/y-sweet-worker/src/config.rs index 30b4b322..8aab0ac1 100644 --- a/crates/y-sweet-worker/src/config.rs +++ b/crates/y-sweet-worker/src/config.rs @@ -88,6 +88,7 @@ fn parse_s3_config(env: &Env) -> anyhow::Result { .map_err(|_| anyhow::anyhow!("S3_BUCKET_NAME env var not supplied"))? .to_string(), bucket_prefix: env.var(S3_BUCKET_PREFIX).ok().map(|t| t.to_string()), + path_style: false, }) } diff --git a/crates/y-sweet/src/main.rs b/crates/y-sweet/src/main.rs index 114d2a1a..42b896e2 100644 --- a/crates/y-sweet/src/main.rs +++ b/crates/y-sweet/src/main.rs @@ -89,7 +89,26 @@ const S3_SECRET_ACCESS_KEY: &str = "AWS_SECRET_ACCESS_KEY"; const S3_SESSION_TOKEN: &str = "AWS_SESSION_TOKEN"; const S3_REGION: &str = "AWS_REGION"; const S3_ENDPOINT: &str = "AWS_ENDPOINT_URL_S3"; -fn parse_s3_config_from_env_and_args(bucket: String, prefix: String) -> anyhow::Result { +const S3_USE_PATH_STYLE: &str = "AWS_S3_USE_PATH_STYLE"; +fn parse_s3_config_from_env_and_args( + bucket: String, + prefix: Option, +) -> anyhow::Result { + let use_path_style = env::var(S3_USE_PATH_STYLE).ok(); + let path_style = if let Some(use_path_style) = use_path_style { + if use_path_style.to_lowercase() == "true" { + true + } else if use_path_style.to_lowercase() == "false" || use_path_style.is_empty() { + false + } else { + anyhow::bail!( + "If AWS_S3_USE_PATH_STYLE is set, it must be either \"true\" or \"false\"" + ) + } + } else { + false + }; + Ok(S3Config { key: env::var(S3_ACCESS_KEY_ID) .map_err(|_| anyhow::anyhow!("{} env var not supplied", S3_ACCESS_KEY_ID))?, @@ -104,7 +123,9 @@ fn parse_s3_config_from_env_and_args(bucket: String, prefix: String) -> anyhow:: .map_err(|_| anyhow::anyhow!("{} env var not supplied", S3_SECRET_ACCESS_KEY))?, token: env::var(S3_SESSION_TOKEN).ok(), bucket, - bucket_prefix: Some(prefix), + bucket_prefix: prefix, + // If the endpoint is overridden, we assume that the user wants path-style URLs. + path_style, }) } @@ -116,6 +137,7 @@ fn get_store_from_opts(store_path: &str) -> Result> { .ok_or_else(|| anyhow::anyhow!("Invalid S3 URL"))? .to_owned(); let bucket_prefix = url.path().trim_start_matches('/').to_owned(); + let bucket_prefix = (!bucket_prefix.is_empty()).then(|| bucket_prefix); // "" => None let config = parse_s3_config_from_env_and_args(bucket, bucket_prefix)?; let store = S3Store::new(config); Ok(Box::new(store)) @@ -239,6 +261,7 @@ async fn main() -> Result<()> { let store = match (bucket, prefix) { (Some(bucket), Some(prefix)) => { + let prefix = (!prefix.is_empty()).then(|| prefix); let s3_config = parse_s3_config_from_env_and_args(bucket, prefix) .context("Failed to parse S3 configuration")?; let store = S3Store::new(s3_config);