use crate::client::TokenCredentialProvider;
use crate::gcp::client::{GoogleCloudStorageClient, GoogleCloudStorageConfig};
use crate::gcp::credential::{
ApplicationDefaultCredentials, InstanceCredentialProvider, ServiceAccountCredentials,
DEFAULT_GCS_BASE_URL,
};
use crate::gcp::{
credential, GcpCredential, GcpCredentialProvider, GcpSigningCredential,
GcpSigningCredentialProvider, GoogleCloudStorage, STORE,
};
use crate::{ClientConfigKey, ClientOptions, Result, RetryConfig, StaticCredentialProvider};
use serde::{Deserialize, Serialize};
use snafu::{OptionExt, ResultExt, Snafu};
use std::str::FromStr;
use std::sync::Arc;
use url::Url;
use super::credential::{AuthorizedUserSigningCredentials, InstanceSigningCredentialProvider};
#[derive(Debug, Snafu)]
enum Error {
#[snafu(display("Missing bucket name"))]
MissingBucketName {},
#[snafu(display("One of service account path or service account key may be provided."))]
ServiceAccountPathAndKeyProvided,
#[snafu(display("Unable parse source url. Url: {}, Error: {}", url, source))]
UnableToParseUrl {
source: url::ParseError,
url: String,
},
#[snafu(display(
"Unknown url scheme cannot be parsed into storage location: {}",
scheme
))]
UnknownUrlScheme { scheme: String },
#[snafu(display("URL did not match any known pattern for scheme: {}", url))]
UrlNotRecognised { url: String },
#[snafu(display("Configuration key: '{}' is not known.", key))]
UnknownConfigurationKey { key: String },
#[snafu(display("GCP credential error: {}", source))]
Credential { source: credential::Error },
}
impl From<Error> for crate::Error {
fn from(err: Error) -> Self {
match err {
Error::UnknownConfigurationKey { key } => {
Self::UnknownConfigurationKey { store: STORE, key }
}
_ => Self::Generic {
store: STORE,
source: Box::new(err),
},
}
}
}
#[derive(Debug, Clone)]
pub struct GoogleCloudStorageBuilder {
bucket_name: Option<String>,
url: Option<String>,
service_account_path: Option<String>,
service_account_key: Option<String>,
application_credentials_path: Option<String>,
retry_config: RetryConfig,
client_options: ClientOptions,
credentials: Option<GcpCredentialProvider>,
signing_credentials: Option<GcpSigningCredentialProvider>,
}
#[derive(PartialEq, Eq, Hash, Clone, Debug, Copy, Serialize, Deserialize)]
#[non_exhaustive]
pub enum GoogleConfigKey {
ServiceAccount,
ServiceAccountKey,
Bucket,
ApplicationCredentials,
Client(ClientConfigKey),
}
impl AsRef<str> for GoogleConfigKey {
fn as_ref(&self) -> &str {
match self {
Self::ServiceAccount => "google_service_account",
Self::ServiceAccountKey => "google_service_account_key",
Self::Bucket => "google_bucket",
Self::ApplicationCredentials => "google_application_credentials",
Self::Client(key) => key.as_ref(),
}
}
}
impl FromStr for GoogleConfigKey {
type Err = crate::Error;
fn from_str(s: &str) -> Result<Self, Self::Err> {
match s {
"google_service_account"
| "service_account"
| "google_service_account_path"
| "service_account_path" => Ok(Self::ServiceAccount),
"google_service_account_key" | "service_account_key" => Ok(Self::ServiceAccountKey),
"google_bucket" | "google_bucket_name" | "bucket" | "bucket_name" => Ok(Self::Bucket),
"google_application_credentials" => Ok(Self::ApplicationCredentials),
_ => match s.parse() {
Ok(key) => Ok(Self::Client(key)),
Err(_) => Err(Error::UnknownConfigurationKey { key: s.into() }.into()),
},
}
}
}
impl Default for GoogleCloudStorageBuilder {
fn default() -> Self {
Self {
bucket_name: None,
service_account_path: None,
service_account_key: None,
application_credentials_path: None,
retry_config: Default::default(),
client_options: ClientOptions::new().with_allow_http(true),
url: None,
credentials: None,
signing_credentials: None,
}
}
}
impl GoogleCloudStorageBuilder {
pub fn new() -> Self {
Default::default()
}
pub fn from_env() -> Self {
let mut builder = Self::default();
if let Ok(service_account_path) = std::env::var("SERVICE_ACCOUNT") {
builder.service_account_path = Some(service_account_path);
}
for (os_key, os_value) in std::env::vars_os() {
if let (Some(key), Some(value)) = (os_key.to_str(), os_value.to_str()) {
if key.starts_with("GOOGLE_") {
if let Ok(config_key) = key.to_ascii_lowercase().parse() {
builder = builder.with_config(config_key, value);
}
}
}
}
builder
}
pub fn with_url(mut self, url: impl Into<String>) -> Self {
self.url = Some(url.into());
self
}
pub fn with_config(mut self, key: GoogleConfigKey, value: impl Into<String>) -> Self {
match key {
GoogleConfigKey::ServiceAccount => self.service_account_path = Some(value.into()),
GoogleConfigKey::ServiceAccountKey => self.service_account_key = Some(value.into()),
GoogleConfigKey::Bucket => self.bucket_name = Some(value.into()),
GoogleConfigKey::ApplicationCredentials => {
self.application_credentials_path = Some(value.into())
}
GoogleConfigKey::Client(key) => {
self.client_options = self.client_options.with_config(key, value)
}
};
self
}
pub fn get_config_value(&self, key: &GoogleConfigKey) -> Option<String> {
match key {
GoogleConfigKey::ServiceAccount => self.service_account_path.clone(),
GoogleConfigKey::ServiceAccountKey => self.service_account_key.clone(),
GoogleConfigKey::Bucket => self.bucket_name.clone(),
GoogleConfigKey::ApplicationCredentials => self.application_credentials_path.clone(),
GoogleConfigKey::Client(key) => self.client_options.get_config_value(key),
}
}
fn parse_url(&mut self, url: &str) -> Result<()> {
let parsed = Url::parse(url).context(UnableToParseUrlSnafu { url })?;
let host = parsed.host_str().context(UrlNotRecognisedSnafu { url })?;
match parsed.scheme() {
"gs" => self.bucket_name = Some(host.to_string()),
scheme => return Err(UnknownUrlSchemeSnafu { scheme }.build().into()),
}
Ok(())
}
pub fn with_bucket_name(mut self, bucket_name: impl Into<String>) -> Self {
self.bucket_name = Some(bucket_name.into());
self
}
pub fn with_service_account_path(mut self, service_account_path: impl Into<String>) -> Self {
self.service_account_path = Some(service_account_path.into());
self
}
pub fn with_service_account_key(mut self, service_account: impl Into<String>) -> Self {
self.service_account_key = Some(service_account.into());
self
}
pub fn with_application_credentials(
mut self,
application_credentials_path: impl Into<String>,
) -> Self {
self.application_credentials_path = Some(application_credentials_path.into());
self
}
pub fn with_credentials(mut self, credentials: GcpCredentialProvider) -> Self {
self.credentials = Some(credentials);
self
}
pub fn with_retry(mut self, retry_config: RetryConfig) -> Self {
self.retry_config = retry_config;
self
}
pub fn with_proxy_url(mut self, proxy_url: impl Into<String>) -> Self {
self.client_options = self.client_options.with_proxy_url(proxy_url);
self
}
pub fn with_proxy_ca_certificate(mut self, proxy_ca_certificate: impl Into<String>) -> Self {
self.client_options = self
.client_options
.with_proxy_ca_certificate(proxy_ca_certificate);
self
}
pub fn with_proxy_excludes(mut self, proxy_excludes: impl Into<String>) -> Self {
self.client_options = self.client_options.with_proxy_excludes(proxy_excludes);
self
}
pub fn with_client_options(mut self, options: ClientOptions) -> Self {
self.client_options = options;
self
}
pub fn build(mut self) -> Result<GoogleCloudStorage> {
if let Some(url) = self.url.take() {
self.parse_url(&url)?;
}
let bucket_name = self.bucket_name.ok_or(Error::MissingBucketName {})?;
let service_account_credentials =
match (self.service_account_path, self.service_account_key) {
(Some(path), None) => {
Some(ServiceAccountCredentials::from_file(path).context(CredentialSnafu)?)
}
(None, Some(key)) => {
Some(ServiceAccountCredentials::from_key(&key).context(CredentialSnafu)?)
}
(None, None) => None,
(Some(_), Some(_)) => return Err(Error::ServiceAccountPathAndKeyProvided.into()),
};
let application_default_credentials =
ApplicationDefaultCredentials::read(self.application_credentials_path.as_deref())?;
let disable_oauth = service_account_credentials
.as_ref()
.map(|c| c.disable_oauth)
.unwrap_or(false);
let gcs_base_url: String = service_account_credentials
.as_ref()
.and_then(|c| c.gcs_base_url.clone())
.unwrap_or_else(|| DEFAULT_GCS_BASE_URL.to_string());
let credentials = if let Some(credentials) = self.credentials {
credentials
} else if disable_oauth {
Arc::new(StaticCredentialProvider::new(GcpCredential {
bearer: "".to_string(),
})) as _
} else if let Some(credentials) = service_account_credentials.clone() {
Arc::new(TokenCredentialProvider::new(
credentials.token_provider()?,
self.client_options.client()?,
self.retry_config.clone(),
)) as _
} else if let Some(credentials) = application_default_credentials.clone() {
match credentials {
ApplicationDefaultCredentials::AuthorizedUser(token) => {
Arc::new(TokenCredentialProvider::new(
token,
self.client_options.client()?,
self.retry_config.clone(),
)) as _
}
ApplicationDefaultCredentials::ServiceAccount(token) => {
Arc::new(TokenCredentialProvider::new(
token.token_provider()?,
self.client_options.client()?,
self.retry_config.clone(),
)) as _
}
}
} else {
Arc::new(TokenCredentialProvider::new(
InstanceCredentialProvider::default(),
self.client_options.metadata_client()?,
self.retry_config.clone(),
)) as _
};
let signing_credentials = if let Some(signing_credentials) = self.signing_credentials {
signing_credentials
} else if disable_oauth {
Arc::new(StaticCredentialProvider::new(GcpSigningCredential {
email: "".to_string(),
private_key: None,
})) as _
} else if let Some(credentials) = service_account_credentials.clone() {
credentials.signing_credentials()?
} else if let Some(credentials) = application_default_credentials.clone() {
match credentials {
ApplicationDefaultCredentials::AuthorizedUser(token) => {
Arc::new(TokenCredentialProvider::new(
AuthorizedUserSigningCredentials::from(token)?,
self.client_options.client()?,
self.retry_config.clone(),
)) as _
}
ApplicationDefaultCredentials::ServiceAccount(token) => {
token.signing_credentials()?
}
}
} else {
Arc::new(TokenCredentialProvider::new(
InstanceSigningCredentialProvider::default(),
self.client_options.metadata_client()?,
self.retry_config.clone(),
)) as _
};
let config = GoogleCloudStorageConfig::new(
gcs_base_url,
credentials,
signing_credentials,
bucket_name,
self.retry_config,
self.client_options,
);
Ok(GoogleCloudStorage {
client: Arc::new(GoogleCloudStorageClient::new(config)?),
})
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::collections::HashMap;
use std::io::Write;
use tempfile::NamedTempFile;
const FAKE_KEY: &str = r#"{"private_key": "private_key", "private_key_id": "private_key_id", "client_email":"client_email", "disable_oauth":true}"#;
#[test]
fn gcs_test_service_account_key_and_path() {
let mut tfile = NamedTempFile::new().unwrap();
write!(tfile, "{FAKE_KEY}").unwrap();
let _ = GoogleCloudStorageBuilder::new()
.with_service_account_key(FAKE_KEY)
.with_service_account_path(tfile.path().to_str().unwrap())
.with_bucket_name("foo")
.build()
.unwrap_err();
}
#[test]
fn gcs_test_config_from_map() {
let google_service_account = "object_store:fake_service_account".to_string();
let google_bucket_name = "object_store:fake_bucket".to_string();
let options = HashMap::from([
("google_service_account", google_service_account.clone()),
("google_bucket_name", google_bucket_name.clone()),
]);
let builder = options
.iter()
.fold(GoogleCloudStorageBuilder::new(), |builder, (key, value)| {
builder.with_config(key.parse().unwrap(), value)
});
assert_eq!(
builder.service_account_path.unwrap(),
google_service_account.as_str()
);
assert_eq!(builder.bucket_name.unwrap(), google_bucket_name.as_str());
}
#[test]
fn gcs_test_config_aliases() {
for alias in [
"google_service_account",
"service_account",
"google_service_account_path",
"service_account_path",
] {
let builder = GoogleCloudStorageBuilder::new()
.with_config(alias.parse().unwrap(), "/fake/path.json");
assert_eq!("/fake/path.json", builder.service_account_path.unwrap());
}
for alias in ["google_service_account_key", "service_account_key"] {
let builder =
GoogleCloudStorageBuilder::new().with_config(alias.parse().unwrap(), FAKE_KEY);
assert_eq!(FAKE_KEY, builder.service_account_key.unwrap());
}
for alias in [
"google_bucket",
"google_bucket_name",
"bucket",
"bucket_name",
] {
let builder =
GoogleCloudStorageBuilder::new().with_config(alias.parse().unwrap(), "fake_bucket");
assert_eq!("fake_bucket", builder.bucket_name.unwrap());
}
}
#[tokio::test]
async fn gcs_test_proxy_url() {
let mut tfile = NamedTempFile::new().unwrap();
write!(tfile, "{FAKE_KEY}").unwrap();
let service_account_path = tfile.path();
let gcs = GoogleCloudStorageBuilder::new()
.with_service_account_path(service_account_path.to_str().unwrap())
.with_bucket_name("foo")
.with_proxy_url("https://example.com")
.build();
assert!(gcs.is_ok());
let err = GoogleCloudStorageBuilder::new()
.with_service_account_path(service_account_path.to_str().unwrap())
.with_bucket_name("foo")
.with_proxy_url("asdf://example.com")
.build()
.unwrap_err()
.to_string();
assert_eq!("Generic HTTP client error: builder error", err);
}
#[test]
fn gcs_test_urls() {
let mut builder = GoogleCloudStorageBuilder::new();
builder.parse_url("gs://bucket/path").unwrap();
assert_eq!(builder.bucket_name.as_deref(), Some("bucket"));
builder.parse_url("gs://bucket.mydomain/path").unwrap();
assert_eq!(builder.bucket_name.as_deref(), Some("bucket.mydomain"));
builder.parse_url("mailto://bucket/path").unwrap_err();
}
#[test]
fn gcs_test_service_account_key_only() {
let _ = GoogleCloudStorageBuilder::new()
.with_service_account_key(FAKE_KEY)
.with_bucket_name("foo")
.build()
.unwrap();
}
#[test]
fn gcs_test_config_get_value() {
let google_service_account = "object_store:fake_service_account".to_string();
let google_bucket_name = "object_store:fake_bucket".to_string();
let builder = GoogleCloudStorageBuilder::new()
.with_config(GoogleConfigKey::ServiceAccount, &google_service_account)
.with_config(GoogleConfigKey::Bucket, &google_bucket_name);
assert_eq!(
builder
.get_config_value(&GoogleConfigKey::ServiceAccount)
.unwrap(),
google_service_account
);
assert_eq!(
builder.get_config_value(&GoogleConfigKey::Bucket).unwrap(),
google_bucket_name
);
}
}