diff --git a/Cargo.lock b/Cargo.lock index 3ddc12ed..a536c4aa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -182,7 +182,7 @@ dependencies = [ "futures-lite", "parking", "polling", - "rustix 1.0.7", + "rustix", "slab", "tracing", "windows-sys 0.59.0", @@ -214,7 +214,7 @@ dependencies = [ "cfg-if", "event-listener 5.4.0", "futures-lite", - "rustix 1.0.7", + "rustix", "tracing", ] @@ -230,7 +230,7 @@ dependencies = [ "cfg-if", "futures-core", "futures-io", - "rustix 1.0.7", + "rustix", "signal-hook-registry", "slab", "windows-sys 0.59.0", @@ -371,29 +371,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "aws-lc-rs" -version = "1.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93fcc8f365936c834db5514fc45aee5b1202d677e6b40e48468aaaa8183ca8c7" -dependencies = [ - "aws-lc-sys", - "zeroize", -] - -[[package]] -name = "aws-lc-sys" -version = "0.29.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61b1d86e7705efe1be1b569bab41d4fa1e14e220b60a160f78de2db687add079" -dependencies = [ - "bindgen", - "cc", - "cmake", - "dunce", - "fs_extra", -] - [[package]] name = "aws-runtime" version = "1.5.7" @@ -538,29 +515,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "aws-smithy-http-client" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e44697a9bded898dcd0b1cb997430d949b87f4f8940d91023ae9062bf218250" -dependencies = [ - "aws-smithy-async", - "aws-smithy-runtime-api", - "aws-smithy-types", - "h2", - "http 1.3.1", - "hyper 1.6.0", - "hyper-rustls", - "hyper-util", - "pin-project-lite", - "rustls 0.23.27", - "rustls-native-certs", - "rustls-pki-types", - "tokio", - "tower", - "tracing", -] - [[package]] name = "aws-smithy-json" version = "0.61.3" @@ -597,7 +551,6 @@ checksum = "14302f06d1d5b7d333fd819943075b13d27c7700b414f574c3c35859bfb55d5e" dependencies = [ "aws-smithy-async", "aws-smithy-http", - "aws-smithy-http-client", "aws-smithy-observability", "aws-smithy-runtime-api", "aws-smithy-types", @@ -755,29 +708,6 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "383d29d513d8764dcdc42ea295d979eb99c3c9f00607b3692cf68a431f7dca72" -[[package]] -name = "bindgen" -version = "0.69.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" -dependencies = [ - "bitflags", - "cexpr", - "clang-sys", - "itertools", - "lazy_static", - "lazycell", - "log", - "prettyplease", - "proc-macro2", - "quote", - "regex", - "rustc-hash 1.1.0", - "shlex", - "syn", - "which 4.4.2", -] - [[package]] name = "bitflags" version = "2.9.1" @@ -930,32 +860,15 @@ version = "1.2.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0fc897dc1e865cc67c0e05a836d9d3f1df3cbe442aa4a9473b18e12624a4951" dependencies = [ - "jobserver", - "libc", "shlex", ] -[[package]] -name = "cexpr" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" -dependencies = [ - "nom 7.1.3", -] - [[package]] name = "cfg-if" version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" -[[package]] -name = "cfg_aliases" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" - [[package]] name = "chrono" version = "0.4.41" @@ -1012,26 +925,6 @@ dependencies = [ "inout", ] -[[package]] -name = "clang-sys" -version = "1.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" -dependencies = [ - "glob", - "libc", - "libloading", -] - -[[package]] -name = "cmake" -version = "0.1.54" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7caa3f9de89ddbe2c607f4101924c5abec803763ae9534e4f4d7d8f84aa81f0" -dependencies = [ - "cc", -] - [[package]] name = "codemap" version = "0.1.3" @@ -1112,16 +1005,6 @@ dependencies = [ "libc", ] -[[package]] -name = "core-foundation" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" -dependencies = [ - "core-foundation-sys", - "libc", -] - [[package]] name = "core-foundation-sys" version = "0.8.7" @@ -1517,12 +1400,6 @@ dependencies = [ "syn", ] -[[package]] -name = "dunce" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" - [[package]] name = "either" version = "1.15.0" @@ -1696,12 +1573,6 @@ dependencies = [ "percent-encoding", ] -[[package]] -name = "fs_extra" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" - [[package]] name = "futures" version = "0.3.31" @@ -2226,12 +2097,10 @@ dependencies = [ "hyper 1.6.0", "hyper-util", "rustls 0.23.27", - "rustls-native-certs", "rustls-pki-types", "tokio", "tokio-rustls 0.26.2", "tower-service", - "webpki-roots", ] [[package]] @@ -2479,15 +2348,6 @@ dependencies = [ "windows-sys 0.59.0", ] -[[package]] -name = "itertools" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" -dependencies = [ - "either", -] - [[package]] name = "itoa" version = "1.0.15" @@ -2511,16 +2371,6 @@ dependencies = [ "uuid", ] -[[package]] -name = "jobserver" -version = "0.1.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38f262f097c174adebe41eb73d66ae9c06b2844fb0da69969647bbddd9b0538a" -dependencies = [ - "getrandom 0.3.3", - "libc", -] - [[package]] name = "js-sys" version = "0.3.77" @@ -2573,12 +2423,6 @@ dependencies = [ "spin", ] -[[package]] -name = "lazycell" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" - [[package]] name = "lettre" version = "0.11.16" @@ -2616,16 +2460,6 @@ version = "0.2.172" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa" -[[package]] -name = "libloading" -version = "0.8.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07033963ba89ebaf1584d767badaa2e8fcec21aedea6b8c0346d487d49c28667" -dependencies = [ - "cfg-if", - "windows-targets 0.53.0", -] - [[package]] name = "libm" version = "0.2.15" @@ -2653,12 +2487,6 @@ dependencies = [ "vcpkg", ] -[[package]] -name = "linux-raw-sys" -version = "0.4.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" - [[package]] name = "linux-raw-sys" version = "0.9.4" @@ -2724,12 +2552,6 @@ dependencies = [ "tracing-subscriber", ] -[[package]] -name = "lru-slab" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" - [[package]] name = "macros" version = "0.1.0" @@ -2886,7 +2708,7 @@ dependencies = [ "openssl-probe", "openssl-sys", "schannel", - "security-framework 2.11.1", + "security-framework", "security-framework-sys", "tempfile", ] @@ -3457,7 +3279,7 @@ dependencies = [ "concurrent-queue", "hermit-abi 0.5.1", "pin-project-lite", - "rustix 1.0.7", + "rustix", "tracing", "windows-sys 0.59.0", ] @@ -3502,16 +3324,6 @@ dependencies = [ "vcpkg", ] -[[package]] -name = "prettyplease" -version = "0.2.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "664ec5419c51e34154eec046ebcba56312d5a2fc3b09a06da188e1ad21afadf6" -dependencies = [ - "proc-macro2", - "syn", -] - [[package]] name = "proc-macro2" version = "1.0.95" @@ -3584,61 +3396,6 @@ dependencies = [ "serde", ] -[[package]] -name = "quinn" -version = "0.11.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "626214629cda6781b6dc1d316ba307189c85ba657213ce642d9c77670f8202c8" -dependencies = [ - "bytes", - "cfg_aliases", - "pin-project-lite", - "quinn-proto", - "quinn-udp", - "rustc-hash 2.1.1", - "rustls 0.23.27", - "socket2", - "thiserror 2.0.12", - "tokio", - "tracing", - "web-time", -] - -[[package]] -name = "quinn-proto" -version = "0.11.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49df843a9161c85bb8aae55f101bc0bac8bcafd637a620d9122fd7e0b2f7422e" -dependencies = [ - "bytes", - "getrandom 0.3.3", - "lru-slab", - "rand 0.9.1", - "ring", - "rustc-hash 2.1.1", - "rustls 0.23.27", - "rustls-pki-types", - "slab", - "thiserror 2.0.12", - "tinyvec", - "tracing", - "web-time", -] - -[[package]] -name = "quinn-udp" -version = "0.5.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee4e529991f949c5e25755532370b8af5d114acae52326361d68d47af64aa842" -dependencies = [ - "cfg_aliases", - "libc", - "once_cell", - "socket2", - "tracing", - "windows-sys 0.59.0", -] - [[package]] name = "quote" version = "1.0.40" @@ -3892,8 +3649,6 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "quinn", - "rustls 0.23.27", "rustls-pki-types", "serde", "serde_json", @@ -3901,7 +3656,6 @@ dependencies = [ "sync_wrapper", "tokio", "tokio-native-tls", - "tokio-rustls 0.26.2", "tokio-socks", "tokio-util", "tower", @@ -3912,7 +3666,6 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams", "web-sys", - "webpki-roots", ] [[package]] @@ -4110,18 +3863,6 @@ version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" -[[package]] -name = "rustc-hash" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" - -[[package]] -name = "rustc-hash" -version = "2.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" - [[package]] name = "rustc_version" version = "0.4.1" @@ -4131,19 +3872,6 @@ dependencies = [ "semver", ] -[[package]] -name = "rustix" -version = "0.38.44" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" -dependencies = [ - "bitflags", - "errno", - "libc", - "linux-raw-sys 0.4.15", - "windows-sys 0.59.0", -] - [[package]] name = "rustix" version = "1.0.7" @@ -4153,7 +3881,7 @@ dependencies = [ "bitflags", "errno", "libc", - "linux-raw-sys 0.9.4", + "linux-raw-sys", "windows-sys 0.59.0", ] @@ -4175,27 +3903,13 @@ version = "0.23.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "730944ca083c1c233a75c09f199e973ca499344a2b7ba9e755c457e86fb4a321" dependencies = [ - "aws-lc-rs", "once_cell", - "ring", "rustls-pki-types", "rustls-webpki 0.103.3", "subtle", "zeroize", ] -[[package]] -name = "rustls-native-certs" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fcff2dd52b58a8d98a70243663a0d234c4e2b79235637849d15913394a247d3" -dependencies = [ - "openssl-probe", - "rustls-pki-types", - "schannel", - "security-framework 3.2.0", -] - [[package]] name = "rustls-pemfile" version = "1.0.4" @@ -4211,7 +3925,6 @@ version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "229a4a4c221013e7e1f1a043678c5cc39fe5171437c88fb47151a21e6f5b5c79" dependencies = [ - "web-time", "zeroize", ] @@ -4231,7 +3944,6 @@ version = "0.103.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e4a72fe2bcf7a6ac6fd7d0b9e5cb68aeb7d4c0a0271730218b3e92d43b4eb435" dependencies = [ - "aws-lc-rs", "ring", "rustls-pki-types", "untrusted", @@ -4325,20 +4037,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" dependencies = [ "bitflags", - "core-foundation 0.9.4", - "core-foundation-sys", - "libc", - "security-framework-sys", -] - -[[package]] -name = "security-framework" -version = "3.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "271720403f46ca04f7ba6f55d438f8bd878d6b8ca0a1046e8228c4145bcbb316" -dependencies = [ - "bitflags", - "core-foundation 0.10.1", + "core-foundation", "core-foundation-sys", "libc", "security-framework-sys", @@ -4656,7 +4355,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" dependencies = [ "bitflags", - "core-foundation 0.9.4", + "core-foundation", "system-configuration-sys", ] @@ -4685,7 +4384,7 @@ dependencies = [ "fastrand", "getrandom 0.3.3", "once_cell", - "rustix 1.0.7", + "rustix", "windows-sys 0.59.0", ] @@ -5225,6 +4924,7 @@ dependencies = [ "argon2", "aws-config", "aws-credential-types", + "aws-smithy-runtime-api", "bigdecimal", "bytes", "cached", @@ -5249,6 +4949,7 @@ dependencies = [ "handlebars", "hickory-resolver", "html5gum", + "http 1.3.1", "job_scheduler_ng", "jsonwebtoken", "lettre", @@ -5286,7 +4987,7 @@ dependencies = [ "url", "uuid", "webauthn-rs", - "which 7.0.3", + "which", "yubico_ng", ] @@ -5465,27 +5166,6 @@ dependencies = [ "url", ] -[[package]] -name = "webpki-roots" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2853738d1cc4f2da3a225c18ec6c3721abb31961096e9dbf5ab35fa88b19cfdb" -dependencies = [ - "rustls-pki-types", -] - -[[package]] -name = "which" -version = "4.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" -dependencies = [ - "either", - "home", - "once_cell", - "rustix 0.38.44", -] - [[package]] name = "which" version = "7.0.3" @@ -5494,7 +5174,7 @@ checksum = "24d643ce3fd3e5b54854602a080f34fb10ab75e0b813ee32d00ca2b44fa74762" dependencies = [ "either", "env_home", - "rustix 1.0.7", + "rustix", "winsafe", ] diff --git a/Cargo.toml b/Cargo.toml index a0e8a61b..1811e4c1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,7 +32,7 @@ enable_mimalloc = ["dep:mimalloc"] # You also need to set an env variable `QUERY_LOGGER=1` to fully activate this so you do not have to re-compile # if you want to turn off the logging for a specific run. query_logger = ["dep:diesel_logger"] -s3 = ["opendal/services-s3", "dep:aws-config", "dep:aws-credential-types", "dep:anyhow", "dep:reqsign"] +s3 = ["opendal/services-s3", "dep:aws-config", "dep:aws-credential-types", "dep:aws-smithy-runtime-api", "dep:anyhow", "dep:http", "dep:reqsign"] # Enable unstable features, requires nightly # Currently only used to enable rusts official ip support @@ -179,12 +179,14 @@ rpassword = "7.4.0" grass_compiler = { version = "0.13.4", default-features = false } # File are accessed through Apache OpenDAL -opendal = { version = "0.53.3", features = ["services-fs"] } +opendal = { version = "0.53.3", features = ["services-fs"], default-features = false } # For retrieving AWS credentials, including temporary SSO credentials anyhow = { version = "1.0.98", optional = true } -aws-config = { version = "1.6.3", features = ["behavior-version-latest"], optional = true } +aws-config = { version = "1.6.3", features = ["behavior-version-latest", "rt-tokio", "credentials-process", "sso"], default-features = false, optional = true } aws-credential-types = { version = "1.2.3", optional = true } +aws-smithy-runtime-api = { version = "1.8.0", optional = true } +http = { version = "1.3.1", optional = true } reqsign = { version = "0.16.3", optional = true } # Strip debuginfo from the release builds diff --git a/src/config.rs b/src/config.rs index 5b995c6d..f0a0aea4 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1188,6 +1188,9 @@ fn opendal_operator_for_path(path: &str) -> Result { #[cfg(s3)] fn opendal_s3_operator_for_path(path: &str) -> Result { + use crate::http_client::aws::AwsReqwestConnector; + use aws_config::{default_provider::credentials::DefaultCredentialsChain, provider_config::ProviderConfig}; + // This is a custom AWS credential loader that uses the official AWS Rust // SDK config crate to load credentials. This ensures maximum compatibility // with AWS credential configurations. For example, OpenDAL doesn't support @@ -1200,12 +1203,19 @@ fn opendal_s3_operator_for_path(path: &str) -> Result use aws_credential_types::provider::ProvideCredentials as _; use tokio::sync::OnceCell; - static DEFAULT_CREDENTIAL_CHAIN: OnceCell< - aws_config::default_provider::credentials::DefaultCredentialsChain, - > = OnceCell::const_new(); + static DEFAULT_CREDENTIAL_CHAIN: OnceCell = OnceCell::const_new(); let chain = DEFAULT_CREDENTIAL_CHAIN - .get_or_init(|| aws_config::default_provider::credentials::DefaultCredentialsChain::builder().build()) + .get_or_init(|| { + let reqwest_client = reqwest::Client::builder().build().unwrap(); + let connector = AwsReqwestConnector { + client: reqwest_client, + }; + + let conf = ProviderConfig::default().with_http_client(connector); + + DefaultCredentialsChain::builder().configure(conf).build() + }) .await; let creds = chain.provide_credentials().await?; diff --git a/src/http_client.rs b/src/http_client.rs index 5feeeedd..fc3b1b42 100644 --- a/src/http_client.rs +++ b/src/http_client.rs @@ -244,3 +244,61 @@ impl Resolve for CustomDnsResolver { }) } } + +#[cfg(s3)] +pub(crate) mod aws { + use aws_smithy_runtime_api::client::{ + http::{HttpClient, HttpConnector, HttpConnectorFuture, HttpConnectorSettings, SharedHttpConnector}, + orchestrator::HttpResponse, + result::ConnectorError, + runtime_components::RuntimeComponents, + }; + use reqwest::Client; + + // Adapter that wraps reqwest to be compatible with the AWS SDK + #[derive(Debug)] + pub(crate) struct AwsReqwestConnector { + pub(crate) client: Client, + } + + impl HttpConnector for AwsReqwestConnector { + fn call(&self, request: aws_smithy_runtime_api::client::orchestrator::HttpRequest) -> HttpConnectorFuture { + // Convert the AWS-style request to a reqwest request + let client = self.client.clone(); + let future = async move { + let method = reqwest::Method::from_bytes(request.method().as_bytes()) + .map_err(|e| ConnectorError::user(Box::new(e)))?; + let mut req_builder = client.request(method, request.uri().to_string()); + + for (name, value) in request.headers() { + req_builder = req_builder.header(name, value); + } + + if let Some(body_bytes) = request.body().bytes() { + req_builder = req_builder.body(body_bytes.to_vec()); + } + + let response = req_builder.send().await.map_err(|e| ConnectorError::io(Box::new(e)))?; + + let status = response.status().into(); + let bytes = response.bytes().await.map_err(|e| ConnectorError::io(Box::new(e)))?; + + Ok(HttpResponse::new(status, bytes.into())) + }; + + HttpConnectorFuture::new(Box::pin(future)) + } + } + + impl HttpClient for AwsReqwestConnector { + fn http_connector( + &self, + _settings: &HttpConnectorSettings, + _components: &RuntimeComponents, + ) -> SharedHttpConnector { + SharedHttpConnector::new(AwsReqwestConnector { + client: self.client.clone(), + }) + } + } +}