diff --git a/Cargo.toml b/Cargo.toml index e28177b..ebf7e2d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,7 @@ members = ["codegen", "examples", "performance_measurement", "performance_measur [package] name = "worktable" -version = "0.9.0-alpha6" +version = "0.9.0-alpha7" edition = "2024" authors = ["Handy-caT"] license = "MIT" @@ -17,40 +17,37 @@ s3-support = ["dep:rust-s3", "dep:aws-creds", "dep:aws-region", "dep:walkdir", " # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -worktable_codegen = { path = "codegen", version = "=0.9.0-alpha4" } - async-trait = "0.1.89" -eyre = "0.6.12" -derive_more = { version = "2.0.1", features = ["from", "error", "display", "debug", "into"] } -tokio = { version = "1", features = ["full"] } -tracing = "0.1" -rkyv = { version = "0.8.9", features = ["uuid-1"] } -lockfree = { version = "0.5.1" } -fastrand = "2.3.0" -futures = "0.3.30" -uuid = { version = "1.10.0", features = ["v4", "v7"] } -psc-nanoid = { version = "3.1.1", features = ["rkyv", "packed"] } +aws-creds = { version = "0.39", optional = true, default-features = false, features = ["rustls-tls"] } +aws-region = { version = "0.28", optional = true } +convert_case = "0.6.0" data_bucket = "=0.3.12" # data_bucket = { git = "https://github.com/pathscale/DataBucket", branch = "page_cdc_correction", version = "0.2.7" } # data_bucket = { path = "../DataBucket", version = "0.3.11" } -performance_measurement_codegen = { path = "performance_measurement/codegen", version = "0.1.0", optional = true } -performance_measurement = { path = "performance_measurement", version = "0.1.0", optional = true } +derive_more = { version = "2.0.1", features = ["from", "error", "display", "debug", "into"] } +eyre = "0.6.12" +fastrand = "2.3.0" +futures = "0.3.30" indexset = { version = "=0.14.0", features = ["concurrent", "cdc", "multimap"] } # indexset = { package = "wt-indexset", path = "../indexset", version = "0.12.10", features = ["concurrent", "cdc", "multimap"] } # indexset = { package = "wt-indexset", version = "=0.12.12", features = ["concurrent", "cdc", "multimap"] } -convert_case = "0.6.0" +lockfree = { version = "0.5.1" } +log = "0.4.29" ordered-float = "5.0.0" parking_lot = "0.12.3" +performance_measurement = { path = "performance_measurement", version = "0.1.0", optional = true } +performance_measurement_codegen = { path = "performance_measurement/codegen", version = "0.1.0", optional = true } prettytable-rs = "^0.10" -smart-default = "0.7.1" -log = "0.4.29" - -# S3 support (optional) +psc-nanoid = { version = "3.1.1", features = ["rkyv", "packed"] } +rkyv = { version = "0.8.9", features = ["uuid-1"] } rust-s3 = { version = "0.37", optional = true, default-features = false, features = ["tokio-rustls-tls"] } -aws-creds = { version = "0.39", optional = true, default-features = false, features = ["rustls-tls"] } -aws-region = { version = "0.28", optional = true } +smart-default = "0.7.1" +tokio = { version = "1", features = ["full"] } +tracing = "0.1" +uuid = { version = "1.10.0", features = ["v4", "v7"] } walkdir = { version = "2", optional = true } +worktable_codegen = { path = "codegen", version = "=0.9.0-alpha4" } [dev-dependencies] -rand = "0.9.1" chrono = "0.4.43" +rand = "0.9.1" diff --git a/src/features/s3_support.rs b/src/features/s3_support.rs index 4ca21f4..84aead1 100644 --- a/src/features/s3_support.rs +++ b/src/features/s3_support.rs @@ -8,13 +8,15 @@ use awsregion::Region; use s3::Bucket; use walkdir::WalkDir; -use crate::TableSecondaryIndexEventsOps; use crate::persistence::operation::{BatchOperation, Operation}; use crate::persistence::{ DiskConfig, DiskPersistenceEngine, PersistenceConfig, PersistenceEngine, SpaceDataOps, SpaceIndexOps, SpaceSecondaryIndexOps, }; -use crate::prelude::{PrimaryKeyGeneratorState, TablePrimaryKey}; +use crate::prelude::{ + PrimaryKeyGeneratorState, TablePrimaryKey, WT_DATA_EXTENSION, WT_INDEX_EXTENSION, +}; +use crate::TableSecondaryIndexEventsOps; #[derive(Debug, Clone)] pub struct S3Config { @@ -142,8 +144,6 @@ where .ok_or_else(|| eyre::eyre!("Invalid table path"))?; let s3_key = Self::full_s3_path(prefix, &relative.to_string_lossy(), table_name); - println!("Syncing S3 to {}", s3_key); - println!("File {} {}", local_path.display(), s3_key); tracing::debug!(local_path = %local_path.display(), s3_key = %s3_key, "Uploading file to S3"); let content = tokio::fs::read(local_path).await?; @@ -190,6 +190,12 @@ where let filename = s3_key.rsplit('/').next().unwrap_or(s3_key); + if !filename.ends_with(WT_DATA_EXTENSION) && !filename.ends_with(WT_INDEX_EXTENSION) + { + tracing::debug!(s3_key = %s3_key, "Skipping non-table file"); + continue; + } + let local_path = table_path.join(filename); tracing::debug!(s3_key = %s3_key, local_path = %local_path.display(), "Downloading file from S3");