diff --git a/crates/cli/src/commands/rb.rs b/crates/cli/src/commands/rb.rs index 21b4b5f..0b0f2d0 100644 --- a/crates/cli/src/commands/rb.rs +++ b/crates/cli/src/commands/rb.rs @@ -7,6 +7,7 @@ use rc_core::{AliasManager, ObjectStore as _}; use rc_s3::S3Client; use serde::Serialize; +use super::rm::{self, RmArgs}; use crate::exit_code::ExitCode; use crate::output::{Formatter, OutputConfig}; @@ -21,7 +22,7 @@ pub struct RbArgs { pub force: bool, /// Remove bucket even if it has incomplete multipart uploads - #[arg(long)] + #[arg(long, hide = true)] pub dangerous: bool, } @@ -37,10 +38,10 @@ struct RbOutput { pub async fn execute(args: RbArgs, output_config: OutputConfig) -> ExitCode { let formatter = Formatter::new(output_config); - if args.force || args.dangerous { + if args.dangerous { return formatter.fail( ExitCode::UnsupportedFeature, - "--force and --dangerous are not implemented for bucket removal", + "--dangerous is not implemented for bucket removal", ); } @@ -92,6 +93,25 @@ pub async fn execute(args: RbArgs, output_config: OutputConfig) -> ExitCode { } } + if args.force { + let rm_args = RmArgs { + paths: vec![format!("{alias_name}/{bucket}")], + recursive: true, + force: true, + dry_run: false, + incomplete: false, + versions: true, + bypass: false, + purge: true, + }; + + if let Err((code, _)) = + rm::delete_recursive(&client, &alias_name, &bucket, "", &rm_args, &formatter).await + { + return code; + } + } + // Delete the bucket match client.delete_bucket(&bucket).await { Ok(()) => { diff --git a/crates/cli/src/commands/rm.rs b/crates/cli/src/commands/rm.rs index 310b94b..4fc57c6 100644 --- a/crates/cli/src/commands/rm.rs +++ b/crates/cli/src/commands/rm.rs @@ -38,7 +38,7 @@ pub struct RmArgs { pub dry_run: bool, /// Remove incomplete multipart uploads older than specified duration - #[arg(long)] + #[arg(long, hide = true)] pub incomplete: bool, /// Include versions (requires versioning support) @@ -67,10 +67,10 @@ struct RmOutput { pub async fn execute(args: RmArgs, output_config: OutputConfig) -> ExitCode { let formatter = Formatter::new(output_config); - if args.incomplete || args.versions || args.bypass { + if args.incomplete { return formatter.fail( ExitCode::UnsupportedFeature, - "--incomplete, --versions, and --bypass are not implemented; refusing to continue with a silently ignored destructive option", + "--incomplete is not implemented", ); } @@ -244,7 +244,7 @@ async fn delete_single( } } -async fn delete_recursive( +pub(crate) async fn delete_recursive( client: &S3Client, alias_name: &str, bucket: &str, @@ -254,50 +254,11 @@ async fn delete_recursive( ) -> Result, (ExitCode, Vec)> { let path = RemotePath::new(alias_name, bucket, prefix); - // Collect all objects to delete - let mut keys_to_delete = Vec::new(); - let mut continuation_token: Option = None; - - loop { - let options = ListOptions { - recursive: true, - max_keys: Some(1000), - continuation_token: continuation_token.clone(), - ..Default::default() - }; - - match client.list_objects(&path, options).await { - Ok(result) => { - for item in result.items { - if !item.is_dir { - keys_to_delete.push(item.key); - } - } - - if result.truncated { - continuation_token = result.continuation_token; - } else { - break; - } - } - Err(e) => { - let err_str = e.to_string(); - if err_str.contains("NotFound") || err_str.contains("NoSuchBucket") { - let code = formatter.fail_with_suggestion( - ExitCode::NotFound, - &format!("Bucket not found: {bucket}"), - "Check the bucket path and retry the remove command.", - ); - return Err((code, vec![])); - } - let code = formatter.fail( - ExitCode::NetworkError, - &format!("Failed to list objects: {e}"), - ); - return Err((code, vec![])); - } - } - } + let keys_to_delete = if args.versions || args.purge { + list_version_keys(client, &path, bucket, formatter).await? + } else { + list_object_keys(client, &path, bucket, formatter).await? + }; if keys_to_delete.is_empty() { if !args.force { @@ -425,10 +386,110 @@ fn parse_rm_path(path: &str) -> Result<(String, String, String), String> { fn delete_request_options(args: &RmArgs) -> DeleteRequestOptions { DeleteRequestOptions { - force_delete: args.purge, + force_delete: args.purge || args.versions, + bypass_governance_retention: args.bypass, } } +async fn list_object_keys( + client: &S3Client, + path: &RemotePath, + bucket: &str, + formatter: &Formatter, +) -> Result, (ExitCode, Vec)> { + let mut keys_to_delete = Vec::new(); + let mut continuation_token: Option = None; + + loop { + let options = ListOptions { + recursive: true, + max_keys: Some(1000), + continuation_token: continuation_token.clone(), + ..Default::default() + }; + + match client.list_objects(path, options).await { + Ok(result) => { + for item in result.items { + if !item.is_dir { + keys_to_delete.push(item.key); + } + } + + if result.truncated { + continuation_token = result.continuation_token; + } else { + break; + } + } + Err(e) => return Err(list_error(e, bucket, formatter)), + } + } + + Ok(keys_to_delete) +} + +async fn list_version_keys( + client: &S3Client, + path: &RemotePath, + bucket: &str, + formatter: &Formatter, +) -> Result, (ExitCode, Vec)> { + let mut keys_to_delete = HashSet::new(); + let mut key_marker: Option = None; + let mut version_id_marker: Option = None; + + loop { + match client + .list_object_versions_page_with_markers( + path, + Some(1000), + key_marker.as_deref(), + version_id_marker.as_deref(), + ) + .await + { + Ok(result) => { + for item in result.items { + keys_to_delete.insert(item.key); + } + + if result.truncated { + key_marker = result.continuation_token; + version_id_marker = result.version_id_marker; + } else { + break; + } + } + Err(e) => return Err(list_error(e, bucket, formatter)), + } + } + + Ok(keys_to_delete.into_iter().collect()) +} + +fn list_error( + error: rc_core::Error, + bucket: &str, + formatter: &Formatter, +) -> (ExitCode, Vec) { + let err_str = error.to_string(); + if err_str.contains("NotFound") || err_str.contains("NoSuchBucket") { + let code = formatter.fail_with_suggestion( + ExitCode::NotFound, + &format!("Bucket not found: {bucket}"), + "Check the bucket path and retry the remove command.", + ); + return (code, vec![]); + } + + let code = formatter.fail( + ExitCode::NetworkError, + &format!("Failed to list objects: {error}"), + ); + (code, vec![]) +} + fn validate_removal_scope(key: &str, recursive: bool) -> Result<(), String> { if !recursive && (key.is_empty() || key.ends_with('/')) { return Err("Bucket and prefix removal requires the explicit --recursive flag".to_string()); @@ -526,6 +587,40 @@ mod tests { assert!(options.force_delete); } + #[test] + fn test_delete_request_options_enable_force_delete_for_versions() { + let args = RmArgs { + paths: vec!["test/bucket/object.txt".to_string()], + recursive: false, + force: false, + dry_run: false, + incomplete: false, + versions: true, + bypass: false, + purge: false, + }; + + let options = delete_request_options(&args); + assert!(options.force_delete); + } + + #[test] + fn test_delete_request_options_enable_governance_bypass() { + let args = RmArgs { + paths: vec!["test/bucket/object.txt".to_string()], + recursive: false, + force: false, + dry_run: false, + incomplete: false, + versions: false, + bypass: true, + purge: false, + }; + + let options = delete_request_options(&args); + assert!(options.bypass_governance_retention); + } + #[test] fn test_delete_request_options_keep_force_delete_disabled_by_default() { let args = RmArgs { diff --git a/crates/cli/tests/help_contract.rs b/crates/cli/tests/help_contract.rs index 1a0db13..32c8cf0 100644 --- a/crates/cli/tests/help_contract.rs +++ b/crates/cli/tests/help_contract.rs @@ -240,7 +240,6 @@ fn top_level_command_help_contract() { "--recursive", "--force", "--dry-run", - "--incomplete", "--versions", "--bypass", "--purge", @@ -268,7 +267,7 @@ fn top_level_command_help_contract() { HelpCase { args: &["rb"], usage: "Usage: rc rb [OPTIONS] ", - expected_tokens: &["--force", "--dangerous"], + expected_tokens: &["--force"], }, HelpCase { args: &["cat"], @@ -321,7 +320,6 @@ fn top_level_command_help_contract() { "--force", "--purge", "--dry-run", - "--incomplete", "--versions", "--bypass", "Examples:", diff --git a/crates/cli/tests/rm_purge.rs b/crates/cli/tests/rm_purge.rs index a729a7e..ac01f7f 100644 --- a/crates/cli/tests/rm_purge.rs +++ b/crates/cli/tests/rm_purge.rs @@ -233,6 +233,7 @@ fn header_value<'a>(headers: &'a [(String, String)], name: &str) -> Option<&'a s fn response_for(request: &CapturedRequest) -> String { match request.method.as_str() { "GET" if request.path.contains("list-type=2") => xml_response(200, list_objects_body()), + "GET" if request.path.contains("versions") => xml_response(200, list_versions_body()), "DELETE" => xml_response(204, ""), "POST" if request.path.contains("delete") => xml_response(200, delete_objects_body()), _ => xml_response(500, "UnexpectedRequest"), @@ -277,6 +278,42 @@ fn list_objects_body() -> &'static str { "# } +fn list_versions_body() -> &'static str { + r#" + + bucket + purge-prefix/ + + + 1000 + false + + purge-prefix/a.txt + v1 + false + 2026-05-01T00:00:00.000Z + "etag-a" + 1 + STANDARD + + + purge-prefix/a.txt + v2 + true + 2026-05-01T00:00:01.000Z + + + purge-prefix/nested/b.txt + v3 + true + 2026-05-01T00:00:00.000Z + "etag-b" + 1 + STANDARD + +"# +} + fn delete_objects_body() -> &'static str { r#" @@ -319,12 +356,12 @@ fn rm_recursive_purge_deletes_each_key_with_force_header() { let requests = server.captured_requests(); let list_requests: Vec<_> = requests .iter() - .filter(|request| request.method == "GET" && request.path.contains("list-type=2")) + .filter(|request| request.method == "GET" && request.path.contains("versions")) .collect(); assert_eq!(list_requests.len(), 1, "requests: {requests:#?}"); assert!( list_requests[0].path.contains("prefix=purge-prefix%2F"), - "list request should include the recursive prefix: {requests:#?}" + "version list request should include the recursive prefix: {requests:#?}" ); let delete_requests: Vec<_> = requests diff --git a/crates/s3/src/client.rs b/crates/s3/src/client.rs index 4bd14fc..709bcac 100644 --- a/crates/s3/src/client.rs +++ b/crates/s3/src/client.rs @@ -50,6 +50,7 @@ const SINGLE_PUT_OBJECT_MAX_SIZE: u64 = crate::multipart::DEFAULT_PART_SIZE; const S3_SERVICE_NAME: &str = "s3"; const S3_REPLICATION_XML_NAMESPACE: &str = "http://s3.amazonaws.com/doc/2006-03-01/"; const RUSTFS_FORCE_DELETE_HEADER: &str = "x-rustfs-force-delete"; +const S3_BYPASS_GOVERNANCE_RETENTION_HEADER: &str = "x-amz-bypass-governance-retention"; static DOWNLOAD_TEMP_SEQUENCE: AtomicU64 = AtomicU64::new(0); #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -861,6 +862,8 @@ pub struct S3Client { pub struct DeleteRequestOptions { /// Ask RustFS to permanently delete data instead of creating delete markers. pub force_delete: bool, + /// Ask S3-compatible servers to bypass governance retention. + pub bypass_governance_retention: bool, } #[derive(Debug, Clone)] @@ -1349,6 +1352,13 @@ impl S3Client { .insert(RUSTFS_FORCE_DELETE_HEADER, "true"); }); } + if options.bypass_governance_retention { + request = request.mutate_request(|request| { + request + .headers_mut() + .insert(S3_BYPASS_GOVERNANCE_RETENTION_HEADER, "true"); + }); + } request.send().await.map_err(|e| { let err_str = Self::format_sdk_error(&e); @@ -1418,6 +1428,13 @@ impl S3Client { .insert(RUSTFS_FORCE_DELETE_HEADER, "true"); }); } + if options.bypass_governance_retention { + request = request.mutate_request(|request| { + request + .headers_mut() + .insert(S3_BYPASS_GOVERNANCE_RETENTION_HEADER, "true"); + }); + } let response = request .send() @@ -4473,13 +4490,41 @@ mod tests { let path = RemotePath::new("test", "bucket", "key.txt"); let _ = client - .delete_object_with_options(&path, DeleteRequestOptions { force_delete: true }) + .delete_object_with_options( + &path, + DeleteRequestOptions { + force_delete: true, + ..Default::default() + }, + ) .await; let request = request_receiver.expect_request(); assert_eq!(request.headers().get("x-rustfs-force-delete"), Some("true")); } + #[tokio::test] + async fn delete_object_with_bypass_sets_governance_header() { + let (client, request_receiver) = test_s3_client(None); + let path = RemotePath::new("test", "bucket", "key.txt"); + + let _ = client + .delete_object_with_options( + &path, + DeleteRequestOptions { + bypass_governance_retention: true, + ..Default::default() + }, + ) + .await; + + let request = request_receiver.expect_request(); + assert_eq!( + request.headers().get("x-amz-bypass-governance-retention"), + Some("true") + ); + } + #[tokio::test] async fn conditional_mirror_writes_and_deletes_set_precondition_headers() { let put_response = http::Response::builder() @@ -5104,7 +5149,10 @@ mod tests { .delete_objects_with_options( "bucket", vec!["key.txt".to_string()], - DeleteRequestOptions { force_delete: true }, + DeleteRequestOptions { + force_delete: true, + ..Default::default() + }, ) .await; @@ -5112,6 +5160,35 @@ mod tests { assert_eq!(request.headers().get("x-rustfs-force-delete"), Some("true")); } + #[tokio::test] + async fn delete_objects_with_bypass_sets_governance_header() { + let response = http::Response::builder() + .status(200) + .body(SdkBody::from( + r#" +"#, + )) + .expect("build delete objects response"); + let (client, request_receiver) = test_s3_client(Some(response)); + + let _ = client + .delete_objects_with_options( + "bucket", + vec!["key.txt".to_string()], + DeleteRequestOptions { + bypass_governance_retention: true, + ..Default::default() + }, + ) + .await; + + let request = request_receiver.expect_request(); + assert_eq!( + request.headers().get("x-amz-bypass-governance-retention"), + Some("true") + ); + } + #[tokio::test] async fn delete_objects_without_force_delete_omits_rustfs_header() { let response = http::Response::builder()