i/Aether
1
0
Fork 0
mirror of https://github.com/fawney19/Aether.git synced 2026-10-07 09:57:47 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(security): harden gateway boundaries and usage policies

Browse Source 
Consolidate subscription usage policy enforcement, privacy-safe persistence, and gateway security hardening into one reviewable change.

Includes bounded HTTP and execution envelopes, header and protocol guards, DNS and relay validation, authentication and secret projection hardening, secure backup/install paths, and regression coverage.
This commit is contained in:
elky
2026-09-04 03:45:52 +08:00
parent ddcbeb3ae9
commit 579f2c7cc1
1019 changed files with 190437 additions and 26080 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/aether-gateway/src/backup/config.rs
+137 -3
View File
@@ -5,7 +5,7 @@ use serde_json::{Map, Value};
use super::schedule::{BackupSchedule, BackupScheduleUnit};
use super::scopes::BackupScope;
#[derive(Debug, Clone, PartialEq, Eq)]
#[derive(Clone, PartialEq, Eq)]
pub(crate) struct S3BackupConfig {
pub(crate) enabled: bool,
pub(crate) scope: BackupScope,
@@ -22,6 +22,28 @@ pub(crate) struct S3BackupConfig {
pub(crate) retention_count: u32,
}
impl fmt::Debug for S3BackupConfig {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
let endpoint_origin = sanitized_endpoint_origin(&self.endpoint);
formatter
.debug_struct("S3BackupConfig")
.field("enabled", &self.enabled)
.field("scope", &self.scope)
.field("endpoint_origin", &endpoint_origin)
.field("region", &self.region)
.field("user_agent", &self.user_agent)
.field("bucket", &self.bucket)
.field("prefix", &self.prefix)
.field("has_access_key_id", &!self.access_key_id.is_empty())
.field("has_secret_access_key", &!self.secret_access_key.is_empty())
.field("path_style", &self.path_style)
.field("compression", &self.compression)
.field("schedule", &self.schedule)
.field("retention_count", &self.retention_count)
.finish()
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct BackupConfigError {
message: String,
@@ -84,6 +106,9 @@ impl S3BackupConfig {
"Endpoint（S3 地址）",
enabled,
)?;
if enabled {
validate_s3_endpoint(&endpoint)?;
}
let bucket =
required_or_disabled_string(entries, "backup_s3_bucket", "Bucket（存储桶）", enabled)?;
let access_key_id = required_or_disabled_string(
@@ -99,6 +124,11 @@ impl S3BackupConfig {
enabled,
)?;
let prefix = normalize_s3_prefix(
&optional_string(entries, "backup_s3_prefix")?
.unwrap_or_else(|| "aether/backups/".to_string()),
)?;
Ok(Self {
enabled,
scope,
@@ -108,8 +138,7 @@ impl S3BackupConfig {
user_agent: optional_string(entries, "backup_s3_user_agent")?
.unwrap_or_else(|| "rclone/v1.68.0".to_string()),
bucket,
prefix: optional_string(entries, "backup_s3_prefix")?
.unwrap_or_else(|| "aether/backups/".to_string()),
prefix,
access_key_id,
secret_access_key,
path_style: optional_bool(entries, "backup_s3_path_style")?.unwrap_or(true),
@@ -121,6 +150,48 @@ impl S3BackupConfig {
}
}
fn normalize_s3_prefix(prefix: &str) -> Result<String, BackupConfigError> {
let prefix = prefix.trim().trim_matches('/');
if prefix.is_empty() {
return Ok(String::new());
}
if prefix
.split('/')
.any(|segment| segment.is_empty() || segment == "." || segment == "..")
|| prefix.contains('\\')
{
return Err(BackupConfigError::new(
"Prefix（备份前缀）不能包含空路径段、相对路径段或反斜杠",
));
}
Ok(format!("{prefix}/"))
}
fn validate_s3_endpoint(endpoint: &str) -> Result<(), BackupConfigError> {
let parsed = url::Url::parse(endpoint)
.map_err(|_| BackupConfigError::new("Endpoint（S3 地址）必须是有效的 HTTPS URL"))?;
if parsed.scheme() != "https"
|| parsed.host_str().is_none()
|| !parsed.username().is_empty()
|| parsed.password().is_some()
|| parsed.query().is_some()
|| parsed.fragment().is_some()
{
return Err(BackupConfigError::new(
"Endpoint（S3 地址）必须使用 HTTPS，且不能包含用户凭据、查询参数或片段",
));
}
Ok(())
}
fn sanitized_endpoint_origin(endpoint: &str) -> String {
url::Url::parse(endpoint)
.ok()
.map(|parsed| parsed.origin().ascii_serialization())
.unwrap_or_else(|| "<invalid>".to_string())
}
fn validate_range(label: &str, value: u32, min: u32, max: u32) -> Result<(), BackupConfigError> {
if (min..=max).contains(&value) {
Ok(())
@@ -374,6 +445,69 @@ mod tests {
assert!(err.to_string().contains("Endpoint"));
}
#[test]
fn rejects_insecure_or_credential_bearing_endpoints() {
for endpoint in [
"http://s3.example.com",
"https://user:[email protected]",
"https://s3.example.com?token=secret",
"https://s3.example.com/#fragment",
] {
let entries = serde_json::json!({
"backup_s3_enabled": true,
"backup_s3_endpoint": endpoint,
"backup_s3_bucket": "aether-backups",
"backup_s3_access_key_id": "access",
"backup_s3_secret_access_key": "secret"
});
let error = S3BackupConfig::from_json_map(entries.as_object().unwrap())
.expect_err("unsafe endpoint should fail closed");
assert!(error.to_string().contains("Endpoint"));
}
}
#[test]
fn debug_output_does_not_expose_s3_credentials() {
let entries = serde_json::json!({
"backup_s3_enabled": true,
"backup_s3_endpoint": "https://s3.example.com/path",
"backup_s3_bucket": "aether-backups",
"backup_s3_access_key_id": "access-key-value",
"backup_s3_secret_access_key": "secret-key-value"
});
let config = S3BackupConfig::from_json_map(entries.as_object().unwrap())
.expect("config should parse");
let debug = format!("{config:?}");
assert!(debug.contains("https://s3.example.com"));
assert!(!debug.contains("/path"));
assert!(!debug.contains("access-key-value"));
assert!(!debug.contains("secret-key-value"));
}
#[test]
fn canonicalizes_s3_backup_prefix_once() {
let entries = serde_json::json!({
"backup_s3_enabled": true,
"backup_s3_endpoint": "https://s3.example.com",
"backup_s3_bucket": "aether-backups",
"backup_s3_prefix": "/prod/backups//",
"backup_s3_access_key_id": "access",
"backup_s3_secret_access_key": "secret"
});
let config = S3BackupConfig::from_json_map(entries.as_object().unwrap())
.expect("prefix should be canonicalized");
assert_eq!(config.prefix, "prod/backups/");
for invalid_prefix in ["prod//backups", "prod/../backups", "prod\\backups"] {
let mut entries = entries.clone();
entries["backup_s3_prefix"] = serde_json::json!(invalid_prefix);
assert!(S3BackupConfig::from_json_map(entries.as_object().unwrap()).is_err());
}
}
#[test]
fn applies_default_values_from_system_config_contract() {
let entries = serde_json::json!({
apps/aether-gateway/src/backup/executor.rs
+1086 -29
View File
File diff suppressed because it is too large Load Diff
apps/aether-gateway/src/backup/mod.rs
+162
View File
@@ -6,5 +6,167 @@ pub(crate) mod store;
pub(crate) mod task;
pub(crate) mod worker;
pub use executor::{
restore_backup_json, BackupDecryptionKey, BackupRestoreError, BackupRestoreLimits,
RestoredBackupJson, DEFAULT_BACKUP_MAX_ENCRYPTED_BYTES, DEFAULT_BACKUP_MAX_JSON_BYTES,
};
use axum::body::Bytes;
use serde_json::Value;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum BackupRestoreScope {
Config,
Users,
Data,
}
impl BackupRestoreScope {
pub const fn as_str(self) -> &'static str {
match self {
Self::Config => "config",
Self::Users => "users",
Self::Data => "data",
}
}
}
#[derive(Debug, thiserror::Error)]
#[error("backup database apply failed: {0}")]
pub struct BackupApplyError(String);
pub async fn apply_restored_backup(
app: &crate::AppState,
restored: RestoredBackupJson,
scope: BackupRestoreScope,
operator_id: Option<&str>,
) -> Result<Result<Value, (http::StatusCode, Value)>, BackupApplyError> {
let (json_bytes, authority) = restored.into_authenticated_parts();
if authority.scope() != scope {
return Err(BackupApplyError(format!(
"authenticated {} backup cannot be applied to {} scope",
authority.scope().as_str(),
scope.as_str(),
)));
}
let request_body = Bytes::from(json_bytes);
let state = crate::admin_api::AdminAppState::new(app);
let result = crate::admin_api::execute_admin_system_import_exclusively(app, async {
match scope {
BackupRestoreScope::Config => {
state
.restore_admin_system_config_backup(&request_body, authority)
.await
}
BackupRestoreScope::Users => {
state
.restore_admin_system_users_backup(&request_body, operator_id, authority)
.await
}
BackupRestoreScope::Data => {
state
.restore_admin_system_data_backup(&request_body, operator_id, authority)
.await
}
}
})
.await
.map_err(|error| {
let message = match error {
crate::admin_api::AdminSystemImportLockError::Conflict => {
"another system import or restore is already running"
}
crate::admin_api::AdminSystemImportLockError::Unavailable => {
"system import coordination is unavailable"
}
crate::admin_api::AdminSystemImportLockError::Lost => {
"system import coordination lease was lost; restore was cancelled and may have partially applied changes"
}
};
BackupApplyError(message.to_string())
})?;
result.map_err(|error| BackupApplyError(error.into_message()))
}
pub(crate) const S3_BACKUP_ENABLED_KEY: &str = "backup_s3_enabled";
pub(crate) const S3_BACKUP_LAST_SLOT_KEY: &str = "backup_s3_last_slot";
#[cfg(test)]
mod tests {
use super::{
apply_restored_backup, BackupDecryptionKey, BackupRestoreLimits, BackupRestoreScope,
RestoredBackupJson,
};
use crate::backup::executor::encrypt_backup_bytes;
use aether_crypto::DEVELOPMENT_ENCRYPTION_KEY;
use serde_json::json;
fn authenticated_users_backup() -> RestoredBackupJson {
let object_key = "prod/aether-users-backup-20260830-120000.json.zst.aes256gcm";
let compressed = zstd::stream::encode_all(
serde_json::to_vec(&json!({
"version": "1.5",
"exported_at": "2026-08-30T12:00:00Z",
"users": [],
"standalone_keys": [],
}))
.expect("test backup should serialize")
.as_slice(),
0,
)
.expect("test backup should compress");
let (envelope, _) =
encrypt_backup_bytes(DEVELOPMENT_ENCRYPTION_KEY, object_key, &compressed)
.expect("test backup should encrypt");
super::restore_backup_json(
object_key,
&envelope,
&[BackupDecryptionKey::current(DEVELOPMENT_ENCRYPTION_KEY)
.expect("test restore key should build")],
BackupRestoreLimits::default(),
)
.expect("test backup should authenticate")
}
#[tokio::test]
async fn authenticated_backup_cannot_be_applied_to_a_different_scope() {
let restored = authenticated_users_backup();
let error = apply_restored_backup(
&crate::AppState::new().expect("test state should build"),
restored,
BackupRestoreScope::Config,
None,
)
.await
.expect_err("scope mismatch must fail before database access");
assert_eq!(
error.to_string(),
"backup database apply failed: authenticated users backup cannot be applied to config scope"
);
}
#[tokio::test]
async fn authenticated_backup_apply_uses_the_shared_system_import_lock() {
let app = crate::AppState::new().expect("test state should build");
let lock = crate::admin_api::try_acquire_admin_system_import_lease(&app)
.await
.expect("test should acquire the shared import lease");
let error = apply_restored_backup(
&app,
authenticated_users_backup(),
BackupRestoreScope::Users,
None,
)
.await
.expect_err("restore must not interleave with another system import");
assert_eq!(
error.to_string(),
"backup database apply failed: another system import or restore is already running"
);
crate::admin_api::release_admin_system_import_lease(&app, &lock).await;
}
}
apps/aether-gateway/src/backup/scopes.rs
+153 -15
View File
@@ -1,5 +1,8 @@
use std::fmt;
const ENCRYPTED_BACKUP_FILE_SUFFIX: &str = ".json.zst.aes256gcm";
const LEGACY_PLAINTEXT_BACKUP_FILE_SUFFIX: &str = ".json.zst";
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum BackupScope {
Config,
@@ -52,10 +55,81 @@ impl BackupScope {
}
}
pub(crate) fn from_encrypted_object_key(object_key: &str) -> Option<Self> {
if object_key.is_empty()
|| object_key.starts_with('/')
|| object_key.contains('\0')
|| object_key.contains('\\')
{
return None;
}
let mut segments = object_key.split('/').peekable();
let mut file_name = None;
while let Some(segment) = segments.next() {
if segment.is_empty()
|| segment == "."
|| segment == ".."
|| segment.chars().any(char::is_control)
{
return None;
}
if segments.peek().is_none() {
file_name = Some(segment);
}
}
let file_name = file_name?;
[Self::Config, Self::Users, Self::Data]
.into_iter()
.find(|scope| {
file_name
.strip_prefix(&format!("{}-", scope.file_stem()))
.and_then(|rest| rest.strip_suffix(ENCRYPTED_BACKUP_FILE_SUFFIX))
.is_some_and(is_aether_backup_object_id)
})
}
#[cfg(test)]
pub(crate) fn matching_backup_keys(
self,
prefix: &str,
keys: impl IntoIterator<Item = String>,
) -> Vec<String> {
self.matching_backup_keys_with_suffixes(
prefix,
keys,
&[
ENCRYPTED_BACKUP_FILE_SUFFIX,
LEGACY_PLAINTEXT_BACKUP_FILE_SUFFIX,
],
)
}
pub(crate) fn matching_encrypted_backup_keys(
self,
prefix: &str,
keys: impl IntoIterator<Item = String>,
) -> Vec<String> {
self.matching_backup_keys_with_suffixes(prefix, keys, &[ENCRYPTED_BACKUP_FILE_SUFFIX])
}
pub(crate) fn matching_legacy_plaintext_backup_keys(
self,
prefix: &str,
keys: impl IntoIterator<Item = String>,
) -> Vec<String> {
self.matching_backup_keys_with_suffixes(
prefix,
keys,
&[LEGACY_PLAINTEXT_BACKUP_FILE_SUFFIX],
)
}
fn matching_backup_keys_with_suffixes(
self,
prefix: &str,
keys: impl IntoIterator<Item = String>,
file_suffixes: &[&str],
) -> Vec<String> {
let normalized_prefix = normalized_prefix(prefix);
let expected_prefix = if normalized_prefix.is_empty() {
@@ -64,7 +138,6 @@ impl BackupScope {
format!("{normalized_prefix}/")
};
let file_prefix = format!("{}-", self.file_stem());
let file_suffix = ".json.zst";
keys.into_iter()
.filter(|key| {
@@ -74,20 +147,24 @@ impl BackupScope {
if file_name.contains('/') {
return false;
}
let Some(timestamp) = file_name
.strip_prefix(&file_prefix)
.and_then(|rest| rest.strip_suffix(file_suffix))
else {
let Some(timestamp) = file_name.strip_prefix(&file_prefix).and_then(|rest| {
file_suffixes
.iter()
.find_map(|suffix| rest.strip_suffix(suffix))
}) else {
return false;
};
is_aether_backup_timestamp(timestamp)
is_aether_backup_object_id(timestamp)
})
.collect()
}
fn file_name(self, timestamp: &str) -> String {
format!("{}-{timestamp}.json.zst", self.file_stem())
format!(
"{}-{timestamp}{ENCRYPTED_BACKUP_FILE_SUFFIX}",
self.file_stem()
)
}
}
@@ -110,6 +187,25 @@ fn is_aether_backup_timestamp(timestamp: &str) -> bool {
&& bytes[9..].iter().all(|byte| byte.is_ascii_digit())
}
fn is_aether_backup_object_id(value: &str) -> bool {
if is_aether_backup_timestamp(value) {
return true;
}
let Some((timestamp, collision_digest)) = value.split_once('-').and_then(|(date, rest)| {
let (time, digest) = rest.split_once('-')?;
Some((format!("{date}-{time}"), digest))
}) else {
return false;
};
is_aether_backup_timestamp(&timestamp)
&& collision_digest.len() == 64
&& collision_digest
.bytes()
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
}
#[cfg(test)]
mod tests {
use super::BackupScope;
@@ -130,15 +226,15 @@ mod tests {
assert_eq!(
BackupScope::Config.object_key("prod/", "20260524-031500"),
"prod/aether-config-backup-20260524-031500.json.zst"
"prod/aether-config-backup-20260524-031500.json.zst.aes256gcm"
);
assert_eq!(
BackupScope::Users.object_key("prod/", "20260524-031500"),
"prod/aether-users-backup-20260524-031500.json.zst"
"prod/aether-users-backup-20260524-031500.json.zst.aes256gcm"
);
assert_eq!(
BackupScope::Data.object_key("prod/", "20260524-031500"),
"prod/aether-data-backup-20260524-031500.json.zst"
"prod/aether-data-backup-20260524-031500.json.zst.aes256gcm"
);
}
@@ -146,7 +242,7 @@ mod tests {
fn retention_filter_only_matches_same_scope() {
let keys = vec![
"prod/aether-config-backup-20260524-010000.json.zst".to_string(),
"prod/aether-users-backup-20260524-010000.json.zst".to_string(),
"prod/aether-users-backup-20260524-010000.json.zst.aes256gcm".to_string(),
"prod/aether-data-backup-20260524-010000.json.zst".to_string(),
"prod/random.json.zst".to_string(),
];
@@ -155,14 +251,18 @@ mod tests {
assert_eq!(
matched,
vec!["prod/aether-users-backup-20260524-010000.json.zst"]
vec!["prod/aether-users-backup-20260524-010000.json.zst.aes256gcm"]
);
}
#[test]
fn retention_filter_requires_aether_timestamp_format() {
let collision_digest = "a".repeat(64);
let keys = vec![
"prod/aether-users-backup-20260524-010000.json.zst".to_string(),
format!(
"prod/aether-users-backup-20260524-010000-{collision_digest}.json.zst.aes256gcm"
),
"prod/aether-users-backup-foo.json.zst".to_string(),
"prod/aether-users-backup-2026052-010000.json.zst".to_string(),
"prod/aether-users-backup-202605240-010000.json.zst".to_string(),
@@ -171,13 +271,19 @@ mod tests {
"prod/aether-users-backup-20260524010000.json.zst".to_string(),
"prod/aether-users-backup-2026052a-010000.json.zst".to_string(),
"prod/aether-users-backup-20260524-01000x.json.zst".to_string(),
"prod/aether-users-backup-20260524-010000-short.json.zst.aes256gcm".to_string(),
];
let matched = BackupScope::Users.matching_backup_keys("prod/", keys);
assert_eq!(
matched,
vec!["prod/aether-users-backup-20260524-010000.json.zst"]
vec![
"prod/aether-users-backup-20260524-010000.json.zst".to_string(),
format!(
"prod/aether-users-backup-20260524-010000-{collision_digest}.json.zst.aes256gcm"
),
]
);
}
@@ -185,11 +291,11 @@ mod tests {
fn backup_key_prefix_boundaries_are_exact() {
assert_eq!(
BackupScope::Config.object_key("", "20260524-031500"),
"aether-config-backup-20260524-031500.json.zst"
"aether-config-backup-20260524-031500.json.zst.aes256gcm"
);
assert_eq!(
BackupScope::Config.object_key("prod", "20260524-031500"),
"prod/aether-config-backup-20260524-031500.json.zst"
"prod/aether-config-backup-20260524-031500.json.zst.aes256gcm"
);
let keys = vec![
@@ -208,4 +314,36 @@ mod tests {
vec!["prod/aether-config-backup-20260524-010000.json.zst"]
);
}
#[test]
fn encrypted_object_key_parser_binds_scope_and_rejects_path_traversal() {
let collision_digest = "a".repeat(64);
assert_eq!(
BackupScope::from_encrypted_object_key(
"prod/aether-config-backup-20260524-010000.json.zst.aes256gcm"
),
Some(BackupScope::Config)
);
assert_eq!(
BackupScope::from_encrypted_object_key(&format!(
"prod/aether-users-backup-20260524-010000-{collision_digest}.json.zst.aes256gcm"
)),
Some(BackupScope::Users)
);
for key in [
"../aether-data-backup-20260524-010000.json.zst.aes256gcm",
"/aether-data-backup-20260524-010000.json.zst.aes256gcm",
"prod//aether-data-backup-20260524-010000.json.zst.aes256gcm",
"prod/./aether-data-backup-20260524-010000.json.zst.aes256gcm",
"prod\\aether-data-backup-20260524-010000.json.zst.aes256gcm",
"prod/aether-data-backup-invalid.json.zst.aes256gcm",
"prod/unrelated-20260524-010000.json.zst.aes256gcm",
] {
assert_eq!(
BackupScope::from_encrypted_object_key(key),
None,
"unsafe or unrelated key: {key}"
);
}
}
}
apps/aether-gateway/src/backup/store.rs
+222 -30
View File
@@ -2,23 +2,45 @@ use std::collections::BTreeMap;
use std::fmt;
use std::sync::Arc;
use bytes::Bytes;
use bytes::{Bytes, BytesMut};
use futures_util::TryStreamExt;
use object_store::aws::AmazonS3Builder;
use object_store::path::Path;
use object_store::{ClientOptions, ObjectStore};
use object_store::{ClientOptions, ObjectStore, ObjectStoreExt, PutMode, PutOptions};
use reqwest::header::HeaderValue;
use tokio::sync::RwLock;
use super::config::S3BackupConfig;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum BackupObjectCreateResult {
Created,
AlreadyExists,
}
#[async_trait::async_trait]
pub(crate) trait BackupObjectStore: Send + Sync {
async fn put_object(&self, key: &str, bytes: Bytes) -> Result<(), BackupStoreError>;
async fn list_keys(&self, prefix: &str) -> Result<Vec<String>, BackupStoreError>;
async fn put_object_if_absent(
&self,
key: &str,
bytes: Bytes,
) -> Result<BackupObjectCreateResult, BackupStoreError>;
async fn get_object_limited(
&self,
key: &str,
max_bytes: usize,
) -> Result<Bytes, BackupStoreError>;
async fn delete_object(&self, key: &str) -> Result<(), BackupStoreError>;
async fn list_keys_limited(
&self,
prefix: &str,
max_objects: usize,
) -> Result<Vec<String>, BackupStoreError>;
}
#[derive(Debug, Clone, PartialEq, Eq)]
@@ -60,21 +82,72 @@ impl BackupObjectStore for FakeBackupObjectStore {
Ok(())
}
async fn list_keys(&self, prefix: &str) -> Result<Vec<String>, BackupStoreError> {
async fn put_object_if_absent(
&self,
key: &str,
bytes: Bytes,
) -> Result<BackupObjectCreateResult, BackupStoreError> {
let mut objects = self.objects.write().await;
if objects.contains_key(key) {
Ok(BackupObjectCreateResult::AlreadyExists)
} else {
objects.insert(key.to_string(), bytes);
Ok(BackupObjectCreateResult::Created)
}
}
async fn get_object_limited(
&self,
key: &str,
max_bytes: usize,
) -> Result<Bytes, BackupStoreError> {
let bytes = self
.objects
.read()
.await
.get(key)
.cloned()
.ok_or_else(|| BackupStoreError::new(format!("backup object `{key}` not found")))?;
if bytes.len() > max_bytes {
return Err(BackupStoreError::new(format!(
"backup object `{key}` exceeds the configured {max_bytes} byte read limit"
)));
}
Ok(bytes)
}
async fn delete_object(&self, key: &str) -> Result<(), BackupStoreError> {
self.objects.write().await.remove(key);
Ok(())
}
async fn list_keys_limited(
&self,
prefix: &str,
max_objects: usize,
) -> Result<Vec<String>, BackupStoreError> {
let prefix = directory_list_prefix(prefix);
Ok(self
let keys: Vec<_> = self
.objects
.read()
.await
.keys()
.filter(|key| key.starts_with(&prefix))
.cloned()
.collect())
.collect();
if keys.len() > max_objects {
return Err(BackupStoreError::new(format!(
"backup object listing exceeds the configured {max_objects} object limit"
)));
}
Ok(keys)
}
}
async fn delete_object(&self, key: &str) -> Result<(), BackupStoreError> {
self.objects.write().await.remove(key);
Ok(())
#[cfg(test)]
impl FakeBackupObjectStore {
pub(crate) async fn object_bytes(&self, key: &str) -> Option<Bytes> {
self.objects.read().await.get(key).cloned()
}
}
@@ -125,17 +198,68 @@ impl BackupObjectStore for ObjectStoreS3BackupStore {
.map_err(|error| BackupStoreError::object_store("put", key, error))
}
async fn list_keys(&self, prefix: &str) -> Result<Vec<String>, BackupStoreError> {
let prefix_path = list_prefix_path(prefix);
let mut keys = self
async fn put_object_if_absent(
&self,
key: &str,
bytes: Bytes,
) -> Result<BackupObjectCreateResult, BackupStoreError> {
let options = PutOptions {
mode: PutMode::Create,
..PutOptions::default()
};
match self
.store
.list(prefix_path.as_ref())
.map_ok(|meta| meta.location.to_string())
.try_collect::<Vec<_>>()
.put_opts(&Path::from(key), bytes.into(), options)
.await
.map_err(|error| BackupStoreError::object_store("list", prefix, error))?;
keys.sort();
Ok(keys)
{
Ok(_) => Ok(BackupObjectCreateResult::Created),
Err(object_store::Error::AlreadyExists { .. }) => {
Ok(BackupObjectCreateResult::AlreadyExists)
}
Err(error) => Err(BackupStoreError::object_store(
"conditional put",
key,
error,
)),
}
}
async fn get_object_limited(
&self,
key: &str,
max_bytes: usize,
) -> Result<Bytes, BackupStoreError> {
let result = self
.store
.get(&Path::from(key))
.await
.map_err(|error| BackupStoreError::object_store("get", key, error))?;
if result.meta.size > u64::try_from(max_bytes).unwrap_or(u64::MAX) {
return Err(BackupStoreError::new(format!(
"backup object `{key}` exceeds the configured {max_bytes} byte read limit"
)));
}
let object_size = result.meta.size;
let mut stream = result.into_stream();
let mut bytes = BytesMut::with_capacity(
usize::try_from(object_size)
.unwrap_or(max_bytes)
.min(max_bytes)
.min(8 * 1024 * 1024),
);
while let Some(chunk) = stream
.try_next()
.await
.map_err(|error| BackupStoreError::object_store("read", key, error))?
{
if bytes.len().saturating_add(chunk.len()) > max_bytes {
return Err(BackupStoreError::new(format!(
"backup object `{key}` exceeds the configured {max_bytes} byte read limit"
)));
}
bytes.extend_from_slice(&chunk);
}
Ok(bytes.freeze())
}
async fn delete_object(&self, key: &str) -> Result<(), BackupStoreError> {
@@ -144,6 +268,30 @@ impl BackupObjectStore for ObjectStoreS3BackupStore {
.await
.map_err(|error| BackupStoreError::object_store("delete", key, error))
}
async fn list_keys_limited(
&self,
prefix: &str,
max_objects: usize,
) -> Result<Vec<String>, BackupStoreError> {
let prefix_path = list_prefix_path(prefix);
let mut objects = self.store.list(prefix_path.as_ref());
let mut keys = Vec::new();
while let Some(meta) = objects
.try_next()
.await
.map_err(|error| BackupStoreError::object_store("list", prefix, error))?
{
if keys.len() >= max_objects {
return Err(BackupStoreError::new(format!(
"backup object listing exceeds the configured {max_objects} object limit"
)));
}
keys.push(meta.location.to_string());
}
keys.sort();
Ok(keys)
}
}
fn directory_list_prefix(prefix: &str) -> String {
@@ -166,10 +314,12 @@ fn list_prefix_path(prefix: &str) -> Option<Path> {
#[cfg(test)]
mod tests {
use super::{list_prefix_path, BackupObjectStore, FakeBackupObjectStore};
use super::{
list_prefix_path, BackupObjectCreateResult, BackupObjectStore, FakeBackupObjectStore,
};
#[tokio::test]
async fn fake_backup_object_store_puts_lists_and_deletes() {
async fn fake_backup_object_store_puts_and_lists() {
let store = FakeBackupObjectStore::default();
store
.put_object(
@@ -186,17 +336,59 @@ mod tests {
.await
.unwrap();
let keys = store.list_keys("prod/").await.unwrap();
assert_eq!(keys.len(), 2);
store
.delete_object("prod/aether-data-backup-20260524-010000.json.zst")
.await
.unwrap();
let keys = store.list_keys("prod/").await.unwrap();
let keys = store.list_keys_limited("prod/", 2).await.unwrap();
assert_eq!(
keys,
vec!["prod/aether-data-backup-20260524-020000.json.zst"]
vec![
"prod/aether-data-backup-20260524-010000.json.zst",
"prod/aether-data-backup-20260524-020000.json.zst",
]
);
}
#[tokio::test]
async fn fake_backup_object_store_enforces_read_and_listing_limits() {
let store = FakeBackupObjectStore::default();
store
.put_object("prod/one", bytes::Bytes::from_static(b"1234"))
.await
.unwrap();
store
.put_object("prod/two", bytes::Bytes::from_static(b"5678"))
.await
.unwrap();
assert!(store.get_object_limited("prod/one", 3).await.is_err());
assert_eq!(
store.get_object_limited("prod/one", 4).await.unwrap(),
bytes::Bytes::from_static(b"1234")
);
assert!(store.list_keys_limited("prod/", 1).await.is_err());
assert_eq!(store.list_keys_limited("prod/", 2).await.unwrap().len(), 2);
}
#[tokio::test]
async fn fake_backup_object_store_conditional_put_never_overwrites() {
let store = FakeBackupObjectStore::default();
let key = "prod/aether-data-backup-20260524-010000.json.zst.aes256gcm";
assert_eq!(
store
.put_object_if_absent(key, bytes::Bytes::from_static(b"first"))
.await
.unwrap(),
BackupObjectCreateResult::Created
);
assert_eq!(
store
.put_object_if_absent(key, bytes::Bytes::from_static(b"second"))
.await
.unwrap(),
BackupObjectCreateResult::AlreadyExists
);
assert_eq!(
store.object_bytes(key).await.as_deref(),
Some(b"first".as_slice())
);
}
@@ -218,7 +410,7 @@ mod tests {
.await
.unwrap();
let keys = store.list_keys("prod").await.unwrap();
let keys = store.list_keys_limited("prod", 10).await.unwrap();
assert_eq!(
keys,
apps/aether-gateway/src/backup/task.rs
+410 -55
View File
@@ -1,4 +1,5 @@
use std::fmt;
use std::future::Future;
use std::time::Duration;
use aether_admin::system::admin_system_config_default_value;
@@ -12,14 +13,15 @@ use chrono::Utc;
use futures_util::FutureExt;
use serde::Serialize;
use serde_json::{json, Map, Value};
use tokio::task::{JoinError, JoinHandle};
use tracing::warn;
use super::config::S3BackupConfig;
use super::executor::{run_backup_with_store, BackupRunResult};
use super::scopes::BackupScope;
use super::store::ObjectStoreS3BackupStore;
use crate::admin_api::AdminAppState;
use crate::handlers::shared::decrypt_catalog_secret_with_fallbacks;
use crate::admin_api::{AdminAppState, SystemExportMode};
use crate::handlers::shared::decrypt_or_migrate_system_config_secret;
use crate::task_runtime::{
append_event_with_logging, build_task_run_id, now_unix_secs, spawn_fire_and_forget,
task_definition, update_run_status, upsert_run_with_logging, TASK_KEY_SYSTEM_S3_BACKUP,
@@ -48,6 +50,9 @@ const S3_BACKUP_CONFIG_KEYS: &[&str] = &[
];
const S3_BACKUP_QUEUED_MESSAGE: &str = "S3 备份任务已提交";
const S3_BACKUP_INTERNAL_ERROR_DETAIL: &str = "S3 备份服务暂时不可用";
const S3_BACKUP_TASK_FAILURE_CODE: &str = "s3_backup_failed";
const S3_BACKUP_SLOT_RECORD_FAILURE_CODE: &str = "s3_backup_slot_record_failed";
const S3_BACKUP_TASK_LOCK_KEY: &str = "task_runtime:lock:system.s3.backup";
const S3_BACKUP_TASK_LOCK_TTL: Duration = Duration::from_secs(60 * 60 * 6);
const S3_BACKUP_TASK_HEARTBEAT_INTERVAL: Duration = Duration::from_secs(60 * 5);
@@ -67,6 +72,16 @@ pub(crate) struct S3BackupTaskError {
detail: String,
}
enum BackupLockRenewalFailure<E> {
Lost,
Backend(E),
}
enum BackupLockRaceOutcome<T> {
BackupCompleted(T),
LeaseLost(Result<(), JoinError>),
}
impl S3BackupTaskError {
fn bad_request(detail: impl Into<String>) -> Self {
Self {
@@ -114,8 +129,12 @@ impl fmt::Display for S3BackupTaskError {
impl std::error::Error for S3BackupTaskError {}
impl From<GatewayError> for S3BackupTaskError {
fn from(error: GatewayError) -> Self {
Self::internal(format!("{error:?}"))
fn from(_error: GatewayError) -> Self {
warn!(
error_category = "dependency_failed",
"S3 backup dependency failed"
);
Self::internal(S3_BACKUP_INTERNAL_ERROR_DETAIL)
}
}
@@ -220,8 +239,6 @@ fn s3_backup_task_payload_json(
) -> Value {
let mut payload = json!({
"scope": config.scope.as_config_value(),
"bucket": config.bucket.clone(),
"prefix": config.prefix.clone(),
"compression": config.compression.clone(),
"trigger": trigger,
});
@@ -259,7 +276,7 @@ fn spawn_s3_backup_worker(
Some(100),
Some("S3 备份任务异常退出".to_string()),
None,
Some("S3 backup task panicked".to_string()),
Some("background_task_panicked".to_string()),
None,
Some(now_unix_secs()),
)
@@ -294,16 +311,67 @@ async fn run_s3_backup_worker_inner(
.await;
append_event_with_logging(&app, &run_id, "running", "S3 backup task started", None).await;
let heartbeat = spawn_s3_backup_task_heartbeat(app.clone(), run_id.clone(), lock);
let result = run_s3_backup_once(&app, &config).await;
heartbeat.abort();
let _ = heartbeat.await;
let heartbeat = spawn_s3_backup_task_heartbeat(app.clone(), run_id.clone(), lock.clone());
let result = match race_backup_with_lock_heartbeat(run_s3_backup_once(&app, &config), heartbeat)
.await
{
BackupLockRaceOutcome::BackupCompleted(result) => {
match require_successful_backup_lock_renewal(
app.runtime_state
.lock_renew(&lock, S3_BACKUP_TASK_LOCK_TTL)
.await,
) {
Ok(()) => result,
Err(BackupLockRenewalFailure::Lost) => {
warn!(
run_id = %run_id,
lock_key = %lock.key,
"S3 backup task lost its distributed lock before publishing completion"
);
Err(S3BackupTaskError::service_unavailable(
"S3 备份任务锁已失效，任务完成状态未发布",
))
}
Err(BackupLockRenewalFailure::Backend(error)) => {
warn!(
run_id = %run_id,
lock_key = %lock.key,
error = %error,
"S3 backup task could not verify its distributed lock before publishing completion"
);
Err(S3BackupTaskError::service_unavailable(
"无法确认 S3 备份任务锁所有权，任务完成状态未发布",
))
}
}
}
BackupLockRaceOutcome::LeaseLost(heartbeat_result) => {
match heartbeat_result {
Ok(()) => warn!(
run_id = %run_id,
"S3 backup task stopped after losing its distributed lock"
),
Err(error) => warn!(
run_id = %run_id,
error = %error,
"S3 backup lock heartbeat task failed"
),
}
Err(S3BackupTaskError::service_unavailable(
"S3 备份任务锁已失效，任务已停止",
))
}
};
match result {
Ok(result) => {
if let Some(slot) = scheduled_backup_slot_to_record(scheduled_slot.as_deref(), true) {
if let Err(error) = record_scheduled_backup_slot(&app, &slot).await {
warn!(error = ?error, run_id = %run_id, "S3 backup slot record failed");
if record_scheduled_backup_slot(&app, &slot).await.is_err() {
warn!(
error_category = "slot_record_failed",
run_id = %run_id,
"S3 backup slot record failed"
);
let _ = update_run_status(
&app,
&run_id,
@@ -311,7 +379,7 @@ async fn run_s3_backup_worker_inner(
Some(100),
Some("S3 备份任务完成，但记录调度时间失败".to_string()),
None,
Some(format!("S3 backup slot record failed: {error:?}")),
Some(S3_BACKUP_SLOT_RECORD_FAILURE_CODE.to_string()),
None,
Some(now_unix_secs()),
)
@@ -321,7 +389,7 @@ async fn run_s3_backup_worker_inner(
&run_id,
"failed",
"S3 backup slot record failed",
Some(json!({ "error": format!("{error:?}") })),
Some(json!({ "error_code": S3_BACKUP_SLOT_RECORD_FAILURE_CODE })),
)
.await;
return;
@@ -349,8 +417,12 @@ async fn run_s3_backup_worker_inner(
)
.await;
}
Err(error) => {
warn!(error = %error, run_id = %run_id, "S3 backup task failed");
Err(_) => {
warn!(
error_category = "backup_execution_failed",
run_id = %run_id,
"S3 backup task failed"
);
let _ = update_run_status(
&app,
&run_id,
@@ -358,7 +430,7 @@ async fn run_s3_backup_worker_inner(
Some(100),
Some("S3 备份任务失败".to_string()),
None,
Some(error.to_string()),
Some(S3_BACKUP_TASK_FAILURE_CODE.to_string()),
None,
Some(now_unix_secs()),
)
@@ -368,13 +440,44 @@ async fn run_s3_backup_worker_inner(
&run_id,
"failed",
"S3 backup task failed",
Some(json!({ "error": error.to_string() })),
Some(json!({ "error_code": S3_BACKUP_TASK_FAILURE_CODE })),
)
.await;
}
}
}
async fn race_backup_with_lock_heartbeat<F, T>(
backup: F,
mut heartbeat: JoinHandle<()>,
) -> BackupLockRaceOutcome<T>
where
F: Future<Output = T>,
{
tokio::pin!(backup);
tokio::select! {
biased;
heartbeat_result = &mut heartbeat => {
BackupLockRaceOutcome::LeaseLost(heartbeat_result)
}
result = &mut backup => {
heartbeat.abort();
let _ = heartbeat.await;
BackupLockRaceOutcome::BackupCompleted(result)
}
}
}
fn require_successful_backup_lock_renewal<E>(
result: Result<bool, E>,
) -> Result<(), BackupLockRenewalFailure<E>> {
match result {
Ok(true) => Ok(()),
Ok(false) => Err(BackupLockRenewalFailure::Lost),
Err(error) => Err(BackupLockRenewalFailure::Backend(error)),
}
}
fn spawn_s3_backup_task_heartbeat(
app: AppState,
run_id: String,
@@ -386,10 +489,30 @@ fn spawn_s3_backup_task_heartbeat(
interval.tick().await;
loop {
interval.tick().await;
let _ = app
.runtime_state
.lock_renew(&lock, S3_BACKUP_TASK_LOCK_TTL)
.await;
match require_successful_backup_lock_renewal(
app.runtime_state
.lock_renew(&lock, S3_BACKUP_TASK_LOCK_TTL)
.await,
) {
Ok(()) => {}
Err(BackupLockRenewalFailure::Lost) => {
warn!(
run_id = %run_id,
lock_key = %lock.key,
"S3 backup task distributed lock is no longer owned"
);
return;
}
Err(BackupLockRenewalFailure::Backend(error)) => {
warn!(
run_id = %run_id,
lock_key = %lock.key,
error = %error,
"S3 backup task distributed lock renewal failed"
);
return;
}
}
let _ = update_run_status(
&app,
&run_id,
@@ -458,9 +581,15 @@ async fn acquire_s3_backup_task_lock(
Ok(None) => Err(S3BackupTaskError::conflict(
"已有 S3 备份任务正在执行，请等待当前任务完成后再试",
)),
Err(error) => Err(S3BackupTaskError::service_unavailable(format!(
"无法获取 S3 备份任务锁：{error}"
))),
Err(_) => {
warn!(
error_category = "lock_acquisition_failed",
"S3 backup task lock acquisition failed"
);
Err(S3BackupTaskError::service_unavailable(
"无法获取 S3 备份任务锁，请稍后重试",
))
}
}
}
@@ -509,25 +638,77 @@ async fn run_s3_backup_once(
app: &AppState,
config: &S3BackupConfig,
) -> Result<BackupRunResult, S3BackupTaskError> {
let admin_state = AdminAppState::new(app);
let payload = match config.scope {
BackupScope::Config => {
admin_state
.build_admin_system_config_export_payload()
.await?
}
BackupScope::Users => {
admin_state
.build_admin_system_users_export_payload()
.await?
}
BackupScope::Data => admin_state.build_admin_system_data_export_payload().await?,
let Some(encryption_secret) = effective_backup_encryption_secret(app) else {
return Err(S3BackupTaskError::service_unavailable(
"S3 备份需要 AETHER_BACKUP_ENCRYPTION_KEY 或可用的数据加密密钥",
));
};
let store = ObjectStoreS3BackupStore::from_config(config)
.map_err(|error| S3BackupTaskError::internal(error.to_string()))?;
run_backup_with_store(config, &store, payload, Utc::now())
let payload = build_s3_backup_payload_exclusively(app, config.scope).await?;
let store = ObjectStoreS3BackupStore::from_config(config).map_err(|_| {
warn!(
error_category = "object_store_initialization_failed",
"S3 backup object store initialization failed"
);
S3BackupTaskError::internal(S3_BACKUP_INTERNAL_ERROR_DETAIL)
})?;
run_backup_with_store(config, &store, payload, Utc::now(), &encryption_secret)
.await
.map_err(|error| S3BackupTaskError::internal(error.to_string()))
.map_err(|_| {
warn!(
error_category = "backup_execution_failed",
"S3 backup execution failed"
);
S3BackupTaskError::internal(S3_BACKUP_INTERNAL_ERROR_DETAIL)
})
}
async fn build_s3_backup_payload_exclusively(
app: &AppState,
scope: BackupScope,
) -> Result<Value, S3BackupTaskError> {
let admin_state = AdminAppState::new(app);
crate::admin_api::execute_admin_system_import_exclusively(app, async {
match scope {
BackupScope::Config => {
admin_state
.build_admin_system_config_export_payload(SystemExportMode::RecoveryBackup)
.await
}
BackupScope::Users => {
admin_state
.build_admin_system_users_export_payload(SystemExportMode::RecoveryBackup)
.await
}
BackupScope::Data => {
admin_state
.build_admin_system_data_export_payload(SystemExportMode::RecoveryBackup)
.await
}
}
})
.await
.map_err(|error| {
warn!(
error_category = "system_import_coordination_failed",
lock_error = ?error,
"S3 backup snapshot could not acquire or retain the system import lock"
);
S3BackupTaskError::service_unavailable(S3_BACKUP_INTERNAL_ERROR_DETAIL)
})?
.map_err(S3BackupTaskError::from)
}
fn effective_backup_encryption_secret(app: &AppState) -> Option<String> {
std::env::var("AETHER_BACKUP_ENCRYPTION_KEY")
.ok()
.map(|value| value.trim().to_string())
.filter(|value| !value.is_empty())
.or_else(|| {
app.encryption_key()
.map(str::trim)
.filter(|value| !value.is_empty())
.map(ToOwned::to_owned)
})
}
async fn load_s3_backup_config_for_run(
@@ -551,7 +732,7 @@ pub(crate) async fn load_s3_backup_config_values(
.or_else(|| admin_system_config_default_value(key));
if let Some(value) = value {
let value = if *key == "backup_s3_secret_access_key" {
decrypt_s3_secret_access_key(app, value)?
decrypt_s3_secret_access_key(app, value).await?
} else {
value
};
@@ -561,39 +742,56 @@ pub(crate) async fn load_s3_backup_config_values(
Ok(values)
}
fn decrypt_s3_secret_access_key(app: &AppState, value: Value) -> Result<Value, S3BackupTaskError> {
let Some(ciphertext) = value
async fn decrypt_s3_secret_access_key(
app: &AppState,
value: Value,
) -> Result<Value, S3BackupTaskError> {
let Some(stored_value) = value
.as_str()
.map(str::trim)
.filter(|value| !value.is_empty())
else {
return Ok(value);
};
let Some(plaintext) = decrypt_catalog_secret_with_fallbacks(app.encryption_key(), ciphertext)
else {
return Err(S3BackupTaskError::bad_request(
let plaintext = decrypt_or_migrate_system_config_secret(
app,
"backup_s3_secret_access_key",
stored_value.to_string(),
)
.await
.map_err(|_| {
S3BackupTaskError::bad_request(
"S3 备份配置无效：Secret Access Key（访问密钥）无法解密，请重新填写",
));
};
)
})?;
Ok(Value::String(plaintext))
}
fn backup_run_result_json(result: &BackupRunResult) -> Value {
json!({
"scope": result.scope.as_config_value(),
"bucket": result.bucket,
"object_key": result.object_key,
"bytes": result.bytes,
"sha256": result.sha256,
"export_version": result.export_version,
"exported_at": result.exported_at,
"compression": result.compression,
"deleted_old_objects": result.deleted_old_objects,
"encryption": result.encryption,
"legacy_encrypted_copies_created": result.legacy_encrypted_copies_created,
"legacy_encrypted_copies_verified": result.legacy_encrypted_copies_verified,
"legacy_plaintext_objects_deleted": result.legacy_plaintext_objects_deleted,
"legacy_plaintext_objects_retained": result.legacy_plaintext_objects_retained,
"retention_cleanup_candidates": result.retention_cleanup_candidates,
"automatic_deletions": result.legacy_plaintext_objects_deleted,
"object_cleanup_mode": "legacy_plaintext_deleted_after_verified_encryption",
"versioned_storage_cleanup_required": result.versioned_storage_cleanup_required,
"versioned_storage_cleanup_notice": "legacy_plaintext_versions_require_external_cleanup",
})
}
#[cfg(test)]
mod tests {
use std::convert::Infallible;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc;
use aether_crypto::{encrypt_python_fernet_plaintext, DEVELOPMENT_ENCRYPTION_KEY};
@@ -603,9 +801,78 @@ mod tests {
};
use crate::data::GatewayDataState;
use crate::handlers::shared::decrypt_system_config_secret;
use crate::state::AppState;
use crate::task_runtime::{now_unix_secs, TASK_KEY_SYSTEM_S3_BACKUP};
#[test]
fn backup_lock_renewal_requires_ownership_and_preserves_backend_errors() {
assert!(matches!(
super::require_successful_backup_lock_renewal::<Infallible>(Ok(true)),
Ok(())
));
assert!(matches!(
super::require_successful_backup_lock_renewal::<Infallible>(Ok(false)),
Err(super::BackupLockRenewalFailure::Lost)
));
assert!(matches!(
super::require_successful_backup_lock_renewal(Err("redis unavailable")),
Err(super::BackupLockRenewalFailure::Backend(
"redis unavailable"
))
));
}
#[tokio::test]
async fn lost_backup_lock_stops_race_without_publishing_backup_result() {
let destructive_stage_reached = Arc::new(AtomicBool::new(false));
let destructive_stage_for_backup = Arc::clone(&destructive_stage_reached);
let backup = async move {
std::future::pending::<()>().await;
destructive_stage_for_backup.store(true, Ordering::Release);
Ok::<(), super::S3BackupTaskError>(())
};
let heartbeat = tokio::spawn(async {});
let outcome = super::race_backup_with_lock_heartbeat(backup, heartbeat).await;
assert!(matches!(
outcome,
super::BackupLockRaceOutcome::LeaseLost(Ok(()))
));
assert!(!destructive_stage_reached.load(Ordering::Acquire));
}
#[tokio::test]
async fn completed_heartbeat_wins_when_backup_completion_is_also_ready() {
let heartbeat = tokio::spawn(async {});
tokio::task::yield_now().await;
let outcome = super::race_backup_with_lock_heartbeat(async { 42_u8 }, heartbeat).await;
assert!(matches!(
outcome,
super::BackupLockRaceOutcome::LeaseLost(Ok(()))
));
}
#[tokio::test]
async fn s3_backup_snapshot_refuses_to_overlap_system_import() {
let app = AppState::new().expect("app state should build");
let lease = crate::admin_api::try_acquire_admin_system_import_lease(&app)
.await
.expect("test should acquire the system import lease");
let error = super::build_s3_backup_payload_exclusively(
&app,
crate::backup::scopes::BackupScope::Config,
)
.await
.expect_err("backup snapshot must not overlap a system import");
crate::admin_api::release_admin_system_import_lease(&app, &lease).await;
assert_eq!(error.status(), axum::http::StatusCode::SERVICE_UNAVAILABLE);
assert_eq!(error.detail(), super::S3_BACKUP_INTERNAL_ERROR_DETAIL);
}
fn valid_s3_backup_config_values() -> Vec<(String, serde_json::Value)> {
vec![
(
@@ -632,6 +899,92 @@ mod tests {
]
}
#[tokio::test]
async fn legacy_plaintext_s3_secret_is_migrated_when_config_loads() {
let plaintext = "legacy-s3-secret-access-key";
let mut entries = valid_s3_backup_config_values();
entries
.iter_mut()
.find(|(key, _)| key == "backup_s3_secret_access_key")
.expect("secret config fixture should exist")
.1 = serde_json::json!(plaintext);
let app = AppState::new()
.expect("app state should build")
.with_data_state_for_tests(
GatewayDataState::disabled()
.with_encryption_key_for_tests(DEVELOPMENT_ENCRYPTION_KEY)
.with_system_config_values_for_tests(entries),
);
let values = super::load_s3_backup_config_values(&app)
.await
.expect("legacy S3 config should load");
assert_eq!(
values.get("backup_s3_secret_access_key"),
Some(&serde_json::json!(plaintext))
);
let stored = app
.read_system_config_json_value_strong("backup_s3_secret_access_key")
.await
.expect("stored S3 secret should read")
.and_then(|value| value.as_str().map(ToOwned::to_owned))
.expect("stored S3 secret should remain a string");
assert_ne!(stored, plaintext);
assert_eq!(
decrypt_system_config_secret(&app, "backup_s3_secret_access_key", &stored)
.expect("migrated S3 secret should decrypt"),
plaintext
);
}
#[tokio::test]
async fn undecryptable_s3_fernet_secret_fails_closed() {
let plaintext = "s3-secret-from-unavailable-key";
let ciphertext = encrypt_python_fernet_plaintext("unavailable-s3-key", plaintext)
.expect("unknown-key fixture should encrypt");
let mut entries = valid_s3_backup_config_values();
entries
.iter_mut()
.find(|(key, _)| key == "backup_s3_secret_access_key")
.expect("secret config fixture should exist")
.1 = serde_json::json!(ciphertext.clone());
let app = AppState::new()
.expect("app state should build")
.with_data_state_for_tests(
GatewayDataState::disabled()
.with_encryption_key_for_tests(DEVELOPMENT_ENCRYPTION_KEY)
.with_system_config_values_for_tests(entries),
);
let error = super::load_s3_backup_config_values(&app)
.await
.expect_err("unknown-key S3 ciphertext must fail closed");
let error_text = error.to_string();
assert!(!error_text.contains(plaintext));
assert!(!error_text.contains(&ciphertext));
assert_eq!(
app.read_system_config_json_value_strong("backup_s3_secret_access_key")
.await
.expect("stored S3 secret should read"),
Some(serde_json::json!(ciphertext))
);
}
#[test]
fn gateway_dependency_errors_are_not_exposed_to_backup_clients() {
let error = super::S3BackupTaskError::from(crate::GatewayError::Internal(
"postgresql://admin:[email protected]/aether".to_string(),
));
assert_eq!(
error.status(),
axum::http::StatusCode::INTERNAL_SERVER_ERROR
);
assert_eq!(error.detail(), super::S3_BACKUP_INTERNAL_ERROR_DETAIL);
assert!(!error.detail().contains("database-secret"));
}
fn stored_s3_backup_run(status: BackgroundTaskStatus) -> StoredBackgroundTaskRun {
let now = now_unix_secs();
StoredBackgroundTaskRun {
@@ -774,7 +1127,9 @@ mod tests {
let payload = super::s3_backup_task_payload_json(&config, "manual", None);
assert!(payload["bucket"].is_string());
assert!(payload.get("bucket").is_none());
assert!(payload.get("prefix").is_none());
assert_eq!(payload["scope"], serde_json::json!("data"));
assert_eq!(payload["trigger"], serde_json::json!("manual"));
assert!(!payload.to_string().contains("secret"));
}
apps/aether-gateway/src/backup/worker.rs
+20 -8
View File
@@ -29,8 +29,11 @@ pub(crate) fn spawn_s3_backup_worker(app: AppState) -> Option<JoinHandle<()>> {
interval.tick().await;
loop {
interval.tick().await;
if let Err(error) = run_s3_backup_schedule_tick(&app, Utc::now()).await {
warn!(error = ?error, "S3 backup schedule tick failed");
if run_s3_backup_schedule_tick(&app, Utc::now()).await.is_err() {
warn!(
error_category = "schedule_tick_failed",
"S3 backup schedule tick failed"
);
}
}
},
@@ -43,15 +46,21 @@ async fn run_s3_backup_schedule_tick(
) -> Result<(), GatewayError> {
let values = match super::task::load_s3_backup_config_values(app).await {
Ok(values) => values,
Err(error) => {
warn!(error = %error, "S3 backup schedule config load failed");
Err(_) => {
warn!(
error_category = "config_load_failed",
"S3 backup schedule config load failed"
);
return Ok(());
}
};
let config = match S3BackupConfig::from_json_map(&values) {
Ok(config) => config,
Err(error) => {
warn!(error = %error, "S3 backup schedule config is invalid");
Err(_) => {
warn!(
error_category = "config_invalid",
"S3 backup schedule config is invalid"
);
return Ok(());
}
};
@@ -68,8 +77,11 @@ async fn run_s3_backup_schedule_tick(
match super::task::start_s3_backup_task_for_schedule(app.clone(), slot).await {
Ok(_) => {}
Err(error) => {
warn!(error = %error, "S3 backup scheduled task submission failed");
Err(_) => {
warn!(
error_category = "task_submission_failed",
"S3 backup scheduled task submission failed"
);
}
}
Ok(())