mirror of
https://github.com/fawney19/Aether.git
synced 2026-10-07 18:07:47 +08:00
feat(security): harden gateway boundaries and usage policies
Consolidate subscription usage policy enforcement, privacy-safe persistence, and gateway security hardening into one reviewable change. Includes bounded HTTP and execution envelopes, header and protocol guards, DNS and relay validation, authentication and secret projection hardening, secure backup/install paths, and regression coverage.
This commit is contained in:
@@ -7,7 +7,10 @@ use tracing::info;
|
||||
// Generated by build.rs from schema/bootstrap/postgres.
|
||||
pub(crate) static EMPTY_DATABASE_SNAPSHOT_SQL: &str =
|
||||
include_str!(concat!(env!("OUT_DIR"), "/empty_database_snapshot.sql"));
|
||||
pub(crate) const EMPTY_DATABASE_SNAPSHOT_CUTOFF_VERSION: i64 = 20260903000000;
|
||||
// Keep data migrations after the privacy/security frontier executable on a
|
||||
// fresh database. The bootstrap SQL is schema-only; stamping later data
|
||||
// migrations would skip required cleanup/anonymization work.
|
||||
pub(crate) const EMPTY_DATABASE_SNAPSHOT_CUTOFF_VERSION: i64 = 20260821130000;
|
||||
|
||||
const PUBLIC_BASE_TABLE_COUNT_SQL: &str = r#"
|
||||
SELECT COUNT(*)::BIGINT
|
||||
|
||||
@@ -3,12 +3,18 @@ use std::collections::{BTreeMap, BTreeSet};
|
||||
#[cfg(all(feature = "postgres", feature = "sqlite"))]
|
||||
use futures_util::TryStreamExt;
|
||||
use serde_json::Value;
|
||||
use sha2::{Digest, Sha256};
|
||||
#[cfg(all(feature = "postgres", feature = "sqlite"))]
|
||||
use sqlx::Acquire;
|
||||
use sqlx::Row;
|
||||
#[cfg(any(feature = "mysql", feature = "sqlite"))]
|
||||
use sqlx::{Column, TypeInfo, ValueRef};
|
||||
|
||||
use aether_data_contracts::repository::candidates::{
|
||||
sanitize_request_candidate_error_type, sanitize_request_candidate_extra_data,
|
||||
sanitize_request_candidate_required_capabilities, sanitize_request_candidate_skip_reason,
|
||||
};
|
||||
|
||||
use crate::error::SqlResultExt;
|
||||
use crate::{DataLayerError, DatabaseDriver, SqlDatabaseConfig};
|
||||
|
||||
@@ -46,6 +52,17 @@ use postgres::normalize_postgres_import_payload;
|
||||
pub const EXPORT_FORMAT_VERSION: u32 = 2;
|
||||
const MIN_SUPPORTED_EXPORT_FORMAT_VERSION: u32 = 1;
|
||||
|
||||
// JSONL imports are ultimately materialized as a `DataImportPlan`, so an
|
||||
// attacker-controlled document can otherwise consume memory in both the input
|
||||
// string and the parsed row/payload vectors. Keep these bounds deliberately
|
||||
// separate from HTTP request limits: database exports may contain large body
|
||||
// blobs, while still needing a finite parser budget. The total budget is kept
|
||||
// below the gateway's 256 MiB request-body ceiling because parsing duplicates
|
||||
// portions of the input in serde values and the import plan.
|
||||
pub const MAX_JSONL_INPUT_BYTES: usize = 256 * 1024 * 1024;
|
||||
pub const MAX_JSONL_LINE_BYTES: usize = 16 * 1024 * 1024;
|
||||
pub const MAX_JSONL_RECORDS: usize = 1_000_000;
|
||||
|
||||
#[derive(
|
||||
Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, serde::Serialize, serde::Deserialize,
|
||||
)]
|
||||
@@ -223,6 +240,14 @@ const AUXILIARY_TABLES: &[AuxiliaryTable] = &[
|
||||
name: "usage_counter_deltas",
|
||||
primary_key: &["id"],
|
||||
},
|
||||
AuxiliaryTable {
|
||||
name: "usage_cost_reservations",
|
||||
primary_key: &["reservation_token"],
|
||||
},
|
||||
AuxiliaryTable {
|
||||
name: "usage_request_admissions",
|
||||
primary_key: &["event_token"],
|
||||
},
|
||||
AuxiliaryTable {
|
||||
name: "background_task_runs",
|
||||
primary_key: &["id"],
|
||||
@@ -447,6 +472,10 @@ impl DataImportPlan {
|
||||
.map(Vec::as_slice)
|
||||
.unwrap_or(&[])
|
||||
}
|
||||
|
||||
fn imports_domain(&self, domain: ExportDomain) -> bool {
|
||||
self.manifest.domains.contains(&domain)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
@@ -455,6 +484,83 @@ pub struct ExportRow {
|
||||
pub payload: Value,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
struct IdentityImportScope {
|
||||
user_ids: Vec<String>,
|
||||
oauth_link_ids: Vec<String>,
|
||||
oauth_provider_types: Vec<String>,
|
||||
finalizes_oauth_links: bool,
|
||||
validates_oauth_login_methods: bool,
|
||||
}
|
||||
|
||||
impl IdentityImportScope {
|
||||
fn from_plan(plan: &DataImportPlan) -> Result<Self, DataLayerError> {
|
||||
let scope = Self {
|
||||
user_ids: imported_payload_ids(plan, ExportDomain::Users, "id")?,
|
||||
oauth_link_ids: imported_payload_ids(plan, ExportDomain::UserOAuthLinks, "id")?,
|
||||
oauth_provider_types: imported_payload_ids(
|
||||
plan,
|
||||
ExportDomain::OAuthProviders,
|
||||
"provider_type",
|
||||
)?,
|
||||
finalizes_oauth_links: plan.imports_domain(ExportDomain::UserOAuthLinks),
|
||||
validates_oauth_login_methods: plan.imports_domain(ExportDomain::UserOAuthLinks)
|
||||
|| plan.imports_domain(ExportDomain::OAuthProviders),
|
||||
};
|
||||
if let Some(provider_type) = scope.oauth_provider_types.iter().find(|provider_type| {
|
||||
provider_type.is_empty()
|
||||
|| provider_type.as_str() != provider_type.trim().to_ascii_lowercase()
|
||||
}) {
|
||||
return Err(DataLayerError::InvalidInput(format!(
|
||||
"OAuth provider import has non-canonical provider_type '{provider_type}'"
|
||||
)));
|
||||
}
|
||||
Ok(scope)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
struct IdentityImportState {
|
||||
affected_user_ids: BTreeSet<String>,
|
||||
}
|
||||
|
||||
fn imported_payload_ids(
|
||||
plan: &DataImportPlan,
|
||||
domain: ExportDomain,
|
||||
payload_field: &str,
|
||||
) -> Result<Vec<String>, DataLayerError> {
|
||||
plan.rows(domain)
|
||||
.iter()
|
||||
.map(|row| {
|
||||
let payload_id = row
|
||||
.payload
|
||||
.as_object()
|
||||
.and_then(|payload| payload.get(payload_field))
|
||||
.and_then(Value::as_str)
|
||||
.map(str::trim)
|
||||
.filter(|id| !id.is_empty())
|
||||
.ok_or_else(|| {
|
||||
DataLayerError::InvalidInput(format!(
|
||||
"{} export row '{}' must contain a non-empty string {}",
|
||||
domain.as_str(),
|
||||
row.id,
|
||||
payload_field
|
||||
))
|
||||
})?;
|
||||
if payload_id != row.id {
|
||||
return Err(DataLayerError::InvalidInput(format!(
|
||||
"{} export row id '{}' does not match payload {} '{}'",
|
||||
domain.as_str(),
|
||||
row.id,
|
||||
payload_field,
|
||||
payload_id
|
||||
)));
|
||||
}
|
||||
Ok(payload_id.to_string())
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
pub struct DataCopyOptions {
|
||||
pub omit_request_body_details: bool,
|
||||
@@ -508,6 +614,139 @@ type PostgresImportColumns = BTreeMap<String, PostgresImportColumn>;
|
||||
#[cfg(any(feature = "mysql", feature = "sqlite"))]
|
||||
type ImportColumnNames = BTreeSet<String>;
|
||||
|
||||
const IMPORTED_CREDENTIAL_REVOKE_REASON: &str = "imported_credentials_revoked";
|
||||
|
||||
fn imported_credential_tombstone() -> String {
|
||||
format!("{:x}", Sha256::digest(uuid::Uuid::new_v4().as_bytes()))
|
||||
}
|
||||
|
||||
fn set_supported_import_value(
|
||||
object: &mut serde_json::Map<String, Value>,
|
||||
target_has_column: &impl Fn(&str) -> bool,
|
||||
column: &str,
|
||||
value: Value,
|
||||
) {
|
||||
if target_has_column(column) {
|
||||
object.insert(column.to_string(), value);
|
||||
}
|
||||
}
|
||||
|
||||
fn deactivate_imported_credentials(
|
||||
table_name: &str,
|
||||
object: &mut serde_json::Map<String, Value>,
|
||||
target_has_column: impl Fn(&str) -> bool,
|
||||
) {
|
||||
let table_name = table_name
|
||||
.rsplit('.')
|
||||
.next()
|
||||
.unwrap_or(table_name)
|
||||
.trim_matches(|ch| matches!(ch, '"' | '`'));
|
||||
|
||||
match table_name {
|
||||
"users" => {
|
||||
if object
|
||||
.get("password_hash")
|
||||
.is_some_and(|value| !value.is_null())
|
||||
{
|
||||
set_supported_import_value(
|
||||
object,
|
||||
&target_has_column,
|
||||
"password_hash",
|
||||
Value::String(format!(
|
||||
"$aether-import-revoked${}",
|
||||
imported_credential_tombstone()
|
||||
)),
|
||||
);
|
||||
}
|
||||
}
|
||||
"api_keys" => {
|
||||
if object.contains_key("key_hash") {
|
||||
set_supported_import_value(
|
||||
object,
|
||||
&target_has_column,
|
||||
"key_hash",
|
||||
Value::String(imported_credential_tombstone()),
|
||||
);
|
||||
}
|
||||
set_supported_import_value(object, &target_has_column, "key_encrypted", Value::Null);
|
||||
set_supported_import_value(
|
||||
object,
|
||||
&target_has_column,
|
||||
"status",
|
||||
Value::String("disabled".to_string()),
|
||||
);
|
||||
set_supported_import_value(object, &target_has_column, "is_active", Value::Bool(false));
|
||||
set_supported_import_value(object, &target_has_column, "is_locked", Value::Bool(true));
|
||||
}
|
||||
"management_tokens" => {
|
||||
if object.contains_key("token_hash") {
|
||||
set_supported_import_value(
|
||||
object,
|
||||
&target_has_column,
|
||||
"token_hash",
|
||||
Value::String(imported_credential_tombstone()),
|
||||
);
|
||||
}
|
||||
set_supported_import_value(object, &target_has_column, "is_active", Value::Bool(false));
|
||||
}
|
||||
"user_sessions" => {
|
||||
if object.contains_key("refresh_token_hash") {
|
||||
set_supported_import_value(
|
||||
object,
|
||||
&target_has_column,
|
||||
"refresh_token_hash",
|
||||
Value::String(imported_credential_tombstone()),
|
||||
);
|
||||
}
|
||||
set_supported_import_value(
|
||||
object,
|
||||
&target_has_column,
|
||||
"prev_refresh_token_hash",
|
||||
Value::Null,
|
||||
);
|
||||
set_supported_import_value(
|
||||
object,
|
||||
&target_has_column,
|
||||
"revoked_at",
|
||||
Value::from(chrono::Utc::now().timestamp()),
|
||||
);
|
||||
set_supported_import_value(
|
||||
object,
|
||||
&target_has_column,
|
||||
"revoke_reason",
|
||||
Value::String(IMPORTED_CREDENTIAL_REVOKE_REASON.to_string()),
|
||||
);
|
||||
}
|
||||
"proxy_nodes" => {
|
||||
set_supported_import_value(
|
||||
object,
|
||||
&target_has_column,
|
||||
"tunnel_generation",
|
||||
Value::String(uuid::Uuid::new_v4().to_string()),
|
||||
);
|
||||
set_supported_import_value(
|
||||
object,
|
||||
&target_has_column,
|
||||
"tunnel_connected",
|
||||
Value::Bool(false),
|
||||
);
|
||||
set_supported_import_value(
|
||||
object,
|
||||
&target_has_column,
|
||||
"status",
|
||||
Value::String("offline".to_string()),
|
||||
);
|
||||
set_supported_import_value(
|
||||
object,
|
||||
&target_has_column,
|
||||
"active_connections",
|
||||
Value::from(0),
|
||||
);
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
const USAGE_REQUEST_BODY_DETAIL_COLUMNS: &[&str] = &[
|
||||
"request_body",
|
||||
"response_body",
|
||||
@@ -691,6 +930,27 @@ pub fn encode_jsonl(records: &[DataExportRecord]) -> Result<String, DataLayerErr
|
||||
for record in records {
|
||||
let line = serde_json::to_string(record)
|
||||
.map_err(|err| DataLayerError::UnexpectedValue(err.to_string()))?;
|
||||
if line.len() > MAX_JSONL_LINE_BYTES {
|
||||
return Err(DataLayerError::InvalidInput(format!(
|
||||
"export JSONL record exceeds the {} byte line limit",
|
||||
MAX_JSONL_LINE_BYTES
|
||||
)));
|
||||
}
|
||||
let output_len = output
|
||||
.len()
|
||||
.checked_add(line.len())
|
||||
.and_then(|length| length.checked_add(1))
|
||||
.ok_or_else(|| {
|
||||
DataLayerError::InvalidInput(
|
||||
"export JSONL exceeds the input size limit".to_string(),
|
||||
)
|
||||
})?;
|
||||
if output_len > MAX_JSONL_INPUT_BYTES {
|
||||
return Err(DataLayerError::InvalidInput(format!(
|
||||
"export JSONL exceeds the {} byte input limit",
|
||||
MAX_JSONL_INPUT_BYTES
|
||||
)));
|
||||
}
|
||||
output.push_str(&line);
|
||||
output.push('\n');
|
||||
}
|
||||
@@ -698,11 +958,42 @@ pub fn encode_jsonl(records: &[DataExportRecord]) -> Result<String, DataLayerErr
|
||||
}
|
||||
|
||||
pub fn decode_jsonl(input: &str) -> Result<Vec<DataExportRecord>, DataLayerError> {
|
||||
decode_jsonl_with_limits(
|
||||
input,
|
||||
MAX_JSONL_INPUT_BYTES,
|
||||
MAX_JSONL_LINE_BYTES,
|
||||
MAX_JSONL_RECORDS,
|
||||
)
|
||||
}
|
||||
|
||||
fn decode_jsonl_with_limits(
|
||||
input: &str,
|
||||
max_input_bytes: usize,
|
||||
max_line_bytes: usize,
|
||||
max_records: usize,
|
||||
) -> Result<Vec<DataExportRecord>, DataLayerError> {
|
||||
if input.len() > max_input_bytes {
|
||||
return Err(DataLayerError::InvalidInput(format!(
|
||||
"export JSONL exceeds the {max_input_bytes} byte input limit"
|
||||
)));
|
||||
}
|
||||
|
||||
let mut records = Vec::new();
|
||||
for (line_index, line) in input.lines().enumerate() {
|
||||
if line.len() > max_line_bytes {
|
||||
return Err(DataLayerError::InvalidInput(format!(
|
||||
"export JSONL record on line {} exceeds the {max_line_bytes} byte line limit",
|
||||
line_index + 1,
|
||||
)));
|
||||
}
|
||||
if line.trim().is_empty() {
|
||||
continue;
|
||||
}
|
||||
if records.len() >= max_records {
|
||||
return Err(DataLayerError::InvalidInput(format!(
|
||||
"export JSONL exceeds the {max_records} record limit"
|
||||
)));
|
||||
}
|
||||
let record = serde_json::from_str::<DataExportRecord>(line).map_err(|err| {
|
||||
DataLayerError::InvalidInput(format!(
|
||||
"invalid export JSONL record on line {}: {err}",
|
||||
@@ -745,6 +1036,12 @@ pub fn build_import_plan(input: &str) -> Result<DataImportPlan, DataLayerError>
|
||||
}
|
||||
|
||||
pub fn validate_export_records(records: &[DataExportRecord]) -> Result<(), DataLayerError> {
|
||||
if records.len() > MAX_JSONL_RECORDS {
|
||||
return Err(DataLayerError::InvalidInput(format!(
|
||||
"export JSONL exceeds the {} record limit",
|
||||
MAX_JSONL_RECORDS
|
||||
)));
|
||||
}
|
||||
let Some(DataExportRecord::Manifest { manifest }) = records.first() else {
|
||||
return Err(DataLayerError::InvalidInput(
|
||||
"export JSONL must start with a manifest record".to_string(),
|
||||
@@ -1154,13 +1451,14 @@ async fn copy_postgres_sqlite_table(
|
||||
let mut imported = 0usize;
|
||||
|
||||
while let Some(row) = rows.try_next().await.map_sql_err()? {
|
||||
let payload = row.try_get::<Value, _>("payload").map_sql_err()?;
|
||||
let object = payload.as_object().ok_or_else(|| {
|
||||
let mut payload = row.try_get::<Value, _>("payload").map_sql_err()?;
|
||||
let object = payload.as_object_mut().ok_or_else(|| {
|
||||
DataLayerError::UnexpectedValue(format!(
|
||||
"postgres copy row for table '{}' did not produce a JSON object",
|
||||
table.table_name
|
||||
))
|
||||
})?;
|
||||
prepare_postgres_sqlite_copy_payload(table, object);
|
||||
let mut query = sqlx::query(&target_sql);
|
||||
for column in &table.columns {
|
||||
let value = object.get(&column.sqlite.name).ok_or_else(|| {
|
||||
@@ -1178,6 +1476,21 @@ async fn copy_postgres_sqlite_table(
|
||||
Ok(imported)
|
||||
}
|
||||
|
||||
#[cfg(all(feature = "postgres", feature = "sqlite"))]
|
||||
fn prepare_postgres_sqlite_copy_payload(
|
||||
table: &SchemaCopyTable,
|
||||
object: &mut serde_json::Map<String, Value>,
|
||||
) {
|
||||
deactivate_imported_credentials(&table.table_name, object, |column_name| {
|
||||
table
|
||||
.columns
|
||||
.iter()
|
||||
.any(|column| column.sqlite.name == column_name)
|
||||
});
|
||||
sanitize_request_candidate_auxiliary_payload(&table.table_name, object);
|
||||
sanitize_payment_security_payload(&table.table_name, object);
|
||||
}
|
||||
|
||||
#[cfg(all(feature = "postgres", feature = "sqlite"))]
|
||||
fn postgres_schema_copy_select_sql(table: &SchemaCopyTable) -> Result<String, DataLayerError> {
|
||||
let table_sql = format!(
|
||||
@@ -1743,10 +2056,85 @@ fn payload_with_table(payload: Value, table_name: &str) -> Result<Value, DataLay
|
||||
DataLayerError::UnexpectedValue("export row payload must be a JSON object".to_string())
|
||||
})?;
|
||||
normalize_billing_payload(table_name, &mut object)?;
|
||||
sanitize_request_candidate_auxiliary_payload(table_name, &mut object);
|
||||
sanitize_payment_security_payload(table_name, &mut object);
|
||||
object.insert("__table".to_string(), Value::String(table_name.to_string()));
|
||||
Ok(Value::Object(object))
|
||||
}
|
||||
|
||||
fn sanitize_request_candidate_auxiliary_payload(
|
||||
table_name: &str,
|
||||
object: &mut serde_json::Map<String, Value>,
|
||||
) {
|
||||
if table_name != "request_candidates" {
|
||||
return;
|
||||
}
|
||||
|
||||
object.insert("error_message".to_string(), Value::Null);
|
||||
sanitize_request_candidate_auxiliary_string(
|
||||
object,
|
||||
"skip_reason",
|
||||
sanitize_request_candidate_skip_reason,
|
||||
);
|
||||
sanitize_request_candidate_auxiliary_string(
|
||||
object,
|
||||
"error_type",
|
||||
sanitize_request_candidate_error_type,
|
||||
);
|
||||
sanitize_request_candidate_auxiliary_json(
|
||||
object,
|
||||
"extra_data",
|
||||
sanitize_request_candidate_extra_data,
|
||||
);
|
||||
sanitize_request_candidate_auxiliary_json(
|
||||
object,
|
||||
"required_capabilities",
|
||||
sanitize_request_candidate_required_capabilities,
|
||||
);
|
||||
}
|
||||
|
||||
fn sanitize_payment_security_payload(
|
||||
table_name: &str,
|
||||
object: &mut serde_json::Map<String, Value>,
|
||||
) {
|
||||
match table_name {
|
||||
"payment_orders" => {
|
||||
object.insert("gateway_response".to_string(), Value::Null);
|
||||
}
|
||||
"payment_callbacks" => {
|
||||
object.insert("payload".to_string(), Value::Null);
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
fn sanitize_request_candidate_auxiliary_string(
|
||||
object: &mut serde_json::Map<String, Value>,
|
||||
field: &str,
|
||||
sanitize: fn(Option<String>) -> Option<String>,
|
||||
) {
|
||||
let value = object
|
||||
.remove(field)
|
||||
.and_then(|value| value.as_str().map(ToOwned::to_owned));
|
||||
object.insert(
|
||||
field.to_string(),
|
||||
sanitize(value).map_or(Value::Null, Value::String),
|
||||
);
|
||||
}
|
||||
|
||||
fn sanitize_request_candidate_auxiliary_json(
|
||||
object: &mut serde_json::Map<String, Value>,
|
||||
field: &str,
|
||||
sanitize: fn(Option<Value>) -> Option<Value>,
|
||||
) {
|
||||
let value = object.remove(field).and_then(|value| match value {
|
||||
Value::Null => None,
|
||||
Value::String(raw) => serde_json::from_str::<Value>(&raw).ok(),
|
||||
value => Some(value),
|
||||
});
|
||||
object.insert(field.to_string(), sanitize(value).unwrap_or(Value::Null));
|
||||
}
|
||||
|
||||
fn normalize_billing_payload(
|
||||
table_name: &str,
|
||||
object: &mut serde_json::Map<String, Value>,
|
||||
@@ -1800,5 +2188,197 @@ fn domain_payload_table(
|
||||
))
|
||||
})?,
|
||||
};
|
||||
sanitize_request_candidate_auxiliary_payload(&table_name, &mut object);
|
||||
sanitize_payment_security_payload(&table_name, &mut object);
|
||||
Ok((table_name, Value::Object(object)))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod payment_export_security_tests {
|
||||
use serde_json::json;
|
||||
|
||||
use super::{domain_payload_table, payload_with_table, ExportRow};
|
||||
|
||||
#[test]
|
||||
fn wallet_exports_and_imports_drop_payment_capabilities_and_raw_callbacks() {
|
||||
let order = payload_with_table(
|
||||
json!({
|
||||
"id": "order-1",
|
||||
"gateway_response": {
|
||||
"client_secret": "pi_1_secret_replayable",
|
||||
"_stripe_client_secret_encrypted": "ciphertext",
|
||||
"customer": {"email": "[email protected]"},
|
||||
"payment_url": "https://pay.example/checkout?token=secret",
|
||||
},
|
||||
}),
|
||||
"payment_orders",
|
||||
)
|
||||
.expect("payment order export should sanitize");
|
||||
assert!(order["gateway_response"].is_null());
|
||||
|
||||
let callback = ExportRow {
|
||||
id: "payment_callbacks:callback-1".to_string(),
|
||||
payload: json!({
|
||||
"__table": "payment_callbacks",
|
||||
"id": "callback-1",
|
||||
"payload": {
|
||||
"client_secret": "pi_1_secret_replayable",
|
||||
"customer_email": "[email protected]",
|
||||
},
|
||||
}),
|
||||
};
|
||||
let (table, callback) = domain_payload_table(&callback, "wallet", Some("wallets"))
|
||||
.expect("payment callback import should sanitize");
|
||||
assert_eq!(table, "payment_callbacks");
|
||||
assert!(callback["payload"].is_null());
|
||||
|
||||
let encoded = format!("{order}{callback}");
|
||||
for forbidden in [
|
||||
"client_secret",
|
||||
"replayable",
|
||||
"ciphertext",
|
||||
"customer",
|
||||
"[email protected]",
|
||||
"token=secret",
|
||||
] {
|
||||
assert!(!encoded.contains(forbidden), "exported {forbidden}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod request_candidate_export_security_tests {
|
||||
use serde_json::json;
|
||||
|
||||
#[cfg(all(feature = "postgres", feature = "sqlite"))]
|
||||
use serde_json::Value;
|
||||
|
||||
use super::{domain_payload_table, payload_with_table, ExportRow};
|
||||
|
||||
#[cfg(all(feature = "postgres", feature = "sqlite"))]
|
||||
use super::{
|
||||
prepare_postgres_sqlite_copy_payload, PostgresImportColumn, SchemaCopyColumn,
|
||||
SchemaCopyTable, SqliteCopyColumn,
|
||||
};
|
||||
|
||||
#[test]
|
||||
fn request_candidate_auxiliary_export_and_import_drop_sensitive_diagnostics() {
|
||||
let raw = json!({
|
||||
"id": "candidate-1",
|
||||
"error_message": "Bearer export-secret",
|
||||
"skip_reason": "secret skip reason",
|
||||
"error_type": "secret error type",
|
||||
"extra_data": "{\"upstream_url\":\"https://user:[email protected]/private/export-secret?token=secret\",\"unknown\":\"secret\",\"header_rules\":[{\"id\":\"secret-rule\",\"action\":\"set\",\"name\":\"authorization\",\"value\":\"secret\"}]}",
|
||||
"required_capabilities": "{\"cache_1h\":\"true\",\"tenant_secret\":\"secret\"}"
|
||||
});
|
||||
|
||||
let exported = payload_with_table(raw, "request_candidates")
|
||||
.expect("candidate export payload should sanitize");
|
||||
assert!(exported["error_message"].is_null());
|
||||
assert_eq!(exported["skip_reason"], "unclassified_skip");
|
||||
assert_eq!(exported["error_type"], "unclassified_error");
|
||||
assert_eq!(
|
||||
exported["extra_data"]["upstream_url"],
|
||||
"https://example.com/"
|
||||
);
|
||||
assert_eq!(exported["extra_data"]["header_rules"]["count"], 1);
|
||||
assert_eq!(exported["required_capabilities"]["cache_1h"], true);
|
||||
let encoded = exported.to_string();
|
||||
for sensitive in [
|
||||
"export-secret",
|
||||
"user:pass",
|
||||
"secret-rule",
|
||||
"authorization",
|
||||
"tenant_secret",
|
||||
] {
|
||||
assert!(!encoded.contains(sensitive));
|
||||
}
|
||||
|
||||
let imported_row = ExportRow {
|
||||
id: "request_candidates:[\"candidate-1\"]".to_string(),
|
||||
payload: json!({
|
||||
"__table": "request_candidates",
|
||||
"id": "candidate-1",
|
||||
"error_message": "Bearer import-secret",
|
||||
"extra_data": {"free_text": "import-secret"},
|
||||
"required_capabilities": {"vision": 1, "secret": "import-secret"}
|
||||
}),
|
||||
};
|
||||
let (table, imported) = domain_payload_table(&imported_row, "auxiliary", None)
|
||||
.expect("candidate import payload should sanitize");
|
||||
assert_eq!(table, "request_candidates");
|
||||
assert!(imported["error_message"].is_null());
|
||||
assert!(imported["extra_data"].is_null());
|
||||
assert_eq!(imported["required_capabilities"], json!({"vision": true}));
|
||||
assert!(!imported.to_string().contains("import-secret"));
|
||||
}
|
||||
|
||||
#[cfg(all(feature = "postgres", feature = "sqlite"))]
|
||||
#[test]
|
||||
fn postgres_to_sqlite_fast_copy_sanitizes_request_candidate_diagnostics() {
|
||||
let table = SchemaCopyTable {
|
||||
table_name: "request_candidates".to_string(),
|
||||
columns: [
|
||||
"error_message",
|
||||
"skip_reason",
|
||||
"error_type",
|
||||
"extra_data",
|
||||
"required_capabilities",
|
||||
]
|
||||
.into_iter()
|
||||
.map(|name| SchemaCopyColumn {
|
||||
sqlite: SqliteCopyColumn {
|
||||
name: name.to_string(),
|
||||
declared_type: "TEXT".to_string(),
|
||||
not_null: false,
|
||||
has_default: false,
|
||||
primary_key_position: 0,
|
||||
},
|
||||
postgres: PostgresImportColumn {
|
||||
data_type: "text".to_string(),
|
||||
udt_name: "text".to_string(),
|
||||
is_nullable: true,
|
||||
has_default: false,
|
||||
},
|
||||
})
|
||||
.collect(),
|
||||
};
|
||||
let mut payload = json!({
|
||||
"error_message": "Bearer fast-copy-secret",
|
||||
"skip_reason": "fast-copy-secret",
|
||||
"error_type": "fast-copy-secret",
|
||||
"extra_data": {
|
||||
"upstream_url": "https://user:[email protected]/private?token=fast-copy-secret",
|
||||
"image_progress": {
|
||||
"phase": "upstream_streaming",
|
||||
"message": "fast-copy-secret"
|
||||
}
|
||||
},
|
||||
"required_capabilities": {
|
||||
"vision": 1,
|
||||
"tenant_secret": "fast-copy-secret"
|
||||
}
|
||||
})
|
||||
.as_object()
|
||||
.cloned()
|
||||
.expect("copy payload should be an object");
|
||||
|
||||
prepare_postgres_sqlite_copy_payload(&table, &mut payload);
|
||||
|
||||
assert!(payload["error_message"].is_null());
|
||||
assert_eq!(payload["skip_reason"], "unclassified_skip");
|
||||
assert_eq!(payload["error_type"], "unclassified_error");
|
||||
assert_eq!(
|
||||
payload["extra_data"]["upstream_url"],
|
||||
"https://example.com/"
|
||||
);
|
||||
assert_eq!(
|
||||
payload["extra_data"]["image_progress"],
|
||||
json!({"phase": "upstream_streaming"})
|
||||
);
|
||||
assert_eq!(payload["required_capabilities"], json!({"vision": true}));
|
||||
assert!(!Value::Object(payload)
|
||||
.to_string()
|
||||
.contains("fast-copy-secret"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -72,7 +72,9 @@ pub async fn import_mysql_plan(
|
||||
pool: &crate::driver::mysql::MysqlPool,
|
||||
plan: &DataImportPlan,
|
||||
) -> Result<usize, DataLayerError> {
|
||||
let identity_scope = IdentityImportScope::from_plan(plan)?;
|
||||
let mut tx = pool.begin().await.map_sql_err()?;
|
||||
let identity_state = capture_mysql_identity_import_state(&mut tx, &identity_scope).await?;
|
||||
let mut imported = 0usize;
|
||||
let mut column_cache = BTreeMap::<String, MysqlImportColumns>::new();
|
||||
for domain in &plan.manifest.domains {
|
||||
@@ -105,10 +107,210 @@ pub async fn import_mysql_plan(
|
||||
imported = imported.saturating_add(1);
|
||||
}
|
||||
}
|
||||
enforce_mysql_identity_import_invariants(&mut tx, &identity_scope, identity_state).await?;
|
||||
tx.commit().await.map_sql_err()?;
|
||||
Ok(imported)
|
||||
}
|
||||
|
||||
async fn capture_mysql_identity_import_state(
|
||||
tx: &mut sqlx::Transaction<'_, sqlx::MySql>,
|
||||
scope: &IdentityImportScope,
|
||||
) -> Result<IdentityImportState, DataLayerError> {
|
||||
let mut affected_user_ids = if scope.finalizes_oauth_links {
|
||||
scope.user_ids.iter().cloned().collect::<BTreeSet<_>>()
|
||||
} else {
|
||||
BTreeSet::new()
|
||||
};
|
||||
for link_id in &scope.oauth_link_ids {
|
||||
if let Some(user_id) =
|
||||
sqlx::query_scalar::<_, String>("SELECT user_id FROM user_oauth_links WHERE id = ?")
|
||||
.bind(link_id)
|
||||
.fetch_optional(&mut **tx)
|
||||
.await
|
||||
.map_sql_err()?
|
||||
{
|
||||
affected_user_ids.insert(user_id);
|
||||
}
|
||||
}
|
||||
for provider_type in &scope.oauth_provider_types {
|
||||
let user_ids = sqlx::query_scalar::<_, String>(
|
||||
"SELECT user_id FROM user_oauth_links WHERE provider_type = ?",
|
||||
)
|
||||
.bind(provider_type)
|
||||
.fetch_all(&mut **tx)
|
||||
.await
|
||||
.map_sql_err()?;
|
||||
affected_user_ids.extend(user_ids);
|
||||
}
|
||||
Ok(IdentityImportState { affected_user_ids })
|
||||
}
|
||||
|
||||
async fn enforce_mysql_identity_import_invariants(
|
||||
tx: &mut sqlx::Transaction<'_, sqlx::MySql>,
|
||||
scope: &IdentityImportScope,
|
||||
mut state: IdentityImportState,
|
||||
) -> Result<(), DataLayerError> {
|
||||
for user_id in &scope.user_ids {
|
||||
let auth_source =
|
||||
sqlx::query_scalar::<_, String>("SELECT auth_source FROM users WHERE id = ? LIMIT 1")
|
||||
.bind(&user_id)
|
||||
.fetch_optional(&mut **tx)
|
||||
.await
|
||||
.map_sql_err()?
|
||||
.ok_or_else(|| {
|
||||
DataLayerError::InvalidInput(format!(
|
||||
"imported users row '{user_id}' did not produce a user record"
|
||||
))
|
||||
})?;
|
||||
if !matches!(auth_source.as_str(), "local" | "ldap" | "oauth") {
|
||||
return Err(DataLayerError::InvalidInput(format!(
|
||||
"imported user '{user_id}' has unsupported auth_source '{auth_source}'"
|
||||
)));
|
||||
}
|
||||
if auth_source == "oauth" {
|
||||
sqlx::query("UPDATE users SET email_verified = 0 WHERE id = ?")
|
||||
.bind(&user_id)
|
||||
.execute(&mut **tx)
|
||||
.await
|
||||
.map_sql_err()?;
|
||||
}
|
||||
}
|
||||
|
||||
for link_id in &scope.oauth_link_ids {
|
||||
let user_id = sqlx::query_scalar::<_, String>(
|
||||
"SELECT user_id FROM user_oauth_links WHERE id = ? LIMIT 1",
|
||||
)
|
||||
.bind(link_id)
|
||||
.fetch_optional(&mut **tx)
|
||||
.await
|
||||
.map_sql_err()?
|
||||
.ok_or_else(|| {
|
||||
DataLayerError::InvalidInput(format!(
|
||||
"imported OAuth link row '{link_id}' did not produce a link record"
|
||||
))
|
||||
})?;
|
||||
state.affected_user_ids.insert(user_id);
|
||||
}
|
||||
|
||||
for link_id in &scope.oauth_link_ids {
|
||||
if let Some((provider_type, provider_user_id)) = sqlx::query_as::<_, (String, String)>(
|
||||
r#"
|
||||
SELECT imported.provider_type, imported.provider_user_id
|
||||
FROM user_oauth_links imported
|
||||
JOIN user_oauth_links duplicate
|
||||
ON duplicate.provider_type = imported.provider_type
|
||||
AND duplicate.provider_user_id = imported.provider_user_id
|
||||
AND duplicate.id <> imported.id
|
||||
WHERE imported.id = ?
|
||||
LIMIT 1
|
||||
"#,
|
||||
)
|
||||
.bind(link_id)
|
||||
.fetch_optional(&mut **tx)
|
||||
.await
|
||||
.map_sql_err()?
|
||||
{
|
||||
return Err(DataLayerError::InvalidInput(format!(
|
||||
"OAuth import assigns provider identity '{provider_type}:{provider_user_id}' more than once"
|
||||
)));
|
||||
}
|
||||
}
|
||||
|
||||
for link_id in &scope.oauth_link_ids {
|
||||
if let Some((user_id, provider_type)) = sqlx::query_as::<_, (String, String)>(
|
||||
r#"
|
||||
SELECT imported.user_id, imported.provider_type
|
||||
FROM user_oauth_links imported
|
||||
JOIN user_oauth_links duplicate
|
||||
ON duplicate.user_id = imported.user_id
|
||||
AND duplicate.provider_type = imported.provider_type
|
||||
AND duplicate.id <> imported.id
|
||||
WHERE imported.id = ?
|
||||
LIMIT 1
|
||||
"#,
|
||||
)
|
||||
.bind(link_id)
|
||||
.fetch_optional(&mut **tx)
|
||||
.await
|
||||
.map_sql_err()?
|
||||
{
|
||||
return Err(DataLayerError::InvalidInput(format!(
|
||||
"OAuth import links user '{user_id}' to provider '{provider_type}' more than once"
|
||||
)));
|
||||
}
|
||||
}
|
||||
|
||||
for link_id in &scope.oauth_link_ids {
|
||||
if let Some(invalid_id) = sqlx::query_scalar::<_, String>(
|
||||
r#"
|
||||
SELECT links.id
|
||||
FROM user_oauth_links links
|
||||
LEFT JOIN users ON users.id = links.user_id
|
||||
LEFT JOIN oauth_providers providers ON providers.provider_type = links.provider_type
|
||||
WHERE links.id = ?
|
||||
AND (
|
||||
users.id IS NULL
|
||||
OR providers.provider_type IS NULL
|
||||
OR BINARY links.provider_type <> BINARY LOWER(TRIM(links.provider_type))
|
||||
OR links.provider_type = ''
|
||||
OR BINARY links.provider_user_id <> BINARY TRIM(links.provider_user_id)
|
||||
OR links.provider_user_id = ''
|
||||
OR BINARY providers.provider_type <> BINARY LOWER(TRIM(providers.provider_type))
|
||||
)
|
||||
LIMIT 1
|
||||
"#,
|
||||
)
|
||||
.bind(link_id)
|
||||
.fetch_optional(&mut **tx)
|
||||
.await
|
||||
.map_sql_err()?
|
||||
{
|
||||
return Err(DataLayerError::InvalidInput(format!(
|
||||
"OAuth import produced invalid or orphaned link '{invalid_id}'"
|
||||
)));
|
||||
}
|
||||
}
|
||||
|
||||
if !scope.validates_oauth_login_methods {
|
||||
return Ok(());
|
||||
}
|
||||
for user_id in state.affected_user_ids {
|
||||
if sqlx::query_scalar::<_, String>(
|
||||
r#"
|
||||
SELECT users.id
|
||||
FROM users
|
||||
WHERE users.id = ?
|
||||
AND users.auth_source = 'oauth'
|
||||
AND users.is_active = 1
|
||||
AND users.is_deleted = 0
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM user_oauth_links links
|
||||
JOIN oauth_providers providers ON providers.provider_type = links.provider_type
|
||||
WHERE links.user_id = users.id
|
||||
AND providers.is_enabled = 1
|
||||
AND BINARY links.provider_type = BINARY LOWER(TRIM(links.provider_type))
|
||||
AND BINARY links.provider_user_id = BINARY TRIM(links.provider_user_id)
|
||||
AND links.provider_user_id <> ''
|
||||
)
|
||||
LIMIT 1
|
||||
"#,
|
||||
)
|
||||
.bind(&user_id)
|
||||
.fetch_optional(&mut **tx)
|
||||
.await
|
||||
.map_sql_err()?
|
||||
.is_some()
|
||||
{
|
||||
return Err(DataLayerError::InvalidInput(format!(
|
||||
"OAuth import would leave active user '{user_id}' without an enabled identity binding"
|
||||
)));
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn mysql_domain_table(
|
||||
domain: ExportDomain,
|
||||
) -> Result<(&'static str, &'static str), DataLayerError> {
|
||||
@@ -262,7 +464,11 @@ async fn import_mysql_row(
|
||||
row: &ExportRow,
|
||||
target_columns: &MysqlImportColumns,
|
||||
) -> Result<(), DataLayerError> {
|
||||
let object = filter_import_payload("mysql", table_name, domain, row, &target_columns.names)?;
|
||||
let mut object =
|
||||
filter_import_payload("mysql", table_name, domain, row, &target_columns.names)?;
|
||||
deactivate_imported_credentials(table_name, &mut object, |column_name| {
|
||||
target_columns.names.contains(column_name)
|
||||
});
|
||||
|
||||
let columns = object.keys().map(String::as_str).collect::<Vec<_>>();
|
||||
for primary_key in &target_columns.primary_key {
|
||||
|
||||
@@ -67,7 +67,9 @@ pub async fn import_postgres_plan(
|
||||
pool: &crate::driver::postgres::PostgresPool,
|
||||
plan: &DataImportPlan,
|
||||
) -> Result<usize, DataLayerError> {
|
||||
let identity_scope = IdentityImportScope::from_plan(plan)?;
|
||||
let mut tx = pool.begin().await.map_sql_err()?;
|
||||
let identity_state = capture_postgres_identity_import_state(&mut tx, &identity_scope).await?;
|
||||
let mut imported = 0usize;
|
||||
let mut column_cache = BTreeMap::<String, PostgresImportColumns>::new();
|
||||
for domain in &plan.manifest.domains {
|
||||
@@ -113,6 +115,7 @@ pub async fn import_postgres_plan(
|
||||
imported = imported.saturating_add(1);
|
||||
}
|
||||
}
|
||||
enforce_postgres_identity_import_invariants(&mut tx, &identity_scope, identity_state).await?;
|
||||
if !plan.rows(ExportDomain::Auxiliary).is_empty() {
|
||||
reset_postgres_auxiliary_sequences(&mut tx).await?;
|
||||
}
|
||||
@@ -120,6 +123,207 @@ pub async fn import_postgres_plan(
|
||||
Ok(imported)
|
||||
}
|
||||
|
||||
async fn capture_postgres_identity_import_state(
|
||||
tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
|
||||
scope: &IdentityImportScope,
|
||||
) -> Result<IdentityImportState, DataLayerError> {
|
||||
let mut affected_user_ids = if scope.finalizes_oauth_links {
|
||||
scope.user_ids.iter().cloned().collect::<BTreeSet<_>>()
|
||||
} else {
|
||||
BTreeSet::new()
|
||||
};
|
||||
for link_id in &scope.oauth_link_ids {
|
||||
if let Some(user_id) = sqlx::query_scalar::<_, String>(
|
||||
"SELECT user_id FROM public.user_oauth_links WHERE id = $1",
|
||||
)
|
||||
.bind(link_id)
|
||||
.fetch_optional(&mut **tx)
|
||||
.await
|
||||
.map_sql_err()?
|
||||
{
|
||||
affected_user_ids.insert(user_id);
|
||||
}
|
||||
}
|
||||
for provider_type in &scope.oauth_provider_types {
|
||||
let user_ids = sqlx::query_scalar::<_, String>(
|
||||
"SELECT user_id FROM public.user_oauth_links WHERE provider_type = $1",
|
||||
)
|
||||
.bind(provider_type)
|
||||
.fetch_all(&mut **tx)
|
||||
.await
|
||||
.map_sql_err()?;
|
||||
affected_user_ids.extend(user_ids);
|
||||
}
|
||||
Ok(IdentityImportState { affected_user_ids })
|
||||
}
|
||||
|
||||
async fn enforce_postgres_identity_import_invariants(
|
||||
tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
|
||||
scope: &IdentityImportScope,
|
||||
mut state: IdentityImportState,
|
||||
) -> Result<(), DataLayerError> {
|
||||
for user_id in &scope.user_ids {
|
||||
let auth_source = sqlx::query_scalar::<_, String>(
|
||||
"SELECT auth_source::text FROM public.users WHERE id = $1 LIMIT 1",
|
||||
)
|
||||
.bind(&user_id)
|
||||
.fetch_optional(&mut **tx)
|
||||
.await
|
||||
.map_sql_err()?
|
||||
.ok_or_else(|| {
|
||||
DataLayerError::InvalidInput(format!(
|
||||
"imported users row '{user_id}' did not produce a user record"
|
||||
))
|
||||
})?;
|
||||
if !matches!(auth_source.as_str(), "local" | "ldap" | "oauth") {
|
||||
return Err(DataLayerError::InvalidInput(format!(
|
||||
"imported user '{user_id}' has unsupported auth_source '{auth_source}'"
|
||||
)));
|
||||
}
|
||||
if auth_source == "oauth" {
|
||||
sqlx::query("UPDATE public.users SET email_verified = FALSE WHERE id = $1")
|
||||
.bind(&user_id)
|
||||
.execute(&mut **tx)
|
||||
.await
|
||||
.map_sql_err()?;
|
||||
}
|
||||
}
|
||||
|
||||
for link_id in &scope.oauth_link_ids {
|
||||
let user_id = sqlx::query_scalar::<_, String>(
|
||||
"SELECT user_id FROM public.user_oauth_links WHERE id = $1 LIMIT 1",
|
||||
)
|
||||
.bind(link_id)
|
||||
.fetch_optional(&mut **tx)
|
||||
.await
|
||||
.map_sql_err()?
|
||||
.ok_or_else(|| {
|
||||
DataLayerError::InvalidInput(format!(
|
||||
"imported OAuth link row '{link_id}' did not produce a link record"
|
||||
))
|
||||
})?;
|
||||
state.affected_user_ids.insert(user_id);
|
||||
}
|
||||
|
||||
for link_id in &scope.oauth_link_ids {
|
||||
if let Some((provider_type, provider_user_id)) = sqlx::query_as::<_, (String, String)>(
|
||||
r#"
|
||||
SELECT imported.provider_type, imported.provider_user_id
|
||||
FROM public.user_oauth_links imported
|
||||
JOIN public.user_oauth_links duplicate
|
||||
ON duplicate.provider_type = imported.provider_type
|
||||
AND duplicate.provider_user_id = imported.provider_user_id
|
||||
AND duplicate.id <> imported.id
|
||||
WHERE imported.id = $1
|
||||
LIMIT 1
|
||||
"#,
|
||||
)
|
||||
.bind(link_id)
|
||||
.fetch_optional(&mut **tx)
|
||||
.await
|
||||
.map_sql_err()?
|
||||
{
|
||||
return Err(DataLayerError::InvalidInput(format!(
|
||||
"OAuth import assigns provider identity '{provider_type}:{provider_user_id}' more than once"
|
||||
)));
|
||||
}
|
||||
}
|
||||
|
||||
for link_id in &scope.oauth_link_ids {
|
||||
if let Some((user_id, provider_type)) = sqlx::query_as::<_, (String, String)>(
|
||||
r#"
|
||||
SELECT imported.user_id, imported.provider_type
|
||||
FROM public.user_oauth_links imported
|
||||
JOIN public.user_oauth_links duplicate
|
||||
ON duplicate.user_id = imported.user_id
|
||||
AND duplicate.provider_type = imported.provider_type
|
||||
AND duplicate.id <> imported.id
|
||||
WHERE imported.id = $1
|
||||
LIMIT 1
|
||||
"#,
|
||||
)
|
||||
.bind(link_id)
|
||||
.fetch_optional(&mut **tx)
|
||||
.await
|
||||
.map_sql_err()?
|
||||
{
|
||||
return Err(DataLayerError::InvalidInput(format!(
|
||||
"OAuth import links user '{user_id}' to provider '{provider_type}' more than once"
|
||||
)));
|
||||
}
|
||||
}
|
||||
|
||||
for link_id in &scope.oauth_link_ids {
|
||||
if let Some(invalid_id) = sqlx::query_scalar::<_, String>(
|
||||
r#"
|
||||
SELECT links.id
|
||||
FROM public.user_oauth_links links
|
||||
LEFT JOIN public.users users ON users.id = links.user_id
|
||||
LEFT JOIN public.oauth_providers providers ON providers.provider_type = links.provider_type
|
||||
WHERE links.id = $1
|
||||
AND (
|
||||
users.id IS NULL
|
||||
OR providers.provider_type IS NULL
|
||||
OR links.provider_type <> LOWER(TRIM(links.provider_type))
|
||||
OR links.provider_type = ''
|
||||
OR links.provider_user_id <> TRIM(links.provider_user_id)
|
||||
OR links.provider_user_id = ''
|
||||
OR providers.provider_type <> LOWER(TRIM(providers.provider_type))
|
||||
)
|
||||
LIMIT 1
|
||||
"#,
|
||||
)
|
||||
.bind(link_id)
|
||||
.fetch_optional(&mut **tx)
|
||||
.await
|
||||
.map_sql_err()?
|
||||
{
|
||||
return Err(DataLayerError::InvalidInput(format!(
|
||||
"OAuth import produced invalid or orphaned link '{invalid_id}'"
|
||||
)));
|
||||
}
|
||||
}
|
||||
|
||||
if !scope.validates_oauth_login_methods {
|
||||
return Ok(());
|
||||
}
|
||||
for user_id in state.affected_user_ids {
|
||||
if sqlx::query_scalar::<_, String>(
|
||||
r#"
|
||||
SELECT users.id
|
||||
FROM public.users users
|
||||
WHERE users.id = $1
|
||||
AND users.auth_source = 'oauth'::public.authsource
|
||||
AND users.is_active IS TRUE
|
||||
AND users.is_deleted IS FALSE
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM public.user_oauth_links links
|
||||
JOIN public.oauth_providers providers ON providers.provider_type = links.provider_type
|
||||
WHERE links.user_id = users.id
|
||||
AND providers.is_enabled IS TRUE
|
||||
AND links.provider_type = LOWER(TRIM(links.provider_type))
|
||||
AND links.provider_user_id = TRIM(links.provider_user_id)
|
||||
AND links.provider_user_id <> ''
|
||||
)
|
||||
LIMIT 1
|
||||
"#,
|
||||
)
|
||||
.bind(&user_id)
|
||||
.fetch_optional(&mut **tx)
|
||||
.await
|
||||
.map_sql_err()?
|
||||
.is_some()
|
||||
{
|
||||
return Err(DataLayerError::InvalidInput(format!(
|
||||
"OAuth import would leave active user '{user_id}' without an enabled identity binding"
|
||||
)));
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn reset_postgres_auxiliary_sequences(
|
||||
tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
|
||||
) -> Result<(), DataLayerError> {
|
||||
@@ -359,7 +563,13 @@ async fn export_postgres_wallet_records(
|
||||
let rows = sqlx::query(&sql).fetch_all(&mut **tx).await.map_sql_err()?;
|
||||
for row in rows {
|
||||
let id = row.try_get::<String, _>("export_id").map_sql_err()?;
|
||||
let payload = row.try_get::<Value, _>("payload").map_sql_err()?;
|
||||
let mut payload = row.try_get::<Value, _>("payload").map_sql_err()?;
|
||||
let object = payload.as_object_mut().ok_or_else(|| {
|
||||
DataLayerError::UnexpectedValue(format!(
|
||||
"wallet export row in table '{export_table}' must be an object"
|
||||
))
|
||||
})?;
|
||||
sanitize_payment_security_payload(export_table, object);
|
||||
records.push(DataExportRecord::row(
|
||||
ExportDomain::Wallets,
|
||||
format!("{export_table}:{id}"),
|
||||
@@ -378,7 +588,10 @@ async fn import_postgres_row(
|
||||
row: &ExportRow,
|
||||
target_columns: &PostgresImportColumns,
|
||||
) -> Result<(), DataLayerError> {
|
||||
let object = normalize_postgres_import_payload(table_name, domain, row, target_columns)?;
|
||||
let mut object = normalize_postgres_import_payload(table_name, domain, row, target_columns)?;
|
||||
deactivate_imported_credentials(table_name, &mut object, |column_name| {
|
||||
target_columns.contains_key(column_name)
|
||||
});
|
||||
|
||||
let columns = object.keys().map(String::as_str).collect::<Vec<_>>();
|
||||
let column_sql = columns
|
||||
|
||||
@@ -66,7 +66,9 @@ pub async fn import_sqlite_plan(
|
||||
pool: &crate::driver::sqlite::SqlitePool,
|
||||
plan: &DataImportPlan,
|
||||
) -> Result<usize, DataLayerError> {
|
||||
let identity_scope = IdentityImportScope::from_plan(plan)?;
|
||||
let mut tx = pool.begin().await.map_sql_err()?;
|
||||
let identity_state = capture_sqlite_identity_import_state(&mut tx, &identity_scope).await?;
|
||||
let mut imported = 0usize;
|
||||
let mut column_cache = BTreeMap::<String, SqliteImportColumns>::new();
|
||||
for domain in &plan.manifest.domains {
|
||||
@@ -99,10 +101,210 @@ pub async fn import_sqlite_plan(
|
||||
imported = imported.saturating_add(1);
|
||||
}
|
||||
}
|
||||
enforce_sqlite_identity_import_invariants(&mut tx, &identity_scope, identity_state).await?;
|
||||
tx.commit().await.map_sql_err()?;
|
||||
Ok(imported)
|
||||
}
|
||||
|
||||
async fn capture_sqlite_identity_import_state(
|
||||
tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>,
|
||||
scope: &IdentityImportScope,
|
||||
) -> Result<IdentityImportState, DataLayerError> {
|
||||
let mut affected_user_ids = if scope.finalizes_oauth_links {
|
||||
scope.user_ids.iter().cloned().collect::<BTreeSet<_>>()
|
||||
} else {
|
||||
BTreeSet::new()
|
||||
};
|
||||
for link_id in &scope.oauth_link_ids {
|
||||
if let Some(user_id) =
|
||||
sqlx::query_scalar::<_, String>("SELECT user_id FROM user_oauth_links WHERE id = ?")
|
||||
.bind(link_id)
|
||||
.fetch_optional(&mut **tx)
|
||||
.await
|
||||
.map_sql_err()?
|
||||
{
|
||||
affected_user_ids.insert(user_id);
|
||||
}
|
||||
}
|
||||
for provider_type in &scope.oauth_provider_types {
|
||||
let user_ids = sqlx::query_scalar::<_, String>(
|
||||
"SELECT user_id FROM user_oauth_links WHERE provider_type = ?",
|
||||
)
|
||||
.bind(provider_type)
|
||||
.fetch_all(&mut **tx)
|
||||
.await
|
||||
.map_sql_err()?;
|
||||
affected_user_ids.extend(user_ids);
|
||||
}
|
||||
Ok(IdentityImportState { affected_user_ids })
|
||||
}
|
||||
|
||||
async fn enforce_sqlite_identity_import_invariants(
|
||||
tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>,
|
||||
scope: &IdentityImportScope,
|
||||
mut state: IdentityImportState,
|
||||
) -> Result<(), DataLayerError> {
|
||||
for user_id in &scope.user_ids {
|
||||
let auth_source =
|
||||
sqlx::query_scalar::<_, String>("SELECT auth_source FROM users WHERE id = ? LIMIT 1")
|
||||
.bind(&user_id)
|
||||
.fetch_optional(&mut **tx)
|
||||
.await
|
||||
.map_sql_err()?
|
||||
.ok_or_else(|| {
|
||||
DataLayerError::InvalidInput(format!(
|
||||
"imported users row '{user_id}' did not produce a user record"
|
||||
))
|
||||
})?;
|
||||
if !matches!(auth_source.as_str(), "local" | "ldap" | "oauth") {
|
||||
return Err(DataLayerError::InvalidInput(format!(
|
||||
"imported user '{user_id}' has unsupported auth_source '{auth_source}'"
|
||||
)));
|
||||
}
|
||||
if auth_source == "oauth" {
|
||||
sqlx::query("UPDATE users SET email_verified = 0 WHERE id = ?")
|
||||
.bind(&user_id)
|
||||
.execute(&mut **tx)
|
||||
.await
|
||||
.map_sql_err()?;
|
||||
}
|
||||
}
|
||||
|
||||
for link_id in &scope.oauth_link_ids {
|
||||
let user_id = sqlx::query_scalar::<_, String>(
|
||||
"SELECT user_id FROM user_oauth_links WHERE id = ? LIMIT 1",
|
||||
)
|
||||
.bind(link_id)
|
||||
.fetch_optional(&mut **tx)
|
||||
.await
|
||||
.map_sql_err()?
|
||||
.ok_or_else(|| {
|
||||
DataLayerError::InvalidInput(format!(
|
||||
"imported OAuth link row '{link_id}' did not produce a link record"
|
||||
))
|
||||
})?;
|
||||
state.affected_user_ids.insert(user_id);
|
||||
}
|
||||
|
||||
for link_id in &scope.oauth_link_ids {
|
||||
if let Some((provider_type, provider_user_id)) = sqlx::query_as::<_, (String, String)>(
|
||||
r#"
|
||||
SELECT imported.provider_type, imported.provider_user_id
|
||||
FROM user_oauth_links imported
|
||||
JOIN user_oauth_links duplicate
|
||||
ON duplicate.provider_type = imported.provider_type
|
||||
AND duplicate.provider_user_id = imported.provider_user_id
|
||||
AND duplicate.id <> imported.id
|
||||
WHERE imported.id = ?
|
||||
LIMIT 1
|
||||
"#,
|
||||
)
|
||||
.bind(link_id)
|
||||
.fetch_optional(&mut **tx)
|
||||
.await
|
||||
.map_sql_err()?
|
||||
{
|
||||
return Err(DataLayerError::InvalidInput(format!(
|
||||
"OAuth import assigns provider identity '{provider_type}:{provider_user_id}' more than once"
|
||||
)));
|
||||
}
|
||||
}
|
||||
|
||||
for link_id in &scope.oauth_link_ids {
|
||||
if let Some((user_id, provider_type)) = sqlx::query_as::<_, (String, String)>(
|
||||
r#"
|
||||
SELECT imported.user_id, imported.provider_type
|
||||
FROM user_oauth_links imported
|
||||
JOIN user_oauth_links duplicate
|
||||
ON duplicate.user_id = imported.user_id
|
||||
AND duplicate.provider_type = imported.provider_type
|
||||
AND duplicate.id <> imported.id
|
||||
WHERE imported.id = ?
|
||||
LIMIT 1
|
||||
"#,
|
||||
)
|
||||
.bind(link_id)
|
||||
.fetch_optional(&mut **tx)
|
||||
.await
|
||||
.map_sql_err()?
|
||||
{
|
||||
return Err(DataLayerError::InvalidInput(format!(
|
||||
"OAuth import links user '{user_id}' to provider '{provider_type}' more than once"
|
||||
)));
|
||||
}
|
||||
}
|
||||
|
||||
for link_id in &scope.oauth_link_ids {
|
||||
if let Some(invalid_id) = sqlx::query_scalar::<_, String>(
|
||||
r#"
|
||||
SELECT links.id
|
||||
FROM user_oauth_links links
|
||||
LEFT JOIN users ON users.id = links.user_id
|
||||
LEFT JOIN oauth_providers providers ON providers.provider_type = links.provider_type
|
||||
WHERE links.id = ?
|
||||
AND (
|
||||
users.id IS NULL
|
||||
OR providers.provider_type IS NULL
|
||||
OR links.provider_type <> LOWER(TRIM(links.provider_type))
|
||||
OR links.provider_type = ''
|
||||
OR links.provider_user_id <> TRIM(links.provider_user_id)
|
||||
OR links.provider_user_id = ''
|
||||
OR providers.provider_type <> LOWER(TRIM(providers.provider_type))
|
||||
)
|
||||
LIMIT 1
|
||||
"#,
|
||||
)
|
||||
.bind(link_id)
|
||||
.fetch_optional(&mut **tx)
|
||||
.await
|
||||
.map_sql_err()?
|
||||
{
|
||||
return Err(DataLayerError::InvalidInput(format!(
|
||||
"OAuth import produced invalid or orphaned link '{invalid_id}'"
|
||||
)));
|
||||
}
|
||||
}
|
||||
|
||||
if !scope.validates_oauth_login_methods {
|
||||
return Ok(());
|
||||
}
|
||||
for user_id in state.affected_user_ids {
|
||||
if sqlx::query_scalar::<_, String>(
|
||||
r#"
|
||||
SELECT users.id
|
||||
FROM users
|
||||
WHERE users.id = ?
|
||||
AND users.auth_source = 'oauth'
|
||||
AND users.is_active = 1
|
||||
AND users.is_deleted = 0
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM user_oauth_links links
|
||||
JOIN oauth_providers providers ON providers.provider_type = links.provider_type
|
||||
WHERE links.user_id = users.id
|
||||
AND providers.is_enabled = 1
|
||||
AND links.provider_type = LOWER(TRIM(links.provider_type))
|
||||
AND links.provider_user_id = TRIM(links.provider_user_id)
|
||||
AND links.provider_user_id <> ''
|
||||
)
|
||||
LIMIT 1
|
||||
"#,
|
||||
)
|
||||
.bind(&user_id)
|
||||
.fetch_optional(&mut **tx)
|
||||
.await
|
||||
.map_sql_err()?
|
||||
.is_some()
|
||||
{
|
||||
return Err(DataLayerError::InvalidInput(format!(
|
||||
"OAuth import would leave active user '{user_id}' without an enabled identity binding"
|
||||
)));
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn sqlite_domain_table(
|
||||
domain: ExportDomain,
|
||||
) -> Result<(&'static str, &'static str), DataLayerError> {
|
||||
@@ -262,7 +464,11 @@ async fn import_sqlite_row(
|
||||
row: &ExportRow,
|
||||
target_columns: &SqliteImportColumns,
|
||||
) -> Result<(), DataLayerError> {
|
||||
let object = filter_import_payload("sqlite", table_name, domain, row, &target_columns.names)?;
|
||||
let mut object =
|
||||
filter_import_payload("sqlite", table_name, domain, row, &target_columns.names)?;
|
||||
deactivate_imported_credentials(table_name, &mut object, |column_name| {
|
||||
target_columns.names.contains(column_name)
|
||||
});
|
||||
|
||||
let columns = object.keys().map(String::as_str).collect::<Vec<_>>();
|
||||
let column_sql = columns
|
||||
|
||||
@@ -1,16 +1,17 @@
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
|
||||
use serde_json::json;
|
||||
use serde_json::{json, Value};
|
||||
|
||||
use super::{
|
||||
build_import_plan, decode_jsonl, encode_jsonl, export_mysql_core_jsonl, export_mysql_jsonl,
|
||||
export_postgres_core_jsonl, export_sqlite_core_jsonl, filter_import_payload,
|
||||
import_mysql_jsonl, import_postgres_jsonl, import_sqlite_jsonl, mysql_core_export_domains,
|
||||
normalize_imported_binary, normalize_imported_integer_timestamp,
|
||||
normalize_postgres_import_payload, postgres_bytea_json_value, postgres_core_export_domains,
|
||||
sqlite_core_export_domains, sqlite_schema_copy_insert_sql, DataExportManifest,
|
||||
DataExportRecord, DataImportPlan, ExportDomain, ExportRow, PostgresImportColumn,
|
||||
SchemaCopyColumn, SchemaCopyTable, SqliteCopyColumn, AUXILIARY_TABLES,
|
||||
build_import_plan, deactivate_imported_credentials, decode_jsonl, decode_jsonl_with_limits,
|
||||
encode_jsonl, export_mysql_core_jsonl, export_mysql_jsonl, export_postgres_core_jsonl,
|
||||
export_sqlite_core_jsonl, filter_import_payload, import_mysql_jsonl, import_postgres_jsonl,
|
||||
import_sqlite_jsonl, mysql_core_export_domains, normalize_imported_binary,
|
||||
normalize_imported_integer_timestamp, normalize_postgres_import_payload,
|
||||
postgres_bytea_json_value, postgres_core_export_domains, sqlite_core_export_domains,
|
||||
sqlite_schema_copy_insert_sql, DataExportManifest, DataExportRecord, DataImportPlan,
|
||||
ExportDomain, ExportRow, PostgresImportColumn, SchemaCopyColumn, SchemaCopyTable,
|
||||
SqliteCopyColumn, AUXILIARY_TABLES,
|
||||
};
|
||||
use crate::driver::postgres::{PostgresPoolConfig, PostgresPoolFactory};
|
||||
use crate::lifecycle::migrate::{
|
||||
@@ -181,6 +182,25 @@ not-json"#,
|
||||
assert!(err.to_string().contains("line 2"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn jsonl_rejects_input_and_record_limits_before_materializing_rows() {
|
||||
let oversized = "x".repeat(11);
|
||||
let err = decode_jsonl_with_limits(&oversized, 10, 100, 10)
|
||||
.expect_err("input byte limit should be enforced");
|
||||
assert!(err.to_string().contains("10 byte input limit"));
|
||||
|
||||
let manifest = r#"{"record_type":"manifest","manifest":{"format_version":1,"created_at_unix_secs":1,"source_driver":null,"domains":[]}}"#;
|
||||
let err = decode_jsonl_with_limits(manifest, usize::MAX, 10, 10)
|
||||
.expect_err("line byte limit should be enforced");
|
||||
assert!(err.to_string().contains("byte line limit"));
|
||||
|
||||
let row = r#"{"record_type":"manifest","manifest":{"format_version":1,"created_at_unix_secs":1,"source_driver":null,"domains":[]}}"#;
|
||||
let input = format!("{row}\n{row}\n");
|
||||
let err = decode_jsonl_with_limits(&input, usize::MAX, usize::MAX, 1)
|
||||
.expect_err("record limit should be enforced");
|
||||
assert!(err.to_string().contains("1 record limit"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn jsonl_rejects_duplicate_domain_ids() {
|
||||
let records = vec![
|
||||
@@ -409,6 +429,231 @@ fn mysql_and_sqlite_import_payloads_ignore_unknown_null_columns() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn imported_identity_credentials_are_replaced_with_disabled_tombstones() {
|
||||
let columns = BTreeSet::from([
|
||||
"password_hash".to_string(),
|
||||
"key_hash".to_string(),
|
||||
"key_encrypted".to_string(),
|
||||
"status".to_string(),
|
||||
"is_active".to_string(),
|
||||
"is_locked".to_string(),
|
||||
"token_hash".to_string(),
|
||||
"refresh_token_hash".to_string(),
|
||||
"prev_refresh_token_hash".to_string(),
|
||||
"revoked_at".to_string(),
|
||||
"revoke_reason".to_string(),
|
||||
]);
|
||||
|
||||
let mut user =
|
||||
serde_json::Map::from_iter([("password_hash".to_string(), json!("$2b$12$backup-hash"))]);
|
||||
deactivate_imported_credentials("users", &mut user, |column| columns.contains(column));
|
||||
assert_ne!(user["password_hash"], json!("$2b$12$backup-hash"));
|
||||
assert!(user["password_hash"]
|
||||
.as_str()
|
||||
.is_some_and(|value| value.starts_with("$aether-import-revoked$")));
|
||||
|
||||
let mut api_key = serde_json::Map::from_iter([
|
||||
("key_hash".to_string(), json!("backup-key-hash")),
|
||||
("key_encrypted".to_string(), json!("backup-ciphertext")),
|
||||
("is_active".to_string(), json!(true)),
|
||||
("is_locked".to_string(), json!(false)),
|
||||
("status".to_string(), json!("active")),
|
||||
]);
|
||||
deactivate_imported_credentials("api_keys", &mut api_key, |column| columns.contains(column));
|
||||
assert_ne!(api_key["key_hash"], json!("backup-key-hash"));
|
||||
assert_eq!(api_key["key_encrypted"], Value::Null);
|
||||
assert_eq!(api_key["is_active"], json!(false));
|
||||
assert_eq!(api_key["is_locked"], json!(true));
|
||||
assert_eq!(api_key["status"], json!("disabled"));
|
||||
|
||||
let mut token = serde_json::Map::from_iter([
|
||||
("token_hash".to_string(), json!("backup-token-hash")),
|
||||
("is_active".to_string(), json!(true)),
|
||||
]);
|
||||
deactivate_imported_credentials("management_tokens", &mut token, |column| {
|
||||
columns.contains(column)
|
||||
});
|
||||
assert_ne!(token["token_hash"], json!("backup-token-hash"));
|
||||
assert_eq!(token["is_active"], json!(false));
|
||||
|
||||
let mut session = serde_json::Map::from_iter([
|
||||
(
|
||||
"refresh_token_hash".to_string(),
|
||||
json!("backup-refresh-hash"),
|
||||
),
|
||||
(
|
||||
"prev_refresh_token_hash".to_string(),
|
||||
json!("backup-previous-hash"),
|
||||
),
|
||||
("revoked_at".to_string(), Value::Null),
|
||||
("revoke_reason".to_string(), Value::Null),
|
||||
]);
|
||||
deactivate_imported_credentials("user_sessions", &mut session, |column| {
|
||||
columns.contains(column)
|
||||
});
|
||||
assert_ne!(session["refresh_token_hash"], json!("backup-refresh-hash"));
|
||||
assert_eq!(session["prev_refresh_token_hash"], Value::Null);
|
||||
assert!(session["revoked_at"].as_i64().is_some());
|
||||
assert_eq!(
|
||||
session["revoke_reason"],
|
||||
json!("imported_credentials_revoked")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn imported_proxy_nodes_receive_a_new_offline_tunnel_generation() {
|
||||
let columns = BTreeSet::from([
|
||||
"tunnel_generation".to_string(),
|
||||
"tunnel_connected".to_string(),
|
||||
"status".to_string(),
|
||||
"active_connections".to_string(),
|
||||
]);
|
||||
let mut node = serde_json::Map::from_iter([
|
||||
(
|
||||
"tunnel_generation".to_string(),
|
||||
json!("backup-tunnel-generation"),
|
||||
),
|
||||
("tunnel_connected".to_string(), json!(true)),
|
||||
("status".to_string(), json!("online")),
|
||||
("active_connections".to_string(), json!(42)),
|
||||
(
|
||||
"proxy_metadata".to_string(),
|
||||
json!({"tunnel_security": {"encryption_key": "preserved-psk"}}),
|
||||
),
|
||||
]);
|
||||
|
||||
deactivate_imported_credentials("public.proxy_nodes", &mut node, |column| {
|
||||
columns.contains(column)
|
||||
});
|
||||
|
||||
let generation = node["tunnel_generation"]
|
||||
.as_str()
|
||||
.expect("imported node generation should be a string");
|
||||
assert_ne!(generation, "backup-tunnel-generation");
|
||||
assert!(uuid::Uuid::parse_str(generation).is_ok());
|
||||
assert_eq!(node["tunnel_connected"], json!(false));
|
||||
assert_eq!(node["status"], json!("offline"));
|
||||
assert_eq!(node["active_connections"], json!(0));
|
||||
assert_eq!(
|
||||
node["proxy_metadata"]["tunnel_security"]["encryption_key"],
|
||||
json!("preserved-psk")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn sqlite_import_rotates_proxy_node_generations_and_clears_online_state() {
|
||||
let pool = sqlx::sqlite::SqlitePoolOptions::new()
|
||||
.max_connections(1)
|
||||
.connect("sqlite::memory:")
|
||||
.await
|
||||
.expect("sqlite pool should connect");
|
||||
run_sqlite_migrations(&pool)
|
||||
.await
|
||||
.expect("sqlite migrations should run");
|
||||
sqlx::query(
|
||||
r#"
|
||||
INSERT INTO proxy_nodes (
|
||||
id, tunnel_generation, name, ip, port, status, active_connections,
|
||||
tunnel_mode, tunnel_connected, proxy_metadata, created_at, updated_at
|
||||
) VALUES (
|
||||
'import-existing-node', 'target-live-generation', 'existing node', '127.0.0.1',
|
||||
8080, 'online', 9, 1, 1, '{"target":"metadata"}', 1, 1
|
||||
)
|
||||
"#,
|
||||
)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.expect("existing proxy node should insert");
|
||||
|
||||
let encoded = encode_jsonl(&[
|
||||
DataExportRecord::manifest(DataExportManifest::new(
|
||||
1_700_000_000,
|
||||
Some(DatabaseDriver::Postgres),
|
||||
vec![ExportDomain::ProxyNodes],
|
||||
)),
|
||||
DataExportRecord::row(
|
||||
ExportDomain::ProxyNodes,
|
||||
"import-existing-node",
|
||||
json!({
|
||||
"id": "import-existing-node",
|
||||
"tunnel_generation": "backup-stale-generation",
|
||||
"name": "restored existing node",
|
||||
"ip": "127.0.0.1",
|
||||
"port": 8080,
|
||||
"status": "online",
|
||||
"active_connections": 42,
|
||||
"tunnel_mode": true,
|
||||
"tunnel_connected": true,
|
||||
"proxy_metadata": {
|
||||
"tunnel_security": {"encryption_key": "preserved-psk"}
|
||||
},
|
||||
"created_at": 1,
|
||||
"updated_at": 2
|
||||
}),
|
||||
),
|
||||
DataExportRecord::row(
|
||||
ExportDomain::ProxyNodes,
|
||||
"import-legacy-node",
|
||||
json!({
|
||||
"id": "import-legacy-node",
|
||||
"name": "legacy backup node",
|
||||
"ip": "127.0.0.2",
|
||||
"port": 8081,
|
||||
"status": "online",
|
||||
"active_connections": 7,
|
||||
"tunnel_mode": true,
|
||||
"tunnel_connected": true,
|
||||
"created_at": 1,
|
||||
"updated_at": 2
|
||||
}),
|
||||
),
|
||||
])
|
||||
.expect("proxy node import fixture should encode");
|
||||
|
||||
assert_eq!(
|
||||
import_sqlite_jsonl(&pool, &encoded)
|
||||
.await
|
||||
.expect("proxy nodes should import"),
|
||||
2
|
||||
);
|
||||
|
||||
let restored = sqlx::query_as::<_, (String, String, bool, i32, Option<String>)>(
|
||||
r#"
|
||||
SELECT tunnel_generation, status, tunnel_connected, active_connections, proxy_metadata
|
||||
FROM proxy_nodes
|
||||
WHERE id = 'import-existing-node'
|
||||
"#,
|
||||
)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.expect("restored proxy node should load");
|
||||
assert_ne!(restored.0, "target-live-generation");
|
||||
assert_ne!(restored.0, "backup-stale-generation");
|
||||
assert!(uuid::Uuid::parse_str(&restored.0).is_ok());
|
||||
assert_eq!(restored.1, "offline");
|
||||
assert!(!restored.2);
|
||||
assert_eq!(restored.3, 0);
|
||||
assert_eq!(
|
||||
restored
|
||||
.4
|
||||
.as_deref()
|
||||
.and_then(|value| serde_json::from_str::<Value>(value).ok())
|
||||
.and_then(|value| value["tunnel_security"]["encryption_key"]
|
||||
.as_str()
|
||||
.map(str::to_string)),
|
||||
Some("preserved-psk".to_string())
|
||||
);
|
||||
|
||||
let legacy_generation: String = sqlx::query_scalar(
|
||||
"SELECT tunnel_generation FROM proxy_nodes WHERE id = 'import-legacy-node'",
|
||||
)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.expect("legacy imported proxy node should load");
|
||||
assert!(uuid::Uuid::parse_str(&legacy_generation).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn postgres_to_sqlite_copy_uses_primary_key_upsert_instead_of_replace() {
|
||||
let table = SchemaCopyTable {
|
||||
@@ -639,6 +884,548 @@ async fn sqlite_import_rolls_back_rows_after_late_failure() {
|
||||
assert_eq!(count, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn sqlite_users_import_fails_closed_for_oauth_email_verification() {
|
||||
let pool = sqlx::sqlite::SqlitePoolOptions::new()
|
||||
.max_connections(1)
|
||||
.connect("sqlite::memory:")
|
||||
.await
|
||||
.expect("sqlite pool should connect");
|
||||
run_sqlite_migrations(&pool)
|
||||
.await
|
||||
.expect("sqlite migrations should run");
|
||||
|
||||
let encoded = encode_jsonl(&[
|
||||
DataExportRecord::manifest(DataExportManifest::new(
|
||||
1_700_000_000,
|
||||
Some(DatabaseDriver::Postgres),
|
||||
vec![ExportDomain::Users],
|
||||
)),
|
||||
DataExportRecord::row(
|
||||
ExportDomain::Users,
|
||||
"oauth-user",
|
||||
json!({
|
||||
"id": "oauth-user",
|
||||
"email": "[email protected]",
|
||||
"email_verified": true,
|
||||
"username": "oauth-user",
|
||||
"role": "user",
|
||||
"auth_source": "oauth",
|
||||
"created_at": 1,
|
||||
"updated_at": 1
|
||||
}),
|
||||
),
|
||||
DataExportRecord::row(
|
||||
ExportDomain::Users,
|
||||
"local-user",
|
||||
json!({
|
||||
"id": "local-user",
|
||||
"email": "[email protected]",
|
||||
"email_verified": true,
|
||||
"username": "local-user",
|
||||
"role": "user",
|
||||
"auth_source": "local",
|
||||
"created_at": 1,
|
||||
"updated_at": 1
|
||||
}),
|
||||
),
|
||||
])
|
||||
.expect("users fixture should encode");
|
||||
|
||||
assert_eq!(
|
||||
import_sqlite_jsonl(&pool, &encoded)
|
||||
.await
|
||||
.expect("users-only staged restore should succeed without OAuth links"),
|
||||
2
|
||||
);
|
||||
let verification =
|
||||
sqlx::query_as::<_, (String, i64)>("SELECT id, email_verified FROM users ORDER BY id ASC")
|
||||
.fetch_all(&pool)
|
||||
.await
|
||||
.expect("verification state should load");
|
||||
assert_eq!(
|
||||
verification,
|
||||
vec![("local-user".to_string(), 1), ("oauth-user".to_string(), 0)]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn sqlite_users_and_providers_can_restore_before_oauth_links() {
|
||||
let pool = sqlx::sqlite::SqlitePoolOptions::new()
|
||||
.max_connections(1)
|
||||
.connect("sqlite::memory:")
|
||||
.await
|
||||
.expect("sqlite pool should connect");
|
||||
run_sqlite_migrations(&pool)
|
||||
.await
|
||||
.expect("sqlite migrations should run");
|
||||
|
||||
let encoded = encode_jsonl(&[
|
||||
DataExportRecord::manifest(DataExportManifest::new(
|
||||
1_700_000_000,
|
||||
Some(DatabaseDriver::Postgres),
|
||||
vec![ExportDomain::Users, ExportDomain::OAuthProviders],
|
||||
)),
|
||||
DataExportRecord::row(
|
||||
ExportDomain::Users,
|
||||
"oauth-user",
|
||||
json!({
|
||||
"id": "oauth-user",
|
||||
"email": "[email protected]",
|
||||
"email_verified": true,
|
||||
"username": "oauth-user",
|
||||
"role": "user",
|
||||
"auth_source": "oauth",
|
||||
"created_at": 1,
|
||||
"updated_at": 1
|
||||
}),
|
||||
),
|
||||
DataExportRecord::row(
|
||||
ExportDomain::OAuthProviders,
|
||||
"linuxdo",
|
||||
json!({
|
||||
"provider_type": "linuxdo",
|
||||
"display_name": "Linux.do",
|
||||
"client_id": "client",
|
||||
"redirect_uri": "https://gateway.example.test/oauth/callback",
|
||||
"frontend_callback_url": "https://app.example.test/auth/callback",
|
||||
"is_enabled": true,
|
||||
"created_at": 1,
|
||||
"updated_at": 1
|
||||
}),
|
||||
),
|
||||
])
|
||||
.expect("staged identity fixture should encode");
|
||||
|
||||
assert_eq!(
|
||||
import_sqlite_jsonl(&pool, &encoded)
|
||||
.await
|
||||
.expect("users and Providers should restore before links"),
|
||||
2
|
||||
);
|
||||
let email_verified: i64 =
|
||||
sqlx::query_scalar("SELECT email_verified FROM users WHERE id = 'oauth-user'")
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.expect("staged OAuth user should load");
|
||||
assert_eq!(email_verified, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn sqlite_oauth_link_import_rolls_back_without_enabled_login_binding() {
|
||||
let pool = sqlx::sqlite::SqlitePoolOptions::new()
|
||||
.max_connections(1)
|
||||
.connect("sqlite::memory:")
|
||||
.await
|
||||
.expect("sqlite pool should connect");
|
||||
run_sqlite_migrations(&pool)
|
||||
.await
|
||||
.expect("sqlite migrations should run");
|
||||
sqlx::raw_sql(
|
||||
r#"
|
||||
INSERT INTO users (
|
||||
id, email, email_verified, username, role, auth_source,
|
||||
is_active, is_deleted, created_at, updated_at
|
||||
) VALUES (
|
||||
'oauth-user', 'oauth@example.test', 0, 'oauth-user', 'user', 'oauth',
|
||||
1, 0, 1, 1
|
||||
);
|
||||
INSERT INTO oauth_providers (
|
||||
provider_type, display_name, client_id, redirect_uri, frontend_callback_url,
|
||||
is_enabled, created_at, updated_at
|
||||
) VALUES (
|
||||
'linuxdo', 'Linux.do', 'client', 'https://gateway.example.test/oauth/callback',
|
||||
'https://app.example.test/auth/callback', 0, 1, 1
|
||||
);
|
||||
"#,
|
||||
)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.expect("OAuth fixtures should seed");
|
||||
|
||||
let encoded = encode_jsonl(&[
|
||||
DataExportRecord::manifest(DataExportManifest::new(
|
||||
1_700_000_000,
|
||||
Some(DatabaseDriver::Postgres),
|
||||
vec![ExportDomain::UserOAuthLinks],
|
||||
)),
|
||||
DataExportRecord::row(
|
||||
ExportDomain::UserOAuthLinks,
|
||||
"link-disabled",
|
||||
json!({
|
||||
"id": "link-disabled",
|
||||
"user_id": "oauth-user",
|
||||
"provider_type": "linuxdo",
|
||||
"provider_user_id": "subject-1",
|
||||
"linked_at": 1
|
||||
}),
|
||||
),
|
||||
])
|
||||
.expect("OAuth link fixture should encode");
|
||||
|
||||
let err = import_sqlite_jsonl(&pool, &encoded)
|
||||
.await
|
||||
.expect_err("disabled-only OAuth binding should fail");
|
||||
assert!(err
|
||||
.to_string()
|
||||
.contains("without an enabled identity binding"));
|
||||
let link_count: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM user_oauth_links")
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.expect("rolled-back OAuth link count should load");
|
||||
assert_eq!(link_count, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn sqlite_oauth_provider_import_rolls_back_if_it_removes_last_enabled_binding() {
|
||||
let pool = sqlx::sqlite::SqlitePoolOptions::new()
|
||||
.max_connections(1)
|
||||
.connect("sqlite::memory:")
|
||||
.await
|
||||
.expect("sqlite pool should connect");
|
||||
run_sqlite_migrations(&pool)
|
||||
.await
|
||||
.expect("sqlite migrations should run");
|
||||
sqlx::raw_sql(
|
||||
r#"
|
||||
INSERT INTO users (
|
||||
id, email, email_verified, username, role, auth_source,
|
||||
is_active, is_deleted, created_at, updated_at
|
||||
) VALUES (
|
||||
'oauth-user', 'oauth@example.test', 0, 'oauth-user', 'user', 'oauth',
|
||||
1, 0, 1, 1
|
||||
);
|
||||
INSERT INTO oauth_providers (
|
||||
provider_type, display_name, client_id, redirect_uri, frontend_callback_url,
|
||||
is_enabled, created_at, updated_at
|
||||
) VALUES (
|
||||
'linuxdo', 'Linux.do', 'client', 'https://gateway.example.test/oauth/callback',
|
||||
'https://app.example.test/auth/callback', 1, 1, 1
|
||||
);
|
||||
INSERT INTO user_oauth_links (
|
||||
id, user_id, provider_type, provider_user_id, linked_at
|
||||
) VALUES (
|
||||
'existing-link', 'oauth-user', 'linuxdo', 'subject-1', 1
|
||||
);
|
||||
"#,
|
||||
)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.expect("OAuth fixtures should seed");
|
||||
|
||||
let encoded = encode_jsonl(&[
|
||||
DataExportRecord::manifest(DataExportManifest::new(
|
||||
1_700_000_000,
|
||||
Some(DatabaseDriver::Postgres),
|
||||
vec![ExportDomain::OAuthProviders],
|
||||
)),
|
||||
DataExportRecord::row(
|
||||
ExportDomain::OAuthProviders,
|
||||
"linuxdo",
|
||||
json!({
|
||||
"provider_type": "linuxdo",
|
||||
"display_name": "Linux.do disabled",
|
||||
"client_id": "client",
|
||||
"redirect_uri": "https://gateway.example.test/oauth/callback",
|
||||
"frontend_callback_url": "https://app.example.test/auth/callback",
|
||||
"is_enabled": false,
|
||||
"created_at": 1,
|
||||
"updated_at": 2
|
||||
}),
|
||||
),
|
||||
])
|
||||
.expect("disabled Provider fixture should encode");
|
||||
|
||||
let err = import_sqlite_jsonl(&pool, &encoded)
|
||||
.await
|
||||
.expect_err("disabling the last OAuth login method should fail");
|
||||
assert!(err
|
||||
.to_string()
|
||||
.contains("without an enabled identity binding"));
|
||||
let provider = sqlx::query_as::<_, (String, i64)>(
|
||||
"SELECT display_name, is_enabled FROM oauth_providers WHERE provider_type = 'linuxdo'",
|
||||
)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.expect("rolled-back Provider should load");
|
||||
assert_eq!(provider, ("Linux.do".to_string(), 1));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn sqlite_oauth_link_reassignment_rolls_back_if_old_owner_loses_last_binding() {
|
||||
let pool = sqlx::sqlite::SqlitePoolOptions::new()
|
||||
.max_connections(1)
|
||||
.connect("sqlite::memory:")
|
||||
.await
|
||||
.expect("sqlite pool should connect");
|
||||
run_sqlite_migrations(&pool)
|
||||
.await
|
||||
.expect("sqlite migrations should run");
|
||||
sqlx::raw_sql(
|
||||
r#"
|
||||
INSERT INTO users (id, username, role, auth_source, created_at, updated_at) VALUES
|
||||
('oauth-owner', 'oauth-owner', 'user', 'oauth', 1, 1),
|
||||
('local-target', 'local-target', 'user', 'local', 1, 1);
|
||||
INSERT INTO oauth_providers (
|
||||
provider_type, display_name, client_id, redirect_uri, frontend_callback_url,
|
||||
is_enabled, created_at, updated_at
|
||||
) VALUES (
|
||||
'linuxdo', 'Linux.do', 'client', 'https://gateway.example.test/oauth/callback',
|
||||
'https://app.example.test/auth/callback', 1, 1, 1
|
||||
);
|
||||
INSERT INTO user_oauth_links (
|
||||
id, user_id, provider_type, provider_user_id, linked_at
|
||||
) VALUES (
|
||||
'reassigned-link', 'oauth-owner', 'linuxdo', 'subject-1', 1
|
||||
);
|
||||
"#,
|
||||
)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.expect("OAuth reassignment fixtures should seed");
|
||||
|
||||
let encoded = encode_jsonl(&[
|
||||
DataExportRecord::manifest(DataExportManifest::new(
|
||||
1_700_000_000,
|
||||
Some(DatabaseDriver::Postgres),
|
||||
vec![ExportDomain::UserOAuthLinks],
|
||||
)),
|
||||
DataExportRecord::row(
|
||||
ExportDomain::UserOAuthLinks,
|
||||
"reassigned-link",
|
||||
json!({
|
||||
"id": "reassigned-link",
|
||||
"user_id": "local-target",
|
||||
"provider_type": "linuxdo",
|
||||
"provider_user_id": "subject-1",
|
||||
"linked_at": 2
|
||||
}),
|
||||
),
|
||||
])
|
||||
.expect("OAuth reassignment fixture should encode");
|
||||
|
||||
let err = import_sqlite_jsonl(&pool, &encoded)
|
||||
.await
|
||||
.expect_err("taking the old owner's last OAuth binding should fail");
|
||||
assert!(err
|
||||
.to_string()
|
||||
.contains("without an enabled identity binding"));
|
||||
let owner: String =
|
||||
sqlx::query_scalar("SELECT user_id FROM user_oauth_links WHERE id = 'reassigned-link'")
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.expect("rolled-back OAuth link should load");
|
||||
assert_eq!(owner, "oauth-owner");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn sqlite_oauth_link_import_ignores_unrelated_legacy_identity_damage() {
|
||||
let pool = sqlx::sqlite::SqlitePoolOptions::new()
|
||||
.max_connections(1)
|
||||
.connect("sqlite::memory:")
|
||||
.await
|
||||
.expect("sqlite pool should connect");
|
||||
run_sqlite_migrations(&pool)
|
||||
.await
|
||||
.expect("sqlite migrations should run");
|
||||
sqlx::raw_sql(
|
||||
r#"
|
||||
INSERT INTO users (id, username, role, auth_source, created_at, updated_at) VALUES
|
||||
('broken-oauth', 'broken-oauth', 'user', 'oauth', 1, 1),
|
||||
('local-user', 'local-user', 'user', 'local', 1, 1);
|
||||
INSERT INTO oauth_providers (
|
||||
provider_type, display_name, client_id, redirect_uri, frontend_callback_url,
|
||||
is_enabled, created_at, updated_at
|
||||
) VALUES (
|
||||
'linuxdo', 'Linux.do', 'client', 'https://gateway.example.test/oauth/callback',
|
||||
'https://app.example.test/auth/callback', 1, 1, 1
|
||||
);
|
||||
INSERT INTO user_oauth_links (
|
||||
id, user_id, provider_type, provider_user_id, linked_at
|
||||
) VALUES (
|
||||
'legacy-orphan', 'missing-user', 'missing-provider', 'legacy-subject', 1
|
||||
);
|
||||
"#,
|
||||
)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.expect("legacy damaged identity fixtures should seed");
|
||||
|
||||
let encoded = encode_jsonl(&[
|
||||
DataExportRecord::manifest(DataExportManifest::new(
|
||||
1_700_000_000,
|
||||
Some(DatabaseDriver::Postgres),
|
||||
vec![ExportDomain::UserOAuthLinks],
|
||||
)),
|
||||
DataExportRecord::row(
|
||||
ExportDomain::UserOAuthLinks,
|
||||
"valid-link",
|
||||
json!({
|
||||
"id": "valid-link",
|
||||
"user_id": "local-user",
|
||||
"provider_type": "linuxdo",
|
||||
"provider_user_id": "valid-subject",
|
||||
"linked_at": 1
|
||||
}),
|
||||
),
|
||||
])
|
||||
.expect("valid OAuth link fixture should encode");
|
||||
|
||||
assert_eq!(
|
||||
import_sqlite_jsonl(&pool, &encoded)
|
||||
.await
|
||||
.expect("unrelated legacy damage must not block a valid scoped import"),
|
||||
1
|
||||
);
|
||||
let valid_link_count: i64 =
|
||||
sqlx::query_scalar("SELECT COUNT(*) FROM user_oauth_links WHERE id = 'valid-link'")
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.expect("valid OAuth link count should load");
|
||||
assert_eq!(valid_link_count, 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn sqlite_oauth_link_import_rejects_duplicate_identity_in_legacy_schema() {
|
||||
let pool = sqlx::sqlite::SqlitePoolOptions::new()
|
||||
.max_connections(1)
|
||||
.connect("sqlite::memory:")
|
||||
.await
|
||||
.expect("sqlite pool should connect");
|
||||
run_sqlite_migrations(&pool)
|
||||
.await
|
||||
.expect("sqlite migrations should run");
|
||||
sqlx::raw_sql(
|
||||
r#"
|
||||
DROP INDEX uq_user_oauth_links_provider_user;
|
||||
INSERT INTO users (id, username, role, auth_source, created_at, updated_at) VALUES
|
||||
('local-a', 'local-a', 'user', 'local', 1, 1),
|
||||
('local-b', 'local-b', 'user', 'local', 1, 1);
|
||||
INSERT INTO oauth_providers (
|
||||
provider_type, display_name, client_id, redirect_uri, frontend_callback_url,
|
||||
is_enabled, created_at, updated_at
|
||||
) VALUES (
|
||||
'linuxdo', 'Linux.do', 'client', 'https://gateway.example.test/oauth/callback',
|
||||
'https://app.example.test/auth/callback', 1, 1, 1
|
||||
);
|
||||
"#,
|
||||
)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.expect("legacy schema fixture should seed");
|
||||
|
||||
let encoded = encode_jsonl(&[
|
||||
DataExportRecord::manifest(DataExportManifest::new(
|
||||
1_700_000_000,
|
||||
Some(DatabaseDriver::Postgres),
|
||||
vec![ExportDomain::UserOAuthLinks],
|
||||
)),
|
||||
DataExportRecord::row(
|
||||
ExportDomain::UserOAuthLinks,
|
||||
"link-a",
|
||||
json!({
|
||||
"id": "link-a",
|
||||
"user_id": "local-a",
|
||||
"provider_type": "linuxdo",
|
||||
"provider_user_id": "same-subject",
|
||||
"linked_at": 1
|
||||
}),
|
||||
),
|
||||
DataExportRecord::row(
|
||||
ExportDomain::UserOAuthLinks,
|
||||
"link-b",
|
||||
json!({
|
||||
"id": "link-b",
|
||||
"user_id": "local-b",
|
||||
"provider_type": "linuxdo",
|
||||
"provider_user_id": "same-subject",
|
||||
"linked_at": 1
|
||||
}),
|
||||
),
|
||||
])
|
||||
.expect("duplicate identity fixture should encode");
|
||||
|
||||
let err = import_sqlite_jsonl(&pool, &encoded)
|
||||
.await
|
||||
.expect_err("duplicate provider identity should fail");
|
||||
assert!(err.to_string().contains("more than once"));
|
||||
let link_count: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM user_oauth_links")
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.expect("rolled-back OAuth link count should load");
|
||||
assert_eq!(link_count, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn sqlite_oauth_link_import_rejects_duplicate_user_provider_in_legacy_schema() {
|
||||
let pool = sqlx::sqlite::SqlitePoolOptions::new()
|
||||
.max_connections(1)
|
||||
.connect("sqlite::memory:")
|
||||
.await
|
||||
.expect("sqlite pool should connect");
|
||||
run_sqlite_migrations(&pool)
|
||||
.await
|
||||
.expect("sqlite migrations should run");
|
||||
sqlx::raw_sql(
|
||||
r#"
|
||||
DROP INDEX uq_user_oauth_links_user_provider;
|
||||
INSERT INTO users (id, username, role, auth_source, created_at, updated_at)
|
||||
VALUES ('local-user', 'local-user', 'user', 'local', 1, 1);
|
||||
INSERT INTO oauth_providers (
|
||||
provider_type, display_name, client_id, redirect_uri, frontend_callback_url,
|
||||
is_enabled, created_at, updated_at
|
||||
) VALUES (
|
||||
'linuxdo', 'Linux.do', 'client', 'https://gateway.example.test/oauth/callback',
|
||||
'https://app.example.test/auth/callback', 1, 1, 1
|
||||
);
|
||||
"#,
|
||||
)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.expect("legacy schema fixture should seed");
|
||||
|
||||
let encoded = encode_jsonl(&[
|
||||
DataExportRecord::manifest(DataExportManifest::new(
|
||||
1_700_000_000,
|
||||
Some(DatabaseDriver::Postgres),
|
||||
vec![ExportDomain::UserOAuthLinks],
|
||||
)),
|
||||
DataExportRecord::row(
|
||||
ExportDomain::UserOAuthLinks,
|
||||
"link-a",
|
||||
json!({
|
||||
"id": "link-a",
|
||||
"user_id": "local-user",
|
||||
"provider_type": "linuxdo",
|
||||
"provider_user_id": "subject-a",
|
||||
"linked_at": 1
|
||||
}),
|
||||
),
|
||||
DataExportRecord::row(
|
||||
ExportDomain::UserOAuthLinks,
|
||||
"link-b",
|
||||
json!({
|
||||
"id": "link-b",
|
||||
"user_id": "local-user",
|
||||
"provider_type": "linuxdo",
|
||||
"provider_user_id": "subject-b",
|
||||
"linked_at": 1
|
||||
}),
|
||||
),
|
||||
])
|
||||
.expect("duplicate user-provider fixture should encode");
|
||||
|
||||
let err = import_sqlite_jsonl(&pool, &encoded)
|
||||
.await
|
||||
.expect_err("duplicate user provider should fail");
|
||||
assert!(err.to_string().contains("more than once"));
|
||||
let link_count: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM user_oauth_links")
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.expect("rolled-back OAuth link count should load");
|
||||
assert_eq!(link_count, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn sqlite_core_export_reads_migrated_database_rows() {
|
||||
let pool = sqlx::sqlite::SqlitePoolOptions::new()
|
||||
@@ -694,6 +1481,24 @@ VALUES (
|
||||
'request-1', 'candidate-1', 2, 'provider-1',
|
||||
'endpoint-1', 'provider-key-1', '1970-01-01T00:00:01Z', '1970-01-01T00:00:02Z'
|
||||
);
|
||||
INSERT INTO usage_cost_reservations (
|
||||
request_id, subject_id, reservation_token, admitted_at,
|
||||
reserved_cost_units, state, reservation_expires_at, retain_until,
|
||||
created_at, updated_at
|
||||
)
|
||||
VALUES (
|
||||
'request-1', 'user-1', 'reservation-1', 1,
|
||||
500, 'reserved', 2, 3,
|
||||
1, 1
|
||||
);
|
||||
INSERT INTO usage_request_admissions (
|
||||
request_id, subject_id, event_token, admitted_at,
|
||||
retain_until, state, created_at
|
||||
)
|
||||
VALUES (
|
||||
'request-1', 'user-1', 'admission-1', 1,
|
||||
3, 'active', 1
|
||||
);
|
||||
"#,
|
||||
)
|
||||
.execute(&pool)
|
||||
@@ -757,6 +1562,19 @@ VALUES (
|
||||
.any(|row| row.payload["__table"] == "usage_routing_snapshots"
|
||||
&& row.payload["candidate_id"] == "candidate-1"
|
||||
&& row.payload["selected_provider_id"] == "provider-1"));
|
||||
assert!(import_plan
|
||||
.rows(ExportDomain::Auxiliary)
|
||||
.iter()
|
||||
.any(|row| row.payload["__table"] == "usage_cost_reservations"
|
||||
&& row.payload["reservation_token"] == "reservation-1"
|
||||
&& row.payload["reserved_cost_units"] == 500
|
||||
&& row.payload["state"] == "reserved"));
|
||||
assert!(import_plan
|
||||
.rows(ExportDomain::Auxiliary)
|
||||
.iter()
|
||||
.any(|row| row.payload["__table"] == "usage_request_admissions"
|
||||
&& row.payload["event_token"] == "admission-1"
|
||||
&& row.payload["state"] == "active"));
|
||||
|
||||
let target_pool = sqlx::sqlite::SqlitePoolOptions::new()
|
||||
.max_connections(1)
|
||||
@@ -769,14 +1587,19 @@ VALUES (
|
||||
let imported = import_sqlite_jsonl(&target_pool, &encoded)
|
||||
.await
|
||||
.expect("sqlite import should load exported rows");
|
||||
assert_eq!(imported, 20);
|
||||
assert_eq!(imported, 22);
|
||||
|
||||
let imported_api_key =
|
||||
sqlx::query_as::<_, (String,)>("SELECT key_encrypted FROM api_keys WHERE id = 'api-key-1'")
|
||||
let imported_api_key = sqlx::query_as::<_, (String, Option<String>, bool, bool, String)>(
|
||||
"SELECT key_hash, key_encrypted, is_active, is_locked, status FROM api_keys WHERE id = 'api-key-1'",
|
||||
)
|
||||
.fetch_one(&target_pool)
|
||||
.await
|
||||
.expect("imported api key should load");
|
||||
assert_eq!(imported_api_key.0, "ciphertext-1");
|
||||
assert_ne!(imported_api_key.0, "hash-1");
|
||||
assert_eq!(imported_api_key.1, None);
|
||||
assert!(!imported_api_key.2);
|
||||
assert!(imported_api_key.3);
|
||||
assert_eq!(imported_api_key.4, "disabled");
|
||||
|
||||
let imported_usage = sqlx::query_as::<_, (String, i64, String)>(
|
||||
"SELECT request_id, created_at_unix_ms, typeof(created_at_unix_ms) FROM \"usage\" WHERE request_id = 'request-1'",
|
||||
@@ -844,6 +1667,36 @@ WHERE request_id = 'request-1'
|
||||
("candidate-1".to_string(), 2, "provider-1".to_string())
|
||||
);
|
||||
|
||||
let imported_reservation = sqlx::query_as::<_, (String, i64, String)>(
|
||||
r#"
|
||||
SELECT subject_id, reserved_cost_units, state
|
||||
FROM usage_cost_reservations
|
||||
WHERE reservation_token = 'reservation-1'
|
||||
"#,
|
||||
)
|
||||
.fetch_one(&target_pool)
|
||||
.await
|
||||
.expect("imported usage cost reservation should load");
|
||||
assert_eq!(
|
||||
imported_reservation,
|
||||
("user-1".to_string(), 500, "reserved".to_string())
|
||||
);
|
||||
|
||||
let imported_admission = sqlx::query_as::<_, (String, String, Option<i64>)>(
|
||||
r#"
|
||||
SELECT subject_id, state, released_at
|
||||
FROM usage_request_admissions
|
||||
WHERE event_token = 'admission-1'
|
||||
"#,
|
||||
)
|
||||
.fetch_one(&target_pool)
|
||||
.await
|
||||
.expect("imported usage request admission should load");
|
||||
assert_eq!(
|
||||
imported_admission,
|
||||
("user-1".to_string(), "active".to_string(), None)
|
||||
);
|
||||
|
||||
if let Some(database_url) = std::env::var("AETHER_TEST_POSTGRES_URL")
|
||||
.ok()
|
||||
.filter(|value| !value.trim().is_empty())
|
||||
@@ -869,15 +1722,19 @@ WHERE request_id = 'request-1'
|
||||
let imported = import_postgres_jsonl(&postgres_pool, &encoded)
|
||||
.await
|
||||
.expect("postgres import should load exported rows");
|
||||
assert_eq!(imported, 20);
|
||||
assert_eq!(imported, 22);
|
||||
|
||||
let imported_api_key = sqlx::query_as::<_, (String,)>(
|
||||
"SELECT key_encrypted FROM api_keys WHERE id = 'api-key-1'",
|
||||
let imported_api_key = sqlx::query_as::<_, (String, Option<String>, bool, bool, String)>(
|
||||
"SELECT key_hash, key_encrypted, is_active, is_locked, status FROM api_keys WHERE id = 'api-key-1'",
|
||||
)
|
||||
.fetch_one(&postgres_pool)
|
||||
.await
|
||||
.expect("imported postgres api key should load");
|
||||
assert_eq!(imported_api_key.0, "ciphertext-1");
|
||||
assert_ne!(imported_api_key.0, "hash-1");
|
||||
assert_eq!(imported_api_key.1, None);
|
||||
assert!(!imported_api_key.2);
|
||||
assert!(imported_api_key.3);
|
||||
assert_eq!(imported_api_key.4, "disabled");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1317,13 +2174,18 @@ async fn mysql_core_export_reads_migrated_database_rows_when_url_is_set() {
|
||||
.expect("mysql import should be idempotent");
|
||||
assert!(imported >= 6);
|
||||
|
||||
let imported_api_key =
|
||||
sqlx::query_as::<_, (String,)>("SELECT key_encrypted FROM api_keys WHERE id = ?")
|
||||
.bind(&api_key_id)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.expect("imported mysql api key should load");
|
||||
assert_eq!(imported_api_key.0, "ciphertext-1");
|
||||
let imported_api_key = sqlx::query_as::<_, (String, Option<String>, bool, bool, String)>(
|
||||
"SELECT key_hash, key_encrypted, is_active, is_locked, status FROM api_keys WHERE id = ?",
|
||||
)
|
||||
.bind(&api_key_id)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.expect("imported mysql api key should load");
|
||||
assert_ne!(imported_api_key.0, "hash-1");
|
||||
assert_eq!(imported_api_key.1, None);
|
||||
assert!(!imported_api_key.2);
|
||||
assert!(imported_api_key.3);
|
||||
assert_eq!(imported_api_key.4, "disabled");
|
||||
}
|
||||
|
||||
fn unique_suffix() -> String {
|
||||
|
||||
@@ -7,7 +7,7 @@ use std::time::{Duration, Instant};
|
||||
|
||||
use sqlx::{
|
||||
migrate::{AppliedMigration, Migrate},
|
||||
query, query_scalar, Connection, PgConnection, PgPool, SqlitePool,
|
||||
query, query_scalar, Connection, PgConnection, PgPool, Row, SqlitePool,
|
||||
};
|
||||
|
||||
use aether_data_contracts::repository::{
|
||||
@@ -23,7 +23,8 @@ use super::{
|
||||
prepare_database_for_startup,
|
||||
};
|
||||
use crate::lifecycle::bootstrap::postgres::{
|
||||
snapshot_migrations as empty_database_snapshot_migrations, EMPTY_DATABASE_SNAPSHOT_SQL,
|
||||
snapshot_migrations as empty_database_snapshot_migrations,
|
||||
EMPTY_DATABASE_SNAPSHOT_CUTOFF_VERSION, EMPTY_DATABASE_SNAPSHOT_SQL,
|
||||
};
|
||||
|
||||
#[derive(Debug)]
|
||||
@@ -411,8 +412,12 @@ fn empty_database_snapshot_covers_current_cutoff_versions() {
|
||||
20260720000000,
|
||||
20260727000000,
|
||||
20260731000000,
|
||||
20260814000000,
|
||||
20260815000000,
|
||||
20260816000000,
|
||||
20260821000000,
|
||||
20260903000000,
|
||||
20260821120000,
|
||||
20260821130000,
|
||||
]
|
||||
);
|
||||
}
|
||||
@@ -490,6 +495,9 @@ fn portable_driver_migrations_create_the_postgres_table_set() {
|
||||
.iter()
|
||||
.filter(|migration| migration.migration_type.is_up_migration())
|
||||
.flat_map(|migration| create_table_names(migration.sql.as_ref()))
|
||||
// SQLite rebuilds tables to add foreign keys. These staging tables are
|
||||
// renamed to the canonical table names before the migration finishes.
|
||||
.filter(|table| !table.ends_with("_with_user_fk"))
|
||||
.collect::<BTreeSet<_>>();
|
||||
|
||||
assert_eq!(mysql_tables, postgres_tables, "MySQL table set drifted");
|
||||
@@ -1023,6 +1031,487 @@ fn worker_boot_cleanup_migration_is_enabled_for_every_driver() {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn request_candidate_sensitive_diagnostic_purge_is_enabled_for_every_driver() {
|
||||
const VERSION: i64 = 20260822000000;
|
||||
|
||||
for (driver, migrator) in [
|
||||
("postgres", &POSTGRES_MIGRATOR),
|
||||
("mysql", &super::mysql::MIGRATOR),
|
||||
("sqlite", &super::sqlite::MIGRATOR),
|
||||
] {
|
||||
let migration = migrator
|
||||
.iter()
|
||||
.find(|migration| migration.version == VERSION)
|
||||
.unwrap_or_else(|| {
|
||||
panic!("{driver} request candidate diagnostic purge should be embedded")
|
||||
});
|
||||
let sql = migration.sql.as_ref();
|
||||
|
||||
for required in [
|
||||
"UPDATE request_candidates",
|
||||
"username = NULL",
|
||||
"api_key_name = NULL",
|
||||
"extra_data = NULL",
|
||||
"required_capabilities = NULL",
|
||||
"error_message = NULL",
|
||||
"error_type = NULL",
|
||||
"skip_reason = NULL",
|
||||
] {
|
||||
assert!(
|
||||
sql.contains(required),
|
||||
"{driver} request candidate diagnostic purge is missing {required}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deleted_user_history_anonymization_is_enabled_for_every_driver() {
|
||||
const VERSION: i64 = 20260827050000;
|
||||
const HISTORY_TABLES: &[&str] = &[
|
||||
"request_candidates",
|
||||
"video_tasks",
|
||||
"usage",
|
||||
"stats_user_daily",
|
||||
"stats_user_summary",
|
||||
"stats_user_daily_model",
|
||||
"stats_user_daily_provider",
|
||||
"stats_user_daily_api_format",
|
||||
"stats_user_daily_model_provider",
|
||||
"stats_user_daily_cost_savings",
|
||||
"stats_user_daily_cost_savings_provider",
|
||||
"stats_user_daily_cost_savings_model",
|
||||
"stats_user_daily_cost_savings_model_provider",
|
||||
];
|
||||
for (driver, migrator) in [
|
||||
("postgres", &POSTGRES_MIGRATOR),
|
||||
("mysql", &super::mysql::MIGRATOR),
|
||||
("sqlite", &super::sqlite::MIGRATOR),
|
||||
] {
|
||||
let migration = migrator
|
||||
.iter()
|
||||
.find(|migration| migration.version == VERSION)
|
||||
.unwrap_or_else(|| panic!("{driver} user-history anonymization should be embedded"));
|
||||
let sql = migration.sql.as_ref();
|
||||
|
||||
for table in HISTORY_TABLES {
|
||||
assert!(
|
||||
sql.contains(&format!("UPDATE {table}"))
|
||||
|| sql.contains(&format!("UPDATE `{table}`"))
|
||||
|| sql.contains(&format!("UPDATE public.{table}")),
|
||||
"{driver} user-history anonymization is missing {table}"
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
sql.matches("SET username = NULL").count(),
|
||||
HISTORY_TABLES.len(),
|
||||
"{driver} must erase every username snapshot"
|
||||
);
|
||||
assert!(
|
||||
sql.matches("api_key_name = NULL").count() >= 4,
|
||||
"{driver} must erase API key names from request, video, usage, and API key aggregates"
|
||||
);
|
||||
assert!(
|
||||
sql.matches("WHERE NOT EXISTS").count() >= HISTORY_TABLES.len() + 1,
|
||||
"{driver} migration must leave existing users untouched"
|
||||
);
|
||||
assert!(
|
||||
sql.contains("UPDATE stats_daily_api_key")
|
||||
|| sql.contains("UPDATE public.stats_daily_api_key"),
|
||||
"{driver} migration must anonymize orphaned API key aggregate names"
|
||||
);
|
||||
for fact_table in [
|
||||
"user_plan_entitlements",
|
||||
"wallets",
|
||||
"user_referrals",
|
||||
"referral_rewards",
|
||||
] {
|
||||
assert!(
|
||||
sql.contains(&format!("UPDATE {fact_table}"))
|
||||
|| sql.contains(&format!("UPDATE public.{fact_table}")),
|
||||
"{driver} deleted-user fact migration is missing {fact_table}"
|
||||
);
|
||||
}
|
||||
for required in [
|
||||
"THEN 'revoked'",
|
||||
"status = 'disabled'",
|
||||
"THEN 'voided'",
|
||||
"source_json = NULL",
|
||||
"failure_reason = NULL",
|
||||
"admin_note = NULL",
|
||||
"payment_callbacks",
|
||||
"payload = NULL",
|
||||
"error_message = NULL",
|
||||
] {
|
||||
assert!(
|
||||
sql.contains(required),
|
||||
"{driver} deleted-user fact migration is missing {required}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let postgres_migration = POSTGRES_MIGRATOR
|
||||
.iter()
|
||||
.find(|migration| migration.version == VERSION)
|
||||
.expect("postgres user-history anonymization should be embedded");
|
||||
for constraint in [
|
||||
"request_candidates_user_id_fkey",
|
||||
"video_tasks_user_id_fkey",
|
||||
"usage_user_id_fkey",
|
||||
"stats_user_daily_user_id_fkey",
|
||||
"stats_user_summary_user_id_fkey",
|
||||
"stats_user_daily_model_user_id_fkey",
|
||||
"stats_user_daily_provider_user_id_fkey",
|
||||
"stats_user_daily_api_format_user_id_fkey",
|
||||
"stats_user_daily_model_provider_user_id_fkey",
|
||||
"stats_user_daily_cost_savings_user_id_fkey",
|
||||
"stats_user_daily_cost_savings_provider_user_id_fkey",
|
||||
"stats_user_daily_cost_savings_model_user_id_fkey",
|
||||
"stats_user_daily_cost_savings_model_provider_user_id_fkey",
|
||||
"stats_hourly_user_model_user_id_fkey",
|
||||
"user_model_usage_counts_user_id_fkey",
|
||||
"request_candidates_api_key_id_fkey",
|
||||
"video_tasks_api_key_id_fkey",
|
||||
"usage_api_key_id_fkey",
|
||||
"stats_daily_api_key_api_key_id_fkey",
|
||||
"audit_logs_user_id_fkey",
|
||||
"payment_orders_user_id_fkey",
|
||||
"refund_requests_user_id_fkey",
|
||||
"wallet_transactions_operator_id_fkey",
|
||||
"wallets_user_id_fkey",
|
||||
"wallets_api_key_id_fkey",
|
||||
"user_plan_entitlements_user_id_fkey",
|
||||
"entitlement_usage_ledgers_user_id_fkey",
|
||||
"user_referrals_inviter_user_id_fkey",
|
||||
"user_referrals_invitee_user_id_fkey",
|
||||
"referral_rewards_inviter_user_id_fkey",
|
||||
"referral_rewards_invitee_user_id_fkey",
|
||||
] {
|
||||
assert!(
|
||||
postgres_migration
|
||||
.sql
|
||||
.contains(&format!("DROP CONSTRAINT IF EXISTS {constraint}")),
|
||||
"postgres migration must decouple {constraint}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deleted_user_history_anonymization_remediation_is_enabled_for_every_driver() {
|
||||
const VERSION: i64 = 20260829000000;
|
||||
const HISTORY_TABLES: &[&str] = &[
|
||||
"request_candidates",
|
||||
"video_tasks",
|
||||
"usage",
|
||||
"stats_user_daily",
|
||||
"stats_user_summary",
|
||||
"stats_user_daily_model",
|
||||
"stats_user_daily_provider",
|
||||
"stats_user_daily_api_format",
|
||||
"stats_user_daily_model_provider",
|
||||
"stats_user_daily_cost_savings",
|
||||
"stats_user_daily_cost_savings_provider",
|
||||
"stats_user_daily_cost_savings_model",
|
||||
"stats_user_daily_cost_savings_model_provider",
|
||||
];
|
||||
const OWNER_HISTORY_TABLES: &[&str] = &[
|
||||
"wallets",
|
||||
"audit_logs",
|
||||
"wallet_transactions",
|
||||
"payment_orders",
|
||||
"payment_callbacks",
|
||||
"refund_requests",
|
||||
"redeem_code_batches",
|
||||
];
|
||||
|
||||
for (driver, migrator) in [
|
||||
("postgres", &POSTGRES_MIGRATOR),
|
||||
("mysql", &super::mysql::MIGRATOR),
|
||||
("sqlite", &super::sqlite::MIGRATOR),
|
||||
] {
|
||||
let migration = migrator
|
||||
.iter()
|
||||
.find(|migration| migration.version == VERSION)
|
||||
.unwrap_or_else(|| panic!("{driver} history remediation should be embedded"));
|
||||
let sql = migration.sql.as_ref();
|
||||
for table in HISTORY_TABLES {
|
||||
assert!(
|
||||
sql.contains(&format!("UPDATE {table}"))
|
||||
|| sql.contains(&format!("UPDATE `{table}`"))
|
||||
|| sql.contains(&format!("UPDATE public.{table}")),
|
||||
"{driver} remediation is missing {table}"
|
||||
);
|
||||
}
|
||||
for table in OWNER_HISTORY_TABLES {
|
||||
assert!(
|
||||
sql.contains(&format!("UPDATE {table}"))
|
||||
|| sql.contains(&format!("UPDATE `{table}`"))
|
||||
|| sql.contains(&format!("UPDATE public.{table}")),
|
||||
"{driver} remediation is missing owner-linked history table {table}"
|
||||
);
|
||||
}
|
||||
for table in ["request_candidates", "video_tasks", "usage"] {
|
||||
let has_api_key_projection = sql.split(';').any(|statement| {
|
||||
let statement = statement.replace('`', "");
|
||||
(statement.contains(&format!("UPDATE {table}"))
|
||||
|| statement.contains(&format!("UPDATE public.{table}")))
|
||||
&& statement.contains("SET api_key_name = NULL")
|
||||
&& statement.contains("api_key_id IS NOT NULL")
|
||||
});
|
||||
assert!(
|
||||
has_api_key_projection,
|
||||
"{driver} remediation must clear orphaned api_key_name values in {table}"
|
||||
);
|
||||
}
|
||||
for field in ["requested_by", "approved_by", "processed_by"] {
|
||||
assert!(
|
||||
sql.contains(&format!("SET {field} = NULL"))
|
||||
&& sql.contains(&format!("{field} IS NOT NULL")),
|
||||
"{driver} remediation must clear only orphaned {field} references"
|
||||
);
|
||||
}
|
||||
let normalized_sql = sql.replace("public.", "").replace('`', "");
|
||||
assert!(
|
||||
normalized_sql.split(';').any(|statement| {
|
||||
statement.contains("UPDATE payment_callbacks")
|
||||
&& statement.contains("SET payload = NULL")
|
||||
&& statement.contains("WHERE payload IS NOT NULL")
|
||||
}),
|
||||
"{driver} remediation must purge every persisted callback payload"
|
||||
);
|
||||
assert!(
|
||||
normalized_sql.contains("NOT EXISTS (\n SELECT 1 FROM wallets")
|
||||
|| normalized_sql.contains("NOT EXISTS (\n SELECT 1 FROM wallets"),
|
||||
"{driver} remediation must clear sensitive fields for missing wallets"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn sqlite_history_remediation_fails_closed_for_orphaned_financial_records() {
|
||||
const VERSION: i64 = 20260829000000;
|
||||
|
||||
let pool = SqlitePool::connect("sqlite::memory:")
|
||||
.await
|
||||
.expect("sqlite pool should connect");
|
||||
let mut connection = pool.acquire().await.expect("sqlite connection should open");
|
||||
connection
|
||||
.ensure_migrations_table()
|
||||
.await
|
||||
.expect("migration table should be created");
|
||||
for migration in super::sqlite::MIGRATOR
|
||||
.iter()
|
||||
.filter(|migration| migration.version < VERSION)
|
||||
{
|
||||
connection
|
||||
.apply(migration)
|
||||
.await
|
||||
.expect("pre-remediation migration should apply");
|
||||
}
|
||||
drop(connection);
|
||||
|
||||
query(
|
||||
r#"
|
||||
INSERT INTO users (id, username, email, auth_source, created_at, updated_at)
|
||||
VALUES ('remediation-live-user', 'remediation-live', 'remediation-live@example.com', 'local', 1, 1);
|
||||
|
||||
INSERT INTO wallets (
|
||||
id, user_id, balance, gift_balance, limit_mode, currency, status,
|
||||
total_recharged, total_consumed, total_refunded, total_adjusted,
|
||||
created_at, updated_at
|
||||
) VALUES (
|
||||
'remediation-live-wallet', 'remediation-live-user', 0, 0, 'finite', 'USD', 'active',
|
||||
0, 0, 0, 0, 1, 1
|
||||
);
|
||||
|
||||
INSERT INTO payment_orders (
|
||||
id, order_no, wallet_id, user_id, amount_usd, refunded_amount_usd,
|
||||
refundable_amount_usd, payment_method, gateway_response, status, created_at
|
||||
) VALUES
|
||||
('remediation-orphan-order', 'remediation-orphan-order-no', 'missing-wallet', NULL,
|
||||
10, 0, 0, 'epay', '{"secret":"orphan"}', 'pending', 1),
|
||||
('remediation-live-order', 'remediation-live-order-no', 'remediation-live-wallet',
|
||||
'remediation-live-user', 10, 0, 0, 'epay', '{"secret":"live"}', 'pending', 1);
|
||||
|
||||
INSERT INTO payment_callbacks (
|
||||
id, payment_order_id, payment_method, callback_key, order_no,
|
||||
payload_hash, signature_valid, status, payload, error_message, created_at
|
||||
) VALUES
|
||||
('remediation-unmatched-callback', NULL, 'epay', 'remediation-unmatched-key',
|
||||
'remediation-unmatched-order-no', 'hash-unmatched', 0, 'failed',
|
||||
'SECRET-UNMATCHED', 'private unmatched error', 1),
|
||||
('remediation-orphan-callback', 'remediation-orphan-order', 'epay',
|
||||
'remediation-orphan-key', 'remediation-orphan-order-no', 'hash-orphan', 0,
|
||||
'failed', 'SECRET-ORPHAN', 'private orphan error', 1),
|
||||
('remediation-live-callback', 'remediation-live-order', 'epay',
|
||||
'remediation-live-key', 'remediation-live-order-no', 'hash-live', 0,
|
||||
'failed', 'SECRET-LIVE', 'retain diagnostic', 1);
|
||||
|
||||
INSERT INTO wallet_transactions (
|
||||
id, wallet_id, category, reason_code, amount, balance_before, balance_after,
|
||||
recharge_balance_before, recharge_balance_after, gift_balance_before,
|
||||
gift_balance_after, description, created_at
|
||||
) VALUES (
|
||||
'remediation-orphan-wallet-tx', 'missing-wallet', 'adjust', 'manual', 1,
|
||||
0, 1, 0, 1, 0, 0, 'private orphan transaction note', 1
|
||||
);
|
||||
|
||||
INSERT INTO refund_requests (
|
||||
id, refund_no, wallet_id, user_id, source_type, refund_mode, amount_usd,
|
||||
status, reason, payout_reference, payout_proof, failure_reason,
|
||||
created_at, updated_at
|
||||
) VALUES (
|
||||
'remediation-orphan-refund', 'remediation-orphan-refund-no', 'missing-wallet',
|
||||
NULL, 'payment_order', 'original', 1, 'pending_approval', 'private refund reason',
|
||||
'private payout reference', 'private payout proof', 'private failure', 1, 1
|
||||
);
|
||||
"#,
|
||||
)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.expect("orphan financial fixtures should insert");
|
||||
|
||||
let migration = super::sqlite::MIGRATOR
|
||||
.iter()
|
||||
.find(|migration| migration.version == VERSION)
|
||||
.expect("history remediation migration should be embedded");
|
||||
sqlx::raw_sql(migration.sql.as_ref())
|
||||
.execute(&pool)
|
||||
.await
|
||||
.expect("history remediation should apply");
|
||||
|
||||
let payload_count: i64 =
|
||||
query_scalar("SELECT COUNT(*) FROM payment_callbacks WHERE payload IS NOT NULL")
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.expect("callback payload count should query");
|
||||
assert_eq!(
|
||||
payload_count, 0,
|
||||
"raw callback payloads must be purged globally"
|
||||
);
|
||||
|
||||
let orphan_error_count: i64 = query_scalar(
|
||||
"SELECT COUNT(*) FROM payment_callbacks WHERE id IN ('remediation-unmatched-callback', 'remediation-orphan-callback') AND error_message IS NOT NULL",
|
||||
)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.expect("orphan callback errors should query");
|
||||
assert_eq!(
|
||||
orphan_error_count, 0,
|
||||
"orphan callback diagnostics must be purged"
|
||||
);
|
||||
|
||||
let live_error: Option<String> = query_scalar(
|
||||
"SELECT error_message FROM payment_callbacks WHERE id = 'remediation-live-callback'",
|
||||
)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.expect("live callback error should query");
|
||||
assert_eq!(live_error.as_deref(), Some("retain diagnostic"));
|
||||
|
||||
let orphan_gateway_response: Option<String> = query_scalar(
|
||||
"SELECT gateway_response FROM payment_orders WHERE id = 'remediation-orphan-order'",
|
||||
)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.expect("orphan gateway response should query");
|
||||
assert_eq!(orphan_gateway_response, None);
|
||||
|
||||
let orphan_description: Option<String> = query_scalar(
|
||||
"SELECT description FROM wallet_transactions WHERE id = 'remediation-orphan-wallet-tx'",
|
||||
)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.expect("orphan transaction description should query");
|
||||
assert_eq!(orphan_description, None);
|
||||
|
||||
let orphan_refund = query(
|
||||
"SELECT reason, payout_reference, payout_proof, failure_reason FROM refund_requests WHERE id = 'remediation-orphan-refund'",
|
||||
)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.expect("orphan refund should query");
|
||||
for column in [
|
||||
"reason",
|
||||
"payout_reference",
|
||||
"payout_proof",
|
||||
"failure_reason",
|
||||
] {
|
||||
assert_eq!(
|
||||
orphan_refund
|
||||
.try_get::<Option<String>, _>(column)
|
||||
.expect("orphan refund column should decode"),
|
||||
None,
|
||||
"orphan refund {column} must be anonymized"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn background_task_sensitive_diagnostic_purge_is_enabled_for_every_driver() {
|
||||
const VERSION: i64 = 20260822020000;
|
||||
|
||||
for (driver, migrator) in [
|
||||
("postgres", &POSTGRES_MIGRATOR),
|
||||
("mysql", &super::mysql::MIGRATOR),
|
||||
("sqlite", &super::sqlite::MIGRATOR),
|
||||
] {
|
||||
let migration = migrator
|
||||
.iter()
|
||||
.find(|migration| migration.version == VERSION)
|
||||
.unwrap_or_else(|| {
|
||||
panic!("{driver} background task diagnostic purge should be embedded")
|
||||
});
|
||||
let sql = migration.sql.as_ref();
|
||||
|
||||
for required in [
|
||||
"owner_instance = NULL",
|
||||
"created_by = CASE",
|
||||
"progress_message = NULL",
|
||||
"payload_json = NULL",
|
||||
"result_json = NULL",
|
||||
"error_message = CASE",
|
||||
"payload_json = NULL",
|
||||
"unclassified_event",
|
||||
] {
|
||||
assert!(
|
||||
sql.contains(required),
|
||||
"{driver} background task diagnostic purge is missing {required}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn identity_oauth_raw_userinfo_purge_is_enabled_for_every_driver() {
|
||||
const VERSION: i64 = 20260827030000;
|
||||
|
||||
for (driver, migrator) in [
|
||||
("postgres", &POSTGRES_MIGRATOR),
|
||||
("mysql", &super::mysql::MIGRATOR),
|
||||
("sqlite", &super::sqlite::MIGRATOR),
|
||||
] {
|
||||
let migration = migrator
|
||||
.iter()
|
||||
.find(|migration| migration.version == VERSION)
|
||||
.unwrap_or_else(|| panic!("{driver} identity OAuth userinfo purge should be embedded"));
|
||||
let sql = migration.sql.as_ref();
|
||||
|
||||
for required in [
|
||||
"UPDATE",
|
||||
"user_oauth_links",
|
||||
"SET extra_data = NULL",
|
||||
"WHERE extra_data IS NOT NULL",
|
||||
] {
|
||||
assert!(
|
||||
sql.contains(required),
|
||||
"{driver} identity OAuth userinfo purge is missing {required}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mysql_and_sqlite_migrations_include_enabled_incrementals() {
|
||||
let mysql_versions = super::mysql::MIGRATOR
|
||||
@@ -1066,8 +1555,29 @@ fn mysql_and_sqlite_migrations_include_enabled_incrementals() {
|
||||
20260725030000,
|
||||
20260727000000,
|
||||
20260731000000,
|
||||
20260814000000,
|
||||
20260815000000,
|
||||
20260816000000,
|
||||
20260817000000,
|
||||
20260821000000,
|
||||
20260821120000,
|
||||
20260821130000,
|
||||
20260822000000,
|
||||
20260822010000,
|
||||
20260822020000,
|
||||
20260827000000,
|
||||
20260827010000,
|
||||
20260827020000,
|
||||
20260827030000,
|
||||
20260827040000,
|
||||
20260827050000,
|
||||
20260829000000,
|
||||
20260831000000,
|
||||
20260831010000,
|
||||
20260831020000,
|
||||
20260831030000,
|
||||
20260903000000,
|
||||
20260903010000,
|
||||
]
|
||||
);
|
||||
assert_eq!(
|
||||
@@ -1102,12 +1612,224 @@ fn mysql_and_sqlite_migrations_include_enabled_incrementals() {
|
||||
20260725040000,
|
||||
20260727000000,
|
||||
20260731000000,
|
||||
20260814000000,
|
||||
20260815000000,
|
||||
20260816000000,
|
||||
20260821000000,
|
||||
20260821120000,
|
||||
20260821130000,
|
||||
20260822000000,
|
||||
20260822010000,
|
||||
20260822020000,
|
||||
20260827000000,
|
||||
20260827010000,
|
||||
20260827020000,
|
||||
20260827030000,
|
||||
20260827040000,
|
||||
20260827050000,
|
||||
20260829000000,
|
||||
20260831000000,
|
||||
20260831010000,
|
||||
20260831020000,
|
||||
20260831030000,
|
||||
20260903000000,
|
||||
20260903010000,
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn sqlite_legacy_oauth_email_verification_migration_fails_closed() {
|
||||
const VERSION: i64 = 20260903010000;
|
||||
|
||||
let pool = SqlitePool::connect("sqlite::memory:")
|
||||
.await
|
||||
.expect("sqlite pool should connect");
|
||||
super::run_sqlite_migrations(&pool)
|
||||
.await
|
||||
.expect("sqlite migrations should run");
|
||||
query(
|
||||
r#"
|
||||
INSERT INTO users (
|
||||
id, email, username, auth_source, email_verified, created_at, updated_at
|
||||
) VALUES
|
||||
('legacy-oauth', 'oauth@example.com', 'legacy-oauth', 'oauth', 1, 1, 1),
|
||||
('local-user', 'local@example.com', 'local-user', 'local', 1, 1, 1),
|
||||
('ldap-user', 'ldap@example.com', 'ldap-user', 'ldap', 1, 1, 1)
|
||||
"#,
|
||||
)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.expect("email verification fixtures should insert");
|
||||
|
||||
let migration = super::sqlite::MIGRATOR
|
||||
.iter()
|
||||
.find(|migration| migration.version == VERSION)
|
||||
.expect("legacy OAuth verification migration should be embedded");
|
||||
sqlx::raw_sql(migration.sql.as_ref())
|
||||
.execute(&pool)
|
||||
.await
|
||||
.expect("legacy OAuth verification migration should apply");
|
||||
|
||||
for (user_id, expected) in [
|
||||
("legacy-oauth", 0_i64),
|
||||
("local-user", 1_i64),
|
||||
("ldap-user", 1_i64),
|
||||
] {
|
||||
let verified: i64 = query_scalar("SELECT email_verified FROM users WHERE id = ?")
|
||||
.bind(user_id)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.expect("user verification state should load");
|
||||
assert_eq!(verified, expected, "unexpected state for {user_id}");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn sqlite_gateway_order_uniqueness_migration_rejects_historical_duplicates() {
|
||||
const VERSION: i64 = 20260821120000;
|
||||
|
||||
let pool = SqlitePool::connect("sqlite::memory:")
|
||||
.await
|
||||
.expect("sqlite pool should connect");
|
||||
let mut connection = pool.acquire().await.expect("sqlite connection should open");
|
||||
connection
|
||||
.ensure_migrations_table()
|
||||
.await
|
||||
.expect("migration table should be created");
|
||||
for migration in super::sqlite::MIGRATOR
|
||||
.iter()
|
||||
.filter(|migration| migration.version < VERSION)
|
||||
{
|
||||
connection
|
||||
.apply(migration)
|
||||
.await
|
||||
.expect("pre-uniqueness migration should apply");
|
||||
}
|
||||
drop(connection);
|
||||
|
||||
query(
|
||||
r#"
|
||||
INSERT INTO wallets (
|
||||
id, user_id, balance, gift_balance, limit_mode, currency, status,
|
||||
total_recharged, total_consumed, total_refunded, total_adjusted,
|
||||
created_at, updated_at
|
||||
) VALUES
|
||||
('duplicate-wallet-a', 'duplicate-user-a', 0, 0, 'finite', 'USD', 'active', 0, 0, 0, 0, 1, 1),
|
||||
('duplicate-wallet-b', 'duplicate-user-b', 0, 0, 'finite', 'USD', 'active', 0, 0, 0, 0, 1, 1);
|
||||
|
||||
INSERT INTO payment_orders (
|
||||
id, order_no, wallet_id, user_id, amount_usd, refunded_amount_usd,
|
||||
refundable_amount_usd, payment_method, gateway_order_id, status, created_at
|
||||
) VALUES
|
||||
('duplicate-order-a', 'duplicate-no-a', 'duplicate-wallet-a', 'duplicate-user-a', 1, 0, 0, ' EPAY ', 'duplicate-gateway-id', 'pending', 1),
|
||||
('duplicate-order-b', 'duplicate-no-b', 'duplicate-wallet-b', 'duplicate-user-b', 1, 0, 0, 'epay', 'duplicate-gateway-id', 'pending', 1);
|
||||
"#,
|
||||
)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.expect("historical duplicate fixtures should insert");
|
||||
|
||||
let migration = super::sqlite::MIGRATOR
|
||||
.iter()
|
||||
.find(|migration| migration.version == VERSION)
|
||||
.expect("gateway-order uniqueness migration should be embedded");
|
||||
let error = sqlx::raw_sql(migration.sql.as_ref())
|
||||
.execute(&pool)
|
||||
.await
|
||||
.expect_err("historical financial duplicates must block migration");
|
||||
assert!(error.to_string().to_ascii_lowercase().contains("unique"));
|
||||
|
||||
let order_count: i64 = query_scalar(
|
||||
"SELECT COUNT(*) FROM payment_orders WHERE gateway_order_id = 'duplicate-gateway-id'",
|
||||
)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.expect("duplicate financial records should remain intact");
|
||||
assert_eq!(order_count, 2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn sqlite_gateway_order_uniqueness_migration_normalizes_legacy_payment_methods() {
|
||||
const VERSION: i64 = 20260821120000;
|
||||
|
||||
let pool = SqlitePool::connect("sqlite::memory:")
|
||||
.await
|
||||
.expect("sqlite pool should connect");
|
||||
let mut connection = pool.acquire().await.expect("sqlite connection should open");
|
||||
connection
|
||||
.ensure_migrations_table()
|
||||
.await
|
||||
.expect("migration table should be created");
|
||||
for migration in super::sqlite::MIGRATOR
|
||||
.iter()
|
||||
.filter(|migration| migration.version < VERSION)
|
||||
{
|
||||
connection
|
||||
.apply(migration)
|
||||
.await
|
||||
.expect("pre-uniqueness migration should apply");
|
||||
}
|
||||
drop(connection);
|
||||
|
||||
query(
|
||||
r#"
|
||||
INSERT INTO wallets (
|
||||
id, user_id, balance, gift_balance, limit_mode, currency, status,
|
||||
total_recharged, total_consumed, total_refunded, total_adjusted,
|
||||
created_at, updated_at
|
||||
) VALUES ('legacy-method-wallet', 'legacy-method-user', 0, 0, 'finite', 'USD', 'active', 0, 0, 0, 0, 1, 1);
|
||||
|
||||
INSERT INTO payment_orders (
|
||||
id, order_no, wallet_id, user_id, amount_usd, refunded_amount_usd,
|
||||
refundable_amount_usd, payment_method, gateway_order_id, status, created_at
|
||||
) VALUES ('legacy-method-order', 'legacy-method-no', 'legacy-method-wallet', 'legacy-method-user', 1, 0, 0, ' EPAY ', 'CaseSensitiveTxn', 'pending', 1);
|
||||
|
||||
INSERT INTO payment_callbacks (
|
||||
id, payment_method, callback_key, signature_valid, status, created_at
|
||||
) VALUES ('legacy-method-callback', ' EPAY ', 'legacy-method-key', 0, 'received', 1);
|
||||
"#,
|
||||
)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.expect("legacy payment methods should insert");
|
||||
|
||||
let migration = super::sqlite::MIGRATOR
|
||||
.iter()
|
||||
.find(|migration| migration.version == VERSION)
|
||||
.expect("gateway-order uniqueness migration should be embedded");
|
||||
sqlx::raw_sql(migration.sql.as_ref())
|
||||
.execute(&pool)
|
||||
.await
|
||||
.expect("non-conflicting legacy payment methods should normalize");
|
||||
|
||||
let order_method: String =
|
||||
query_scalar("SELECT payment_method FROM payment_orders WHERE id = 'legacy-method-order'")
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.expect("normalized order should load");
|
||||
let callback_method: String = query_scalar(
|
||||
"SELECT payment_method FROM payment_callbacks WHERE id = 'legacy-method-callback'",
|
||||
)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.expect("normalized callback should load");
|
||||
assert_eq!(order_method, "epay");
|
||||
assert_eq!(callback_method, "epay");
|
||||
|
||||
query(
|
||||
r#"
|
||||
INSERT INTO payment_orders (
|
||||
id, order_no, wallet_id, user_id, amount_usd, refunded_amount_usd,
|
||||
refundable_amount_usd, payment_method, gateway_order_id, status, created_at
|
||||
) VALUES ('case-sensitive-order', 'case-sensitive-no', 'legacy-method-wallet', 'legacy-method-user', 1, 0, 0, 'epay', 'casesensitivetxn', 'pending', 1);
|
||||
"#,
|
||||
)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.expect("case-distinct opaque gateway identifiers should remain distinct");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn sqlite_imported_timestamp_migration_normalizes_text_storage() {
|
||||
let pool = SqlitePool::connect("sqlite::memory:")
|
||||
@@ -2210,14 +2932,34 @@ fn pending_migrations_from_applied_skips_versions_already_applied() {
|
||||
20260720000000,
|
||||
20260727000000,
|
||||
20260731000000,
|
||||
20260814000000,
|
||||
20260815000000,
|
||||
20260816000000,
|
||||
20260821000000,
|
||||
20260821120000,
|
||||
20260821130000,
|
||||
20260822000000,
|
||||
20260822010000,
|
||||
20260822020000,
|
||||
20260827000000,
|
||||
20260827010000,
|
||||
20260827020000,
|
||||
20260827030000,
|
||||
20260827040000,
|
||||
20260827050000,
|
||||
20260829000000,
|
||||
20260831000000,
|
||||
20260831010000,
|
||||
20260831030000,
|
||||
20260901000000,
|
||||
20260903000000,
|
||||
20260903010000,
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pending_migrations_from_applied_is_empty_after_empty_database_snapshot_stamp() {
|
||||
fn pending_migrations_from_applied_only_returns_post_snapshot_migrations() {
|
||||
let applied = empty_database_snapshot_migrations(&POSTGRES_MIGRATOR)
|
||||
.expect("empty database snapshot migrations should resolve")
|
||||
.into_iter()
|
||||
@@ -2228,11 +2970,15 @@ fn pending_migrations_from_applied_is_empty_after_empty_database_snapshot_stamp(
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
let pending = pending_migrations_from_applied(&applied);
|
||||
let expected = all_up_migrations()
|
||||
.into_iter()
|
||||
.filter(|migration| migration.version > EMPTY_DATABASE_SNAPSHOT_CUTOFF_VERSION)
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
assert!(
|
||||
pending.is_empty(),
|
||||
"empty database snapshot-stamped databases should not require a manual migration before first startup"
|
||||
);
|
||||
assert_eq!(pending, expected);
|
||||
assert!(pending
|
||||
.iter()
|
||||
.all(|migration| migration.version > EMPTY_DATABASE_SNAPSHOT_CUTOFF_VERSION));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
Reference in New Issue
Block a user