fix(gateway): reconcile auth mismatch formats on key update

crates/aether-usage/runtime/src/runtime.rs

@@ -6368,6 +6368,7 @@ mod tests {
 
     #[derive(Clone)]
     struct BlockingWriteQueueConfiguredUsageStore {
+        records: Arc<Mutex<Vec<UpsertUsageRecord>>>,
         queue: Arc<dyn RuntimeQueueStore>,
         write_started: Arc<tokio::sync::Notify>,
         release_writes: Arc<tokio::sync::Notify>,
@@ -7274,10 +7275,11 @@ mod tests {
     impl UsageRecordWriter for BlockingWriteQueueConfiguredUsageStore {
         async fn upsert_usage_record(
             &self,
-            _record: UpsertUsageRecord,
+            record: UpsertUsageRecord,
         ) -> Result<Option<StoredRequestUsageAudit>, DataLayerError> {
             self.write_started.notify_one();
             self.release_writes.notified().await;
+            self.records.lock().expect("records lock").push(record);
             self.writes_completed.fetch_add(1, Ordering::AcqRel);
             Ok(None)
         }
@@ -10277,6 +10279,7 @@ mod tests {
         let release_writes = Arc::new(tokio::sync::Notify::new());
         let writes_completed = Arc::new(AtomicUsize::new(0));
         let store = BlockingWriteQueueConfiguredUsageStore {
+            records: Arc::new(Mutex::new(Vec::new())),
             queue: queue_runner,
             write_started: Arc::clone(&write_started),
             release_writes: Arc::clone(&release_writes),
@@ -10349,6 +10352,7 @@ mod tests {
         let release_writes = Arc::new(tokio::sync::Notify::new());
         let writes_completed = Arc::new(AtomicUsize::new(0));
         let store = BlockingWriteQueueConfiguredUsageStore {
+            records: Arc::new(Mutex::new(Vec::new())),
             queue: queue_runner,
             write_started: Arc::clone(&write_started),
             release_writes: Arc::clone(&release_writes),
@@ -11479,6 +11483,7 @@ mod tests {
         ));
         let queue: Arc<dyn RuntimeQueueStore> = flaky_queue.clone();
         let store = BlockingWriteQueueConfiguredUsageStore {
+            records: Arc::new(Mutex::new(Vec::new())),
             queue,
             write_started: Arc::new(tokio::sync::Notify::new()),
             release_writes: Arc::new(tokio::sync::Notify::new()),
@@ -11676,9 +11681,12 @@ mod tests {
         };
         let queue: Arc<dyn RuntimeQueueStore> =
             Arc::new(RuntimeState::memory(MemoryRuntimeStateConfig::default()));
-        let store = CloneQueueConfiguredUsageStore {
+        let store = BlockingWriteQueueConfiguredUsageStore {
             records: Arc::new(Mutex::new(Vec::new())),
             queue,
+            write_started: Arc::new(tokio::sync::Notify::new()),
+            release_writes: Arc::new(tokio::sync::Notify::new()),
+            writes_completed: Arc::new(AtomicUsize::new(0)),
         };
         let runtime = UsageRuntime::new(config).expect("usage runtime should build");
         let blocker_started = Arc::new(tokio::sync::Notify::new());
@@ -11733,7 +11741,25 @@ mod tests {
         .expect("terminal seed and awaited barrier should queue behind the blocker");
         assert_eq!(runtime.metrics_snapshot().terminal_submission_pending, 0);
 
-        release_blocker.notify_waiters();
+        let first_write_started = store.write_started.notified();
+        release_blocker.notify_one();
+        timeout(Duration::from_secs(1), first_write_started)
+            .await
+            .expect("terminal seed write should start");
+        timeout(Duration::from_secs(1), async {
+            while runtime.metrics_snapshot().terminal_submission_pending != 2 {
+                tokio::task::yield_now().await;
+            }
+        })
+        .await
+        .expect("awaited terminal should queue behind the terminal seed permit");
+
+        let second_write_started = store.write_started.notified();
+        store.release_writes.notify_one();
+        timeout(Duration::from_secs(1), second_write_started)
+            .await
+            .expect("awaited terminal write should start after the seed completes");
+        store.release_writes.notify_one();
         timeout(Duration::from_secs(2), awaited_terminal)
             .await
             .expect("awaited terminal must not deadlock behind the queued seed")
@@ -11741,6 +11767,7 @@ mod tests {
         timeout(Duration::from_secs(2), async {
             while store.records.lock().expect("records lock").len() != 2
                 || runtime.metrics_snapshot().lifecycle_submission_pending != 0
+                || store.writes_completed.load(Ordering::Acquire) != 2
             {
                 tokio::task::yield_now().await;
             }
@@ -11771,9 +11798,12 @@ mod tests {
         };
         let queue: Arc<dyn RuntimeQueueStore> =
             Arc::new(RuntimeState::memory(MemoryRuntimeStateConfig::default()));
-        let store = CloneQueueConfiguredUsageStore {
+        let store = BlockingWriteQueueConfiguredUsageStore {
             records: Arc::new(Mutex::new(Vec::new())),
             queue,
+            write_started: Arc::new(tokio::sync::Notify::new()),
+            release_writes: Arc::new(tokio::sync::Notify::new()),
+            writes_completed: Arc::new(AtomicUsize::new(0)),
         };
         let runtime = UsageRuntime::new(config).expect("usage runtime should build");
         let blocker_started = Arc::new(tokio::sync::Notify::new());
@@ -11815,10 +11845,30 @@ mod tests {
 
         assert_eq!(runtime.metrics_snapshot().terminal_submission_pending, 0);
         assert!(runtime.metrics_snapshot().lifecycle_submission_pending >= 3);
-        release_blocker.notify_waiters();
+
+        let first_write_started = store.write_started.notified();
+        release_blocker.notify_one();
+        timeout(Duration::from_secs(1), first_write_started)
+            .await
+            .expect("terminal seed write should start");
+        timeout(Duration::from_secs(1), async {
+            while runtime.metrics_snapshot().terminal_submission_pending != 2 {
+                tokio::task::yield_now().await;
+            }
+        })
+        .await
+        .expect("submitted terminal should queue behind the terminal seed permit");
+
+        let second_write_started = store.write_started.notified();
+        store.release_writes.notify_one();
+        timeout(Duration::from_secs(1), second_write_started)
+            .await
+            .expect("submitted terminal write should start after the seed completes");
+        store.release_writes.notify_one();
         timeout(Duration::from_secs(2), async {
             while store.records.lock().expect("records lock").len() != 2
                 || runtime.metrics_snapshot().lifecycle_submission_pending != 0
+                || store.writes_completed.load(Ordering::Acquire) != 2
             {
                 tokio::task::yield_now().await;
             }
@@ -12219,6 +12269,7 @@ mod tests {
             ..UsageRuntimeConfig::default()
         };
         let store = BlockingWriteQueueConfiguredUsageStore {
+            records: Arc::new(Mutex::new(Vec::new())),
             queue: Arc::new(RuntimeState::memory(MemoryRuntimeStateConfig::default())),
             write_started: Arc::new(tokio::sync::Notify::new()),
             release_writes: Arc::new(tokio::sync::Notify::new()),