diff --git a/web/src/lib/components/timeline/TimelineAssetViewer.svelte b/web/src/lib/components/timeline/TimelineAssetViewer.svelte
index 17e3ed06f3..a12b708cc4 100644
--- a/web/src/lib/components/timeline/TimelineAssetViewer.svelte
+++ b/web/src/lib/components/timeline/TimelineAssetViewer.svelte
@@ -111,12 +111,12 @@
       case AssetAction.UNARCHIVE:
       case AssetAction.FAVORITE:
       case AssetAction.UNFAVORITE: {
-        timelineManager.updateAssets([action.asset]);
+        timelineManager.upsertAssets([action.asset]);
         break;
       }
 
       case AssetAction.ADD: {
-        timelineManager.addAssets([action.asset]);
+        timelineManager.upsertAssets([action.asset]);
         break;
       }
 
@@ -125,7 +125,7 @@
         break;
       }
       case AssetAction.REMOVE_ASSET_FROM_STACK: {
-        timelineManager.addAssets([toTimelineAsset(action.asset)]);
+        timelineManager.upsertAssets([toTimelineAsset(action.asset)]);
         if (action.stack) {
           //Have to unstack then restack assets in timeline in order to update the stack count in the timeline.
           updateUnstackedAssetInTimeline(
diff --git a/web/src/lib/components/timeline/actions/ArchiveAction.svelte b/web/src/lib/components/timeline/actions/ArchiveAction.svelte
index de778981ca..7dea81076f 100644
--- a/web/src/lib/components/timeline/actions/ArchiveAction.svelte
+++ b/web/src/lib/components/timeline/actions/ArchiveAction.svelte
@@ -1,6 +1,6 @@
 <script lang="ts">
   import { getAssetControlContext } from '$lib/components/timeline/AssetSelectControlBar.svelte';
-  import type { TimelineManager } from '$lib/managers/timeline-manager/timeline-manager.svelte';
+  import type { PhotostreamManager } from '$lib/managers/photostream-manager/PhotostreamManager.svelte';
   import type { OnArchive } from '$lib/utils/actions';
   import { archiveAssets } from '$lib/utils/asset-utils';
   import { AssetVisibility } from '@immich/sdk';
@@ -13,7 +13,7 @@
     onArchive?: OnArchive;
     menuItem?: boolean;
     unarchive?: boolean;
-    manager?: TimelineManager;
+    manager?: PhotostreamManager;
   }
 
   let { onArchive, menuItem = false, unarchive = false, manager }: Props = $props();
diff --git a/web/src/lib/components/timeline/actions/DeleteAssetsAction.svelte b/web/src/lib/components/timeline/actions/DeleteAssetsAction.svelte
index 44680f1805..4ecef2bd14 100644
--- a/web/src/lib/components/timeline/actions/DeleteAssetsAction.svelte
+++ b/web/src/lib/components/timeline/actions/DeleteAssetsAction.svelte
@@ -1,7 +1,7 @@
 <script lang="ts">
   import DeleteAssetDialog from '$lib/components/photos-page/delete-asset-dialog.svelte';
   import { getAssetControlContext } from '$lib/components/timeline/AssetSelectControlBar.svelte';
-  import { TimelineManager } from '$lib/managers/timeline-manager/timeline-manager.svelte';
+  import type { PhotostreamManager } from '$lib/managers/photostream-manager/PhotostreamManager.svelte';
   import type { TimelineAsset } from '$lib/managers/timeline-manager/types';
   import { featureFlags } from '$lib/stores/server-config.store';
   import { type OnDelete, type OnUndoDelete, deleteAssets } from '$lib/utils/actions';
@@ -15,7 +15,7 @@
     onUndoDelete?: OnUndoDelete;
     menuItem?: boolean;
     force?: boolean;
-    manager?: TimelineManager;
+    manager?: PhotostreamManager;
   }
 
   let { onAssetDelete, onUndoDelete, menuItem = false, force = !$featureFlags.trash, manager }: Props = $props();
@@ -40,7 +40,7 @@
     loading = true;
     const assets = [...getOwnedAssets()];
     const undo = (assets: TimelineAsset[]) => {
-      manager?.addAssets(assets);
+      manager?.upsertAssets(assets);
       onUndoDelete?.(assets);
     };
     await deleteAssets(force, onAssetDelete, assets, undo);
diff --git a/web/src/lib/components/timeline/actions/FavoriteAction.svelte b/web/src/lib/components/timeline/actions/FavoriteAction.svelte
index 5adebffe17..3ed61c3b95 100644
--- a/web/src/lib/components/timeline/actions/FavoriteAction.svelte
+++ b/web/src/lib/components/timeline/actions/FavoriteAction.svelte
@@ -5,7 +5,7 @@
     notificationController,
   } from '$lib/components/shared-components/notification/notification';
   import { getAssetControlContext } from '$lib/components/timeline/AssetSelectControlBar.svelte';
-  import type { TimelineManager } from '$lib/managers/timeline-manager/timeline-manager.svelte';
+  import type { PhotostreamManager } from '$lib/managers/photostream-manager/PhotostreamManager.svelte';
   import type { OnFavorite } from '$lib/utils/actions';
   import { handleError } from '$lib/utils/handle-error';
   import { updateAssets } from '@immich/sdk';
@@ -17,7 +17,7 @@
     onFavorite?: OnFavorite;
     menuItem?: boolean;
     removeFavorite: boolean;
-    manager?: TimelineManager;
+    manager?: PhotostreamManager;
   }
 
   let { onFavorite, menuItem = false, removeFavorite, manager }: Props = $props();
diff --git a/web/src/lib/components/timeline/actions/TimelineKeyboardActions.svelte b/web/src/lib/components/timeline/actions/TimelineKeyboardActions.svelte
index 6d7a64d3d8..6f96647ce1 100644
--- a/web/src/lib/components/timeline/actions/TimelineKeyboardActions.svelte
+++ b/web/src/lib/components/timeline/actions/TimelineKeyboardActions.svelte
@@ -51,7 +51,7 @@
       !(isTrashEnabled && !force),
       (assetIds) => timelineManager.removeAssets(assetIds),
       assetInteraction.selectedAssets,
-      !isTrashEnabled || force ? undefined : (assets) => timelineManager.addAssets(assets),
+      !isTrashEnabled || force ? undefined : (assets) => timelineManager.upsertAssets(assets),
     );
     assetInteraction.clearMultiselect();
   };
diff --git a/web/src/lib/managers/photostream-manager/PhotostreamManager.svelte.ts b/web/src/lib/managers/photostream-manager/PhotostreamManager.svelte.ts
index 842f700473..35d504f860 100644
--- a/web/src/lib/managers/photostream-manager/PhotostreamManager.svelte.ts
+++ b/web/src/lib/managers/photostream-manager/PhotostreamManager.svelte.ts
@@ -3,16 +3,21 @@ import { updateGeometry } from '$lib/managers/timeline-manager/internal/layout-s
 import { CancellableTask } from '$lib/utils/cancellable-task';
 import { clamp, debounce } from 'lodash-es';
 
-import type {
+import {
   PhotostreamSegment,
-  SegmentIdentifier,
+  type SegmentIdentifier,
 } from '$lib/managers/photostream-manager/PhotostreamSegment.svelte';
+
+import { updateObject } from '$lib/managers/timeline-manager/internal/utils.svelte';
 import type {
   AssetDescriptor,
+  AssetOperation,
+  MoveAsset,
   TimelineAsset,
   TimelineManagerLayoutOptions,
   Viewport,
 } from '$lib/managers/timeline-manager/types';
+import { setDifference } from '$lib/utils/timeline-util';
 
 export abstract class PhotostreamManager {
   isInitialized = $state(false);
@@ -313,4 +318,98 @@ export abstract class PhotostreamManager {
     }
     return Promise.resolve(range);
   }
+
+  updateAssetOperation(ids: string[], operation: AssetOperation) {
+    return this.#runAssetOperation(new Set(ids), operation);
+  }
+
+  upsertAssets(assets: TimelineAsset[]) {
+    const notExcluded = assets.filter((asset) => !this.isExcluded(asset));
+    const notUpdated = this.#updateAssets(notExcluded);
+    this.addAssetsToSegments([...notUpdated]);
+  }
+
+  #updateAssets(assets: TimelineAsset[]) {
+    const lookup = new Map<string, TimelineAsset>(assets.map((asset) => [asset.id, asset]));
+    const { unprocessedIds } = this.#runAssetOperation(new Set(lookup.keys()), (asset) =>
+      updateObject(asset, lookup.get(asset.id)),
+    );
+    const result: TimelineAsset[] = [];
+    for (const id of unprocessedIds.values()) {
+      result.push(lookup.get(id)!);
+    }
+    return result;
+  }
+
+  removeAssets(ids: string[]) {
+    const { unprocessedIds } = this.#runAssetOperation(new Set(ids), () => ({ remove: true }));
+    return [...unprocessedIds];
+  }
+
+  isExcluded(_: TimelineAsset) {
+    return false;
+  }
+
+  protected createUpsertContext(): unknown {
+    return;
+  }
+
+  protected abstract upsertAssetIntoSegment(asset: TimelineAsset, context: unknown): void;
+
+  protected postCreateSegments(): void {}
+
+  protected postUpsert(_: unknown): void {}
+
+  protected addAssetsToSegments(assets: TimelineAsset[]) {
+    if (assets.length === 0) {
+      return;
+    }
+    const context = this.createUpsertContext();
+    const monthCount = this.months.length;
+    for (const asset of assets) {
+      this.upsertAssetIntoSegment(asset, context);
+    }
+    if (this.months.length !== monthCount) {
+      this.postCreateSegments();
+    }
+    this.postUpsert(context);
+    this.updateIntersections();
+  }
+
+  #runAssetOperation(ids: Set<string>, operation: AssetOperation) {
+    if (ids.size === 0) {
+      return { processedIds: new Set(), unprocessedIds: ids, changedGeometry: false };
+    }
+
+    const changedMonthGroups = new Set<PhotostreamSegment>();
+    let idsToProcess = new Set(ids);
+    const idsProcessed = new Set<string>();
+    const combinedMoveAssets: MoveAsset[][] = [];
+    for (const month of this.months) {
+      if (idsToProcess.size > 0) {
+        const { moveAssets, processedIds, changedGeometry } = month.runAssetOperation(idsToProcess, operation);
+        if (moveAssets.length > 0) {
+          combinedMoveAssets.push(moveAssets);
+        }
+        idsToProcess = setDifference(idsToProcess, processedIds);
+        for (const id of processedIds) {
+          idsProcessed.add(id);
+        }
+        if (changedGeometry) {
+          changedMonthGroups.add(month);
+        }
+      }
+    }
+    if (combinedMoveAssets.length > 0) {
+      this.addAssetsToSegments(combinedMoveAssets.flat().map((a) => a.asset));
+    }
+    const changedGeometry = changedMonthGroups.size > 0;
+    for (const month of changedMonthGroups) {
+      updateGeometry(this, month, { invalidateHeight: true });
+    }
+    if (changedGeometry) {
+      this.updateIntersections();
+    }
+    return { unprocessedIds: idsToProcess, processedIds: idsProcessed, changedGeometry };
+  }
 }
diff --git a/web/src/lib/managers/photostream-manager/PhotostreamSegment.svelte.ts b/web/src/lib/managers/photostream-manager/PhotostreamSegment.svelte.ts
index 6b999b98be..69e6bd1b7a 100644
--- a/web/src/lib/managers/photostream-manager/PhotostreamSegment.svelte.ts
+++ b/web/src/lib/managers/photostream-manager/PhotostreamSegment.svelte.ts
@@ -4,7 +4,7 @@ import { t } from 'svelte-i18n';
 import { get } from 'svelte/store';
 
 import type { PhotostreamManager } from '$lib/managers/photostream-manager/PhotostreamManager.svelte';
-import type { TimelineAsset } from '$lib/managers/timeline-manager/types';
+import type { AssetOperation, MoveAsset, TimelineAsset } from '$lib/managers/timeline-manager/types';
 import type { ViewerAsset } from '$lib/managers/timeline-manager/viewer-asset.svelte';
 
 export type SegmentIdentifier = {
@@ -147,4 +147,46 @@ export abstract class PhotostreamSegment {
   }
 
   abstract findAssetAbsolutePosition(assetId: string): number;
+
+  runAssetOperation(ids: Set<string>, operation: AssetOperation) {
+    if (ids.size === 0) {
+      return {
+        moveAssets: [] as MoveAsset[],
+        processedIds: new Set<string>(),
+        unprocessedIds: ids,
+        changedGeometry: false,
+      };
+    }
+    const unprocessedIds = new Set<string>(ids);
+    const processedIds = new Set<string>();
+    const moveAssets: MoveAsset[] = [];
+    let changedGeometry = false;
+    for (const assetId of unprocessedIds) {
+      const index = this.viewerAssets.findIndex((viewAsset) => viewAsset.id == assetId);
+      if (index === -1) {
+        continue;
+      }
+
+      const asset = this.viewerAssets[index].asset!;
+      const oldTime = { ...asset.localDateTime };
+      const opResult = operation(asset);
+      let remove = false;
+      if (opResult) {
+        remove = (opResult as { remove: boolean }).remove ?? false;
+      }
+      const newTime = asset.localDateTime;
+      if (oldTime.year !== newTime.year || oldTime.month !== newTime.month || oldTime.day !== newTime.day) {
+        const { year, month, day } = newTime;
+        remove = true;
+        moveAssets.push({ asset, date: { year, month, day } });
+      }
+      unprocessedIds.delete(assetId);
+      processedIds.add(assetId);
+      if (remove || this.timelineManager.isExcluded(asset)) {
+        this.viewerAssets.splice(index, 1);
+        changedGeometry = true;
+      }
+    }
+    return { moveAssets, processedIds, unprocessedIds, changedGeometry };
+  }
 }
diff --git a/web/src/lib/managers/timeline-manager/day-group.svelte.ts b/web/src/lib/managers/timeline-manager/day-group.svelte.ts
index 611d917000..0628ff60d5 100644
--- a/web/src/lib/managers/timeline-manager/day-group.svelte.ts
+++ b/web/src/lib/managers/timeline-manager/day-group.svelte.ts
@@ -129,7 +129,11 @@ export class DayGroup {
 
       const asset = this.viewerAssets[index].asset!;
       const oldTime = { ...asset.localDateTime };
-      let { remove } = operation(asset) ?? { remove: false };
+      const opResult = operation(asset);
+      let remove = false;
+      if (opResult) {
+        remove = (opResult as { remove: boolean }).remove ?? false;
+      }
       const newTime = asset.localDateTime;
       if (oldTime.year !== newTime.year || oldTime.month !== newTime.month || oldTime.day !== newTime.day) {
         const { year, month, day } = newTime;
diff --git a/web/src/lib/managers/timeline-manager/internal/operations-support.svelte.ts b/web/src/lib/managers/timeline-manager/internal/operations-support.svelte.ts
deleted file mode 100644
index aff2dd9b65..0000000000
--- a/web/src/lib/managers/timeline-manager/internal/operations-support.svelte.ts
+++ /dev/null
@@ -1,103 +0,0 @@
-import { setDifference, type TimelineDate } from '$lib/utils/timeline-util';
-import { AssetOrder } from '@immich/sdk';
-
-import { SvelteSet } from 'svelte/reactivity';
-import { GroupInsertionCache } from '../group-insertion-cache.svelte';
-import { MonthGroup } from '../month-group.svelte';
-import type { TimelineManager } from '../timeline-manager.svelte';
-import type { AssetOperation, TimelineAsset } from '../types';
-import { updateGeometry } from './layout-support.svelte';
-import { getMonthGroupByDate } from './search-support.svelte';
-
-export function addAssetsToMonthGroups(
-  timelineManager: TimelineManager,
-  assets: TimelineAsset[],
-  options: { order: AssetOrder },
-) {
-  if (assets.length === 0) {
-    return;
-  }
-
-  const addContext = new GroupInsertionCache();
-  const updatedMonthGroups = new SvelteSet<MonthGroup>();
-  const monthCount = timelineManager.months.length;
-  for (const asset of assets) {
-    let month = getMonthGroupByDate(timelineManager, asset.localDateTime);
-
-    if (!month) {
-      month = new MonthGroup(timelineManager, asset.localDateTime, 1, true, options.order);
-      timelineManager.months.push(month);
-    }
-
-    month.addTimelineAsset(asset, addContext);
-    updatedMonthGroups.add(month);
-  }
-
-  if (timelineManager.months.length !== monthCount) {
-    timelineManager.months.sort((a, b) => {
-      return a.yearMonth.year === b.yearMonth.year
-        ? b.yearMonth.month - a.yearMonth.month
-        : b.yearMonth.year - a.yearMonth.year;
-    });
-  }
-
-  for (const group of addContext.existingDayGroups) {
-    group.sortAssets(options.order);
-  }
-
-  for (const monthGroup of addContext.bucketsWithNewDayGroups) {
-    monthGroup.sortDayGroups();
-  }
-
-  for (const month of addContext.updatedBuckets) {
-    month.sortDayGroups();
-    updateGeometry(timelineManager, month, { invalidateHeight: true });
-  }
-  timelineManager.updateIntersections();
-}
-
-export function runAssetOperation(
-  timelineManager: TimelineManager,
-  ids: Set<string>,
-  operation: AssetOperation,
-  options: { order: AssetOrder },
-) {
-  if (ids.size === 0) {
-    return { processedIds: new SvelteSet(), unprocessedIds: ids, changedGeometry: false };
-  }
-
-  const changedMonthGroups = new SvelteSet<MonthGroup>();
-  let idsToProcess = new SvelteSet(ids);
-  const idsProcessed = new SvelteSet<string>();
-  const combinedMoveAssets: { asset: TimelineAsset; date: TimelineDate }[][] = [];
-  for (const month of timelineManager.months) {
-    if (idsToProcess.size > 0) {
-      const { moveAssets, processedIds, changedGeometry } = month.runAssetOperation(idsToProcess, operation);
-      if (moveAssets.length > 0) {
-        combinedMoveAssets.push(moveAssets);
-      }
-      idsToProcess = setDifference(idsToProcess, processedIds);
-      for (const id of processedIds) {
-        idsProcessed.add(id);
-      }
-      if (changedGeometry) {
-        changedMonthGroups.add(month);
-      }
-    }
-  }
-  if (combinedMoveAssets.length > 0) {
-    addAssetsToMonthGroups(
-      timelineManager,
-      combinedMoveAssets.flat().map((a) => a.asset),
-      options,
-    );
-  }
-  const changedGeometry = changedMonthGroups.size > 0;
-  for (const month of changedMonthGroups) {
-    updateGeometry(timelineManager, month, { invalidateHeight: true });
-  }
-  if (changedGeometry) {
-    timelineManager.updateIntersections();
-  }
-  return { unprocessedIds: idsToProcess, processedIds: idsProcessed, changedGeometry };
-}
diff --git a/web/src/lib/managers/timeline-manager/internal/websocket-support.svelte.ts b/web/src/lib/managers/timeline-manager/internal/websocket-support.svelte.ts
index 4ba237c50c..bff2f15cb9 100644
--- a/web/src/lib/managers/timeline-manager/internal/websocket-support.svelte.ts
+++ b/web/src/lib/managers/timeline-manager/internal/websocket-support.svelte.ts
@@ -13,10 +13,10 @@ export class WebsocketSupport {
   #processPendingChanges = throttle(() => {
     const { add, update, remove } = this.#getPendingChangeBatches();
     if (add.length > 0) {
-      this.#timelineManager.addAssets(add);
+      this.#timelineManager.upsertAssets(add);
     }
     if (update.length > 0) {
-      this.#timelineManager.updateAssets(update);
+      this.#timelineManager.upsertAssets(update);
     }
     if (remove.length > 0) {
       this.#timelineManager.removeAssets(remove);
diff --git a/web/src/lib/managers/timeline-manager/timeline-manager.svelte.spec.ts b/web/src/lib/managers/timeline-manager/timeline-manager.svelte.spec.ts
index 1640cb79a1..7619191e8a 100644
--- a/web/src/lib/managers/timeline-manager/timeline-manager.svelte.spec.ts
+++ b/web/src/lib/managers/timeline-manager/timeline-manager.svelte.spec.ts
@@ -173,7 +173,7 @@ describe('TimelineManager', () => {
     });
   });
 
-  describe('addAssets', () => {
+  describe('upsertAssets', () => {
     let timelineManager: TimelineManager;
 
     beforeEach(async () => {
@@ -194,7 +194,7 @@ describe('TimelineManager', () => {
           fileCreatedAt: fromISODateTimeUTCToObject('2024-01-20T12:00:00.000Z'),
         }),
       );
-      timelineManager.addAssets([asset]);
+      timelineManager.upsertAssets([asset]);
 
       expect(timelineManager.months.length).toEqual(1);
       expect(timelineManager.assetCount).toEqual(1);
@@ -210,8 +210,8 @@ describe('TimelineManager', () => {
           fileCreatedAt: fromISODateTimeUTCToObject('2024-01-20T12:00:00.000Z'),
         })
         .map((asset) => deriveLocalDateTimeFromFileCreatedAt(asset));
-      timelineManager.addAssets([assetOne]);
-      timelineManager.addAssets([assetTwo]);
+      timelineManager.upsertAssets([assetOne]);
+      timelineManager.upsertAssets([assetTwo]);
 
       expect(timelineManager.months.length).toEqual(1);
       expect(timelineManager.assetCount).toEqual(2);
@@ -236,7 +236,7 @@ describe('TimelineManager', () => {
           fileCreatedAt: fromISODateTimeUTCToObject('2024-01-16T12:00:00.000Z'),
         }),
       );
-      timelineManager.addAssets([assetOne, assetTwo, assetThree]);
+      timelineManager.upsertAssets([assetOne, assetTwo, assetThree]);
 
       const month = getMonthGroupByDate(timelineManager, { year: 2024, month: 1 });
       expect(month).not.toBeNull();
@@ -262,7 +262,7 @@ describe('TimelineManager', () => {
           fileCreatedAt: fromISODateTimeUTCToObject('2023-01-20T12:00:00.000Z'),
         }),
       );
-      timelineManager.addAssets([assetOne, assetTwo, assetThree]);
+      timelineManager.upsertAssets([assetOne, assetTwo, assetThree]);
 
       expect(timelineManager.months.length).toEqual(3);
       expect(timelineManager.months[0].yearMonth.year).toEqual(2024);
@@ -276,11 +276,11 @@ describe('TimelineManager', () => {
     });
 
     it('updates existing asset', () => {
-      const updateAssetsSpy = vi.spyOn(timelineManager, 'updateAssets');
+      const updateAssetsSpy = vi.spyOn(timelineManager, 'upsertAssets');
       const asset = deriveLocalDateTimeFromFileCreatedAt(timelineAssetFactory.build());
-      timelineManager.addAssets([asset]);
+      timelineManager.upsertAssets([asset]);
 
-      timelineManager.addAssets([asset]);
+      timelineManager.upsertAssets([asset]);
       expect(updateAssetsSpy).toBeCalledWith([asset]);
       expect(timelineManager.assetCount).toEqual(1);
     });
@@ -292,7 +292,7 @@ describe('TimelineManager', () => {
 
       const timelineManager = new TimelineManager();
       await timelineManager.updateOptions({ isTrashed: true });
-      timelineManager.addAssets([asset, trashedAsset]);
+      timelineManager.upsertAssets([asset, trashedAsset]);
       expect(await getAssets(timelineManager)).toEqual([trashedAsset]);
     });
   });
@@ -307,22 +307,15 @@ describe('TimelineManager', () => {
       await timelineManager.updateViewport({ width: 1588, height: 1000 });
     });
 
-    it('ignores non-existing assets', () => {
-      timelineManager.updateAssets([deriveLocalDateTimeFromFileCreatedAt(timelineAssetFactory.build())]);
-
-      expect(timelineManager.months.length).toEqual(0);
-      expect(timelineManager.assetCount).toEqual(0);
-    });
-
     it('updates an asset', () => {
       const asset = deriveLocalDateTimeFromFileCreatedAt(timelineAssetFactory.build({ isFavorite: false }));
       const updatedAsset = { ...asset, isFavorite: true };
 
-      timelineManager.addAssets([asset]);
+      timelineManager.upsertAssets([asset]);
       expect(timelineManager.assetCount).toEqual(1);
       expect(timelineManager.months[0].getFirstAsset().isFavorite).toEqual(false);
 
-      timelineManager.updateAssets([updatedAsset]);
+      timelineManager.upsertAssets([updatedAsset]);
       expect(timelineManager.assetCount).toEqual(1);
       expect(timelineManager.months[0].getFirstAsset().isFavorite).toEqual(true);
     });
@@ -338,12 +331,12 @@ describe('TimelineManager', () => {
         fileCreatedAt: fromISODateTimeUTCToObject('2024-03-20T12:00:00.000Z'),
       });
 
-      timelineManager.addAssets([asset]);
+      timelineManager.upsertAssets([asset]);
       expect(timelineManager.months.length).toEqual(1);
       expect(getMonthGroupByDate(timelineManager, { year: 2024, month: 1 })).not.toBeUndefined();
       expect(getMonthGroupByDate(timelineManager, { year: 2024, month: 1 })?.assets.length).toEqual(1);
 
-      timelineManager.updateAssets([updatedAsset]);
+      timelineManager.upsertAssets([updatedAsset]);
       expect(timelineManager.months.length).toEqual(2);
       expect(getMonthGroupByDate(timelineManager, { year: 2024, month: 1 })).not.toBeUndefined();
       expect(getMonthGroupByDate(timelineManager, { year: 2024, month: 1 })?.assets.length).toEqual(0);
@@ -363,7 +356,7 @@ describe('TimelineManager', () => {
     });
 
     it('ignores invalid IDs', () => {
-      timelineManager.addAssets(
+      timelineManager.upsertAssets(
         timelineAssetFactory
           .buildList(2, {
             fileCreatedAt: fromISODateTimeUTCToObject('2024-01-20T12:00:00.000Z'),
@@ -383,7 +376,7 @@ describe('TimelineManager', () => {
           fileCreatedAt: fromISODateTimeUTCToObject('2024-01-20T12:00:00.000Z'),
         })
         .map((asset) => deriveLocalDateTimeFromFileCreatedAt(asset));
-      timelineManager.addAssets([assetOne, assetTwo]);
+      timelineManager.upsertAssets([assetOne, assetTwo]);
       timelineManager.removeAssets([assetOne.id]);
 
       expect(timelineManager.assetCount).toEqual(1);
@@ -397,7 +390,7 @@ describe('TimelineManager', () => {
           fileCreatedAt: fromISODateTimeUTCToObject('2024-01-20T12:00:00.000Z'),
         })
         .map((asset) => deriveLocalDateTimeFromFileCreatedAt(asset));
-      timelineManager.addAssets(assets);
+      timelineManager.upsertAssets(assets);
       timelineManager.removeAssets(assets.map((asset) => asset.id));
 
       expect(timelineManager.assetCount).toEqual(0);
@@ -429,7 +422,7 @@ describe('TimelineManager', () => {
           fileCreatedAt: fromISODateTimeUTCToObject('2024-01-15T12:00:00.000Z'),
         }),
       );
-      timelineManager.addAssets([assetOne, assetTwo]);
+      timelineManager.upsertAssets([assetOne, assetTwo]);
       expect(timelineManager.getFirstAsset()).toEqual(assetOne);
     });
   });
@@ -554,7 +547,7 @@ describe('TimelineManager', () => {
           fileCreatedAt: fromISODateTimeUTCToObject('2024-02-15T12:00:00.000Z'),
         }),
       );
-      timelineManager.addAssets([assetOne, assetTwo]);
+      timelineManager.upsertAssets([assetOne, assetTwo]);
 
       expect(timelineManager.getMonthGroupByAssetId(assetTwo.id)?.yearMonth.year).toEqual(2024);
       expect(timelineManager.getMonthGroupByAssetId(assetTwo.id)?.yearMonth.month).toEqual(2);
@@ -573,7 +566,7 @@ describe('TimelineManager', () => {
           fileCreatedAt: fromISODateTimeUTCToObject('2024-02-15T12:00:00.000Z'),
         }),
       );
-      timelineManager.addAssets([assetOne, assetTwo]);
+      timelineManager.upsertAssets([assetOne, assetTwo]);
 
       timelineManager.removeAssets([assetTwo.id]);
       expect(timelineManager.getMonthGroupByAssetId(assetOne.id)?.yearMonth.year).toEqual(2024);
diff --git a/web/src/lib/managers/timeline-manager/timeline-manager.svelte.ts b/web/src/lib/managers/timeline-manager/timeline-manager.svelte.ts
index 54ec7d5a9a..31491ff36a 100644
--- a/web/src/lib/managers/timeline-manager/timeline-manager.svelte.ts
+++ b/web/src/lib/managers/timeline-manager/timeline-manager.svelte.ts
@@ -11,14 +11,11 @@ import {
 } from '$lib/utils/timeline-util';
 
 import { isEqual } from 'lodash-es';
-import { SvelteDate, SvelteMap, SvelteSet } from 'svelte/reactivity';
+import { SvelteDate, SvelteSet } from 'svelte/reactivity';
 
 import { PhotostreamManager } from '$lib/managers/photostream-manager/PhotostreamManager.svelte';
+import { GroupInsertionCache } from '$lib/managers/timeline-manager/group-insertion-cache.svelte';
 import { updateGeometry } from '$lib/managers/timeline-manager/internal/layout-support.svelte';
-import {
-  addAssetsToMonthGroups,
-  runAssetOperation,
-} from '$lib/managers/timeline-manager/internal/operations-support.svelte';
 import {
   findMonthGroupForAsset as findMonthGroupForAssetUtil,
   findMonthGroupForDate,
@@ -28,16 +25,9 @@ import {
 } from '$lib/managers/timeline-manager/internal/search-support.svelte';
 import { WebsocketSupport } from '$lib/managers/timeline-manager/internal/websocket-support.svelte';
 import { DayGroup } from './day-group.svelte';
-import { isMismatched, updateObject } from './internal/utils.svelte';
+import { isMismatched } from './internal/utils.svelte';
 import { MonthGroup } from './month-group.svelte';
-import type {
-  AssetDescriptor,
-  AssetOperation,
-  Direction,
-  ScrubberMonth,
-  TimelineAsset,
-  TimelineManagerOptions,
-} from './types';
+import type { AssetDescriptor, Direction, ScrubberMonth, TimelineAsset, TimelineManagerOptions } from './types';
 
 export class TimelineManager extends PhotostreamManager {
   albumAssets: Set<string> = new SvelteSet();
@@ -181,12 +171,6 @@ export class TimelineManager extends PhotostreamManager {
     this.scrubberTimelineHeight = this.timelineHeight;
   }
 
-  addAssets(assets: TimelineAsset[]) {
-    const assetsToUpdate = assets.filter((asset) => !this.isExcluded(asset));
-    const notUpdated = this.updateAssets(assetsToUpdate);
-    addAssetsToMonthGroups(this, [...notUpdated], { order: this.#options.order ?? AssetOrder.Desc });
-  }
-
   async findMonthGroupForAsset(id: string) {
     if (!this.isInitialized) {
       await this.initTask.waitUntilCompletion();
@@ -235,40 +219,6 @@ export class TimelineManager extends PhotostreamManager {
     return month?.getRandomAsset();
   }
 
-  updateAssetOperation(ids: string[], operation: AssetOperation) {
-    runAssetOperation(this, new SvelteSet(ids), operation, { order: this.#options.order ?? AssetOrder.Desc });
-  }
-
-  updateAssets(assets: TimelineAsset[]) {
-    const lookup = new SvelteMap<string, TimelineAsset>(assets.map((asset) => [asset.id, asset]));
-    const { unprocessedIds } = runAssetOperation(
-      this,
-      new SvelteSet(lookup.keys()),
-      (asset) => {
-        updateObject(asset, lookup.get(asset.id));
-        return { remove: false };
-      },
-      { order: this.#options.order ?? AssetOrder.Desc },
-    );
-    const result: TimelineAsset[] = [];
-    for (const id of unprocessedIds.values()) {
-      result.push(lookup.get(id)!);
-    }
-    return result;
-  }
-
-  removeAssets(ids: string[]) {
-    const { unprocessedIds } = runAssetOperation(
-      this,
-      new SvelteSet(ids),
-      () => {
-        return { remove: true };
-      },
-      { order: this.#options.order ?? AssetOrder.Desc },
-    );
-    return [...unprocessedIds];
-  }
-
   refreshLayout() {
     for (const month of this.months) {
       updateGeometry(this, month, { invalidateHeight: true });
@@ -324,4 +274,42 @@ export class TimelineManager extends PhotostreamManager {
   getAssetOrder() {
     return this.#options.order ?? AssetOrder.Desc;
   }
+
+  protected createUpsertContext(): unknown {
+    return new GroupInsertionCache();
+  }
+
+  protected upsertAssetIntoSegment(asset: TimelineAsset, context: GroupInsertionCache): void {
+    let month = getMonthGroupByDate(this, asset.localDateTime);
+
+    if (!month) {
+      month = new MonthGroup(this, asset.localDateTime, 1, true, this.#options.order);
+      this.months.push(month);
+    }
+
+    month.addTimelineAsset(asset, context);
+  }
+
+  protected postCreateSegments(): void {
+    this.months.sort((a, b) => {
+      return a.yearMonth.year === b.yearMonth.year
+        ? b.yearMonth.month - a.yearMonth.month
+        : b.yearMonth.year - a.yearMonth.year;
+    });
+  }
+
+  protected postUpsert(context: GroupInsertionCache): void {
+    for (const group of context.existingDayGroups) {
+      group.sortAssets(this.#options.order);
+    }
+
+    for (const monthGroup of context.bucketsWithNewDayGroups) {
+      monthGroup.sortDayGroups();
+    }
+
+    for (const month of context.updatedBuckets) {
+      month.sortDayGroups();
+      updateGeometry(this, month, { invalidateHeight: true });
+    }
+  }
 }
diff --git a/web/src/lib/managers/timeline-manager/types.ts b/web/src/lib/managers/timeline-manager/types.ts
index 23fe45c7d1..6991a2a087 100644
--- a/web/src/lib/managers/timeline-manager/types.ts
+++ b/web/src/lib/managers/timeline-manager/types.ts
@@ -35,7 +35,7 @@ export type TimelineAsset = {
   longitude?: number | null;
 };
 
-export type AssetOperation = (asset: TimelineAsset) => { remove: boolean } | void;
+export type AssetOperation = (asset: TimelineAsset) => { remove: boolean } | unknown;
 
 export type MoveAsset = { asset: TimelineAsset; date: TimelineDate };
 
diff --git a/web/src/lib/utils/actions.ts b/web/src/lib/utils/actions.ts
index 73fc1111ad..b832d1e169 100644
--- a/web/src/lib/utils/actions.ts
+++ b/web/src/lib/utils/actions.ts
@@ -98,5 +98,5 @@ export function updateUnstackedAssetInTimeline(timelineManager: TimelineManager,
     },
   );
 
-  timelineManager.addAssets(assets);
+  timelineManager.upsertAssets(assets);
 }
diff --git a/web/src/routes/(user)/photos/[[assetId=id]]/+page.svelte b/web/src/routes/(user)/photos/[[assetId=id]]/+page.svelte
index 9ec7da46d2..bd4835e6ff 100644
--- a/web/src/routes/(user)/photos/[[assetId=id]]/+page.svelte
+++ b/web/src/routes/(user)/photos/[[assetId=id]]/+page.svelte
@@ -70,12 +70,12 @@
 
   const handleLink: OnLink = ({ still, motion }) => {
     timelineManager.removeAssets([motion.id]);
-    timelineManager.updateAssets([still]);
+    timelineManager.upsertAssets([still]);
   };
 
   const handleUnlink: OnUnlink = ({ still, motion }) => {
-    timelineManager.addAssets([motion]);
-    timelineManager.updateAssets([still]);
+    timelineManager.upsertAssets([motion]);
+    timelineManager.upsertAssets([still]);
   };
 
   const handleSetVisibility = (assetIds: string[]) => {
diff --git a/web/src/routes/(user)/utilities/geolocation/+page.svelte b/web/src/routes/(user)/utilities/geolocation/+page.svelte
index 5441257df4..77a8a1c671 100644
--- a/web/src/routes/(user)/utilities/geolocation/+page.svelte
+++ b/web/src/routes/(user)/utilities/geolocation/+page.svelte
@@ -63,7 +63,7 @@
       }),
     );
 
-    timelineManager.updateAssets(updatedAssets);
+    timelineManager.upsertAssets(updatedAssets);
 
     handleDeselectAll();
   };