diff --git a/server/src/services/metadata.service.spec.ts b/server/src/services/metadata.service.spec.ts
index 967511633a..52f6609772 100644
--- a/server/src/services/metadata.service.spec.ts
+++ b/server/src/services/metadata.service.spec.ts
@@ -496,6 +496,15 @@ describe(MetadataService.name, () => {
       });
     });
 
+    it('should remove existing tags', async () => {
+      assetMock.getByIds.mockResolvedValue([assetStub.image]);
+      metadataMock.readTags.mockResolvedValue({});
+
+      await sut.handleMetadataExtraction({ id: assetStub.image.id });
+
+      expect(tagMock.upsertAssetTags).toHaveBeenCalledWith({ assetId: 'asset-id', tagIds: [] });
+    });
+
     it('should not apply motion photos if asset is video', async () => {
       assetMock.getByIds.mockResolvedValue([{ ...assetStub.livePhotoMotionAsset, isVisible: true }]);
       mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
diff --git a/server/src/services/metadata.service.ts b/server/src/services/metadata.service.ts
index 5978d0eb74..58e7b99448 100644
--- a/server/src/services/metadata.service.ts
+++ b/server/src/services/metadata.service.ts
@@ -381,11 +381,8 @@ export class MetadataService {
       tags.push(...keywords);
     }
 
-    if (tags.length > 0) {
-      const results = await upsertTags(this.tagRepository, { userId: asset.ownerId, tags: tags.map(String) });
-      const tagIds = results.map((tag) => tag.id);
-      await this.tagRepository.upsertAssetTags({ assetId: asset.id, tagIds });
-    }
+    const results = await upsertTags(this.tagRepository, { userId: asset.ownerId, tags: tags.map(String) });
+    await this.tagRepository.upsertAssetTags({ assetId: asset.id, tagIds: results.map((tag) => tag.id) });
   }
 
   private async applyMotionPhotos(asset: AssetEntity, tags: ImmichTags) {