diff --git a/server/src/repositories/media.repository.ts b/server/src/repositories/media.repository.ts index 33025e73cf..e3e78b3238 100644 --- a/server/src/repositories/media.repository.ts +++ b/server/src/repositories/media.repository.ts @@ -309,9 +309,9 @@ export class MediaRepository { }); } - async getImageDimensions(input: string | Buffer): Promise { - const { width = 0, height = 0 } = await sharp(input).metadata(); - return { width, height }; + async getImageMetadata(input: string | Buffer): Promise { + const { width = 0, height = 0, hasAlpha = false } = await sharp(input).metadata(); + return { width, height, isTransparent: hasAlpha }; } private configureFfmpegCall(input: string, output: string | Writable, options: TranscodeCommand) { diff --git a/server/src/services/media.service.spec.ts b/server/src/services/media.service.spec.ts index 399eb5d6a0..368ece625c 100644 --- a/server/src/services/media.service.spec.ts +++ b/server/src/services/media.service.spec.ts @@ -348,6 +348,7 @@ describe(MediaService.name, () => { : { data: fullsizeBuffer, info: rawInfo as OutputInfo }, // buffer implies embedded image extracted ), ); + mocks.media.getImageMetadata.mockResolvedValue({ width: 100, height: 100, isTransparent: false }); }); it('should skip thumbnail generation if asset not found', async () => { @@ -857,7 +858,7 @@ describe(MediaService.name, () => { .exif({ fileSizeInByte: 5000, profileDescription: 'Adobe RGB', bitsPerSample: 14, orientation: undefined }) .build(); mocks.media.extract.mockResolvedValue({ buffer: extractedBuffer, format: RawExtractedFormat.Jpeg }); - mocks.media.getImageDimensions.mockResolvedValue({ width: 3840, height: 2160 }); + mocks.media.getImageMetadata.mockResolvedValue({ width: 3840, height: 2160, isTransparent: false }); mocks.systemMetadata.get.mockResolvedValue({ image: { extractEmbedded: true } }); mocks.assetJob.getForGenerateThumbnailJob.mockResolvedValue(asset); @@ -871,12 +872,39 @@ describe(MediaService.name, () => { }); }); + it('should not check transparency metadata for raw files without extracted images', async () => { + const asset = AssetFactory.from({ originalFileName: 'file.dng' }) + .exif({ fileSizeInByte: 5000, profileDescription: 'Adobe RGB', bitsPerSample: 14, orientation: undefined }) + .build(); + mocks.systemMetadata.get.mockResolvedValue({ image: { extractEmbedded: false } }); + mocks.assetJob.getForGenerateThumbnailJob.mockResolvedValue(asset); + + await sut.handleGenerateThumbnails({ id: asset.id }); + + expect(mocks.media.getImageMetadata).not.toHaveBeenCalled(); + }); + + it('should not check transparency metadata for raw files with extracted images', async () => { + const asset = AssetFactory.from({ originalFileName: 'file.dng' }) + .exif({ fileSizeInByte: 5000, profileDescription: 'Adobe RGB', bitsPerSample: 14, orientation: undefined }) + .build(); + mocks.media.extract.mockResolvedValue({ buffer: extractedBuffer, format: RawExtractedFormat.Jpeg }); + mocks.media.getImageMetadata.mockResolvedValue({ width: 3840, height: 2160, isTransparent: false }); + mocks.systemMetadata.get.mockResolvedValue({ image: { extractEmbedded: true } }); + mocks.assetJob.getForGenerateThumbnailJob.mockResolvedValue(asset); + + await sut.handleGenerateThumbnails({ id: asset.id }); + + expect(mocks.media.getImageMetadata).toHaveBeenCalledOnce(); + expect(mocks.media.getImageMetadata).toHaveBeenCalledWith(extractedBuffer); + }); + it('should resize original image if embedded image is too small', async () => { const asset = AssetFactory.from({ originalFileName: 'file.dng' }) .exif({ fileSizeInByte: 5000, profileDescription: 'Adobe RGB', bitsPerSample: 14, orientation: undefined }) .build(); mocks.media.extract.mockResolvedValue({ buffer: extractedBuffer, format: RawExtractedFormat.Jpeg }); - mocks.media.getImageDimensions.mockResolvedValue({ width: 1000, height: 1000 }); + mocks.media.getImageMetadata.mockResolvedValue({ width: 1000, height: 1000, isTransparent: false }); mocks.systemMetadata.get.mockResolvedValue({ image: { extractEmbedded: true } }); mocks.assetJob.getForGenerateThumbnailJob.mockResolvedValue(asset); @@ -970,7 +998,7 @@ describe(MediaService.name, () => { image: { fullsize: { enabled: true, format: ImageFormat.Webp }, extractEmbedded: true }, }); mocks.media.extract.mockResolvedValue({ buffer: extractedBuffer, format: RawExtractedFormat.Jpeg }); - mocks.media.getImageDimensions.mockResolvedValue({ width: 3840, height: 2160 }); + mocks.media.getImageMetadata.mockResolvedValue({ width: 3840, height: 2160, isTransparent: false }); mocks.assetJob.getForGenerateThumbnailJob.mockResolvedValue(asset); await sut.handleGenerateThumbnails({ id: asset.id }); @@ -1008,7 +1036,7 @@ describe(MediaService.name, () => { image: { fullsize: { enabled: true, format: ImageFormat.Webp }, extractEmbedded: true }, }); mocks.media.extract.mockResolvedValue({ buffer: extractedBuffer, format: RawExtractedFormat.Jxl }); - mocks.media.getImageDimensions.mockResolvedValue({ width: 3840, height: 2160 }); + mocks.media.getImageMetadata.mockResolvedValue({ width: 3840, height: 2160, isTransparent: false }); mocks.assetJob.getForGenerateThumbnailJob.mockResolvedValue(asset); await sut.handleGenerateThumbnails({ id: asset.id }); @@ -1056,7 +1084,7 @@ describe(MediaService.name, () => { mocks.systemMetadata.get.mockResolvedValue({ image: { fullsize: { enabled: true }, extractEmbedded: false } }); mocks.media.extract.mockResolvedValue({ buffer: extractedBuffer, format: RawExtractedFormat.Jpeg }); - mocks.media.getImageDimensions.mockResolvedValue({ width: 3840, height: 2160 }); + mocks.media.getImageMetadata.mockResolvedValue({ width: 3840, height: 2160, isTransparent: false }); mocks.assetJob.getForGenerateThumbnailJob.mockResolvedValue(asset); await sut.handleGenerateThumbnails({ id: asset.id }); @@ -1100,7 +1128,7 @@ describe(MediaService.name, () => { it('should generate full-size preview from non-web-friendly images', async () => { mocks.systemMetadata.get.mockResolvedValue({ image: { fullsize: { enabled: true } } }); mocks.media.extract.mockResolvedValue({ buffer: extractedBuffer, format: RawExtractedFormat.Jpeg }); - mocks.media.getImageDimensions.mockResolvedValue({ width: 3840, height: 2160 }); + mocks.media.getImageMetadata.mockResolvedValue({ width: 3840, height: 2160, isTransparent: false }); // HEIF/HIF image taken by cameras are not web-friendly, only has limited support on Safari. const asset = AssetFactory.from({ originalFileName: 'image.hif' }) .exif({ @@ -1139,7 +1167,7 @@ describe(MediaService.name, () => { const asset = AssetFactory.from().exif().build(); mocks.systemMetadata.get.mockResolvedValue({ image: { fullsize: { enabled: true } } }); mocks.media.extract.mockResolvedValue({ buffer: extractedBuffer, format: RawExtractedFormat.Jpeg }); - mocks.media.getImageDimensions.mockResolvedValue({ width: 3840, height: 2160 }); + mocks.media.getImageMetadata.mockResolvedValue({ width: 3840, height: 2160, isTransparent: false }); mocks.assetJob.getForGenerateThumbnailJob.mockResolvedValue(asset); await sut.handleGenerateThumbnails({ id: asset.id }); @@ -1162,7 +1190,7 @@ describe(MediaService.name, () => { it('should always generate full-size preview from non-web-friendly panoramas', async () => { mocks.systemMetadata.get.mockResolvedValue({ image: { fullsize: { enabled: false } } }); mocks.media.extract.mockResolvedValue({ buffer: extractedBuffer, format: RawExtractedFormat.Jpeg }); - mocks.media.getImageDimensions.mockResolvedValue({ width: 3840, height: 2160 }); + mocks.media.getImageMetadata.mockResolvedValue({ width: 3840, height: 2160, isTransparent: false }); mocks.media.copyTagGroup.mockResolvedValue(true); const asset = AssetFactory.from({ originalFileName: 'panorama.tif' }) @@ -1208,7 +1236,7 @@ describe(MediaService.name, () => { image: { fullsize: { enabled: true, format: ImageFormat.Webp, quality: 90 } }, }); mocks.media.extract.mockResolvedValue({ buffer: extractedBuffer, format: RawExtractedFormat.Jpeg }); - mocks.media.getImageDimensions.mockResolvedValue({ width: 3840, height: 2160 }); + mocks.media.getImageMetadata.mockResolvedValue({ width: 3840, height: 2160, isTransparent: false }); // HEIF/HIF image taken by cameras are not web-friendly, only has limited support on Safari. const asset = AssetFactory.from({ originalFileName: 'image.hif' }) .exif({ @@ -1248,7 +1276,7 @@ describe(MediaService.name, () => { image: { fullsize: { enabled: true, format: ImageFormat.Jpeg, progressive: true } }, }); mocks.media.extract.mockResolvedValue({ buffer: extractedBuffer, format: RawExtractedFormat.Jpeg }); - mocks.media.getImageDimensions.mockResolvedValue({ width: 3840, height: 2160 }); + mocks.media.getImageMetadata.mockResolvedValue({ width: 3840, height: 2160, isTransparent: false }); const asset = AssetFactory.from({ originalFileName: 'image.hif' }) .exif({ fileSizeInByte: 5000, @@ -1286,6 +1314,7 @@ describe(MediaService.name, () => { : { data: fullsizeBuffer, info: rawInfo as OutputInfo }, // buffer implies embedded image extracted ), ); + mocks.media.getImageMetadata.mockResolvedValue({ width: 100, height: 100, isTransparent: false }); }); it('should skip videos', async () => { @@ -1719,7 +1748,7 @@ describe(MediaService.name, () => { const info = { width: 2160, height: 3840 } as OutputInfo; mocks.media.extract.mockResolvedValue({ buffer: extracted, format: RawExtractedFormat.Jpeg }); mocks.media.decodeImage.mockResolvedValue({ data, info }); - mocks.media.getImageDimensions.mockResolvedValue(info); + mocks.media.getImageMetadata.mockResolvedValue({ width: 2160, height: 3840, isTransparent: false }); await expect(sut.handleGeneratePersonThumbnail({ id: personStub.primaryPerson.id })).resolves.toBe( JobStatus.Success, @@ -1802,7 +1831,7 @@ describe(MediaService.name, () => { const info = { width: 1000, height: 1000 } as OutputInfo; mocks.media.decodeImage.mockResolvedValue({ data, info }); mocks.media.extract.mockResolvedValue({ buffer: extracted, format: RawExtractedFormat.Jpeg }); - mocks.media.getImageDimensions.mockResolvedValue(info); + mocks.media.getImageMetadata.mockResolvedValue({ width: 1000, height: 1000, isTransparent: false }); await expect(sut.handleGeneratePersonThumbnail({ id: personStub.primaryPerson.id })).resolves.toBe( JobStatus.Success, diff --git a/server/src/services/media.service.ts b/server/src/services/media.service.ts index 5fa72cf117..153083142d 100644 --- a/server/src/services/media.service.ts +++ b/server/src/services/media.service.ts @@ -280,14 +280,20 @@ export class MediaService extends BaseService { useEdits; const convertFullsize = generateFullsize && (!extracted || !mimeTypes.isWebSupportedImage(` .${extracted.format}`)); + const thumbSource = extracted ? extracted.buffer : asset.originalPath; const { data, info, colorspace } = await this.decodeImage( - extracted ? extracted.buffer : asset.originalPath, + thumbSource, // only specify orientation to extracted images which don't have EXIF orientation data // or it can double rotate the image extracted ? asset.exifInfo : { ...asset.exifInfo, orientation: null }, convertFullsize ? undefined : image.preview.size, ); + let isTransparent = false; + if (!extracted && mimeTypes.canBeTransparent(asset.originalPath)) { + ({ isTransparent } = await this.mediaRepository.getImageMetadata(asset.originalPath)); + } + return { extracted, data, @@ -295,50 +301,61 @@ export class MediaService extends BaseService { colorspace, convertFullsize, generateFullsize, + isTransparent, }; } private async generateImageThumbnails(asset: ThumbnailAsset, { image }: SystemConfig, useEdits: boolean = false) { + // Handle embedded preview extraction for RAW files + const extractedImage = await this.extractOriginalImage(asset, image, useEdits); + const { info, data, colorspace, generateFullsize, convertFullsize, extracted, isTransparent } = extractedImage; + + const previewFormat = image.preview.format; + this.warnOnTransparencyLoss(isTransparent, previewFormat, asset.id); + + const thumbnailFormat = image.thumbnail.format; + this.warnOnTransparencyLoss(isTransparent, thumbnailFormat, asset.id); + const previewFile = this.getImageFile(asset, { fileType: AssetFileType.Preview, - format: image.preview.format, + format: previewFormat, isEdited: useEdits, - isProgressive: !!image.preview.progressive && image.preview.format !== ImageFormat.Webp, + isProgressive: !!image.preview.progressive && previewFormat !== ImageFormat.Webp, }); const thumbnailFile = this.getImageFile(asset, { fileType: AssetFileType.Thumbnail, - format: image.thumbnail.format, + format: thumbnailFormat, isEdited: useEdits, - isProgressive: !!image.thumbnail.progressive && image.thumbnail.format !== ImageFormat.Webp, + isProgressive: !!image.thumbnail.progressive && thumbnailFormat !== ImageFormat.Webp, }); this.storageCore.ensureFolders(previewFile.path); - // Handle embedded preview extraction for RAW files - const extractedImage = await this.extractOriginalImage(asset, image, useEdits); - const { info, data, colorspace, generateFullsize, convertFullsize, extracted } = extractedImage; - // generate final images - const thumbnailOptions = { colorspace, processInvalidImages: false, raw: info, edits: useEdits ? asset.edits : [] }; + const baseOptions = { colorspace, processInvalidImages: false, raw: info, edits: useEdits ? asset.edits : [] }; + const thumbnailOptions = { ...image.thumbnail, ...baseOptions, format: thumbnailFormat }; + const previewOptions = { ...image.preview, ...baseOptions, format: previewFormat }; const promises = [ - this.mediaRepository.generateThumbhash(data, thumbnailOptions), - this.mediaRepository.generateThumbnail(data, { ...image.thumbnail, ...thumbnailOptions }, thumbnailFile.path), - this.mediaRepository.generateThumbnail(data, { ...image.preview, ...thumbnailOptions }, previewFile.path), + this.mediaRepository.generateThumbhash(data, baseOptions), + this.mediaRepository.generateThumbnail(data, thumbnailOptions, thumbnailFile.path), + this.mediaRepository.generateThumbnail(data, previewOptions, previewFile.path), ]; let fullsizeFile: UpsertFileOptions | undefined; if (convertFullsize) { + const fullsizeFormat = image.fullsize.format; + this.warnOnTransparencyLoss(isTransparent, fullsizeFormat, asset.id); // convert a new fullsize image from the same source as the thumbnail fullsizeFile = this.getImageFile(asset, { fileType: AssetFileType.FullSize, - format: image.fullsize.format, + format: fullsizeFormat, isEdited: useEdits, - isProgressive: !!image.fullsize.progressive && image.fullsize.format !== ImageFormat.Webp, + isProgressive: !!image.fullsize.progressive && fullsizeFormat !== ImageFormat.Webp, }); const fullsizeOptions = { - format: image.fullsize.format, + ...baseOptions, + format: fullsizeFormat, quality: image.fullsize.quality, progressive: image.fullsize.progressive, - ...thumbnailOptions, }; promises.push(this.mediaRepository.generateThumbnail(data, fullsizeOptions, fullsizeFile.path)); } else if (generateFullsize && extracted && extracted.format === RawExtractedFormat.Jpeg) { @@ -758,7 +775,7 @@ export class MediaService extends BaseService { } private async shouldUseExtractedImage(extractedPathOrBuffer: string | Buffer, targetSize: number) { - const { width, height } = await this.mediaRepository.getImageDimensions(extractedPathOrBuffer); + const { width, height } = await this.mediaRepository.getImageMetadata(extractedPathOrBuffer); const extractedSize = Math.min(width, height); return extractedSize >= targetSize; } @@ -857,6 +874,14 @@ export class MediaService extends BaseService { return generated; } + private warnOnTransparencyLoss(isTransparent: boolean, format: ImageFormat, assetId: string) { + if (isTransparent && format === ImageFormat.Jpeg) { + this.logger.warn( + `Asset ${assetId} has transparency but the configured format is ${format} which does not support it, consider using a format that does, such as ${ImageFormat.Webp}`, + ); + } + } + private getImageFile(asset: ThumbnailPathEntity, options: ImagePathOptions & { isProgressive: boolean }) { const path = StorageCore.getImagePath(asset, options); return { diff --git a/server/src/utils/mime-types.spec.ts b/server/src/utils/mime-types.spec.ts index b0e31afe39..862ed310bc 100644 --- a/server/src/utils/mime-types.spec.ts +++ b/server/src/utils/mime-types.spec.ts @@ -153,6 +153,33 @@ describe('mimeTypes', () => { } }); + describe('canBeTransparent', () => { + for (const img of [ + 'a.avif', + 'a.bmp', + 'a.gif', + 'a.heic', + 'a.heif', + 'a.hif', + 'a.jxl', + 'a.png', + 'a.svg', + 'a.tif', + 'a.tiff', + 'a.webp', + ]) { + it(`should return true for ${img}`, () => { + expect(mimeTypes.canBeTransparent(img)).toBe(true); + }); + } + + for (const img of ['a.jpg', 'a.jpeg', 'a.jpe', 'a.insp', 'a.jp2', 'a.cr3', 'a.dng', 'a.nef', 'a.arw']) { + it(`should return false for ${img}`, () => { + expect(mimeTypes.canBeTransparent(img)).toBe(false); + }); + } + }); + describe('animated image', () => { for (const img of ['a.avif', 'a.gif', 'a.webp']) { it('should identify animated image mime types as such', () => { diff --git a/server/src/utils/mime-types.ts b/server/src/utils/mime-types.ts index 4e91bbd7f1..43421e7937 100644 --- a/server/src/utils/mime-types.ts +++ b/server/src/utils/mime-types.ts @@ -77,6 +77,21 @@ const extensionOverrides: Record = { 'image/jpeg': '.jpg', }; +const transparentCapableExtensions = new Set([ + '.avif', + '.bmp', + '.gif', + '.heic', + '.heif', + '.hif', + '.jxl', + '.png', + '.svg', + '.tif', + '.tiff', + '.webp', +]); + const profileExtensions = new Set(['.avif', '.dng', '.heic', '.heif', '.jpeg', '.jpg', '.png', '.webp', '.svg']); const profile: Record = Object.fromEntries( Object.entries(image).filter(([key]) => profileExtensions.has(key)), @@ -134,6 +149,7 @@ export const mimeTypes = { isProfile: (filename: string) => isType(filename, profile), isSidecar: (filename: string) => isType(filename, sidecar), isVideo: (filename: string) => isType(filename, video), + canBeTransparent: (filename: string) => transparentCapableExtensions.has(extname(filename).toLowerCase()), isRaw: (filename: string) => isType(filename, raw), lookup, /** return an extension (including a leading `.`) for a mime-type */ diff --git a/server/test/repositories/media.repository.mock.ts b/server/test/repositories/media.repository.mock.ts index b6b1e82b52..bd8deb4b3a 100644 --- a/server/test/repositories/media.repository.mock.ts +++ b/server/test/repositories/media.repository.mock.ts @@ -12,6 +12,6 @@ export const newMediaRepositoryMock = (): Mocked