From a67374df7514c22706761a2e8da78912adee0b00 Mon Sep 17 00:00:00 2001 From: mertalev <101130780+mertalev@users.noreply.github.com> Date: Mon, 21 Jul 2025 13:29:50 +0300 Subject: [PATCH] draw to buffer inline scale video frame when possible account for different dimensions --- .../alextran/immich/images/Thumbnails.g.kt | 7 +- .../alextran/immich/images/ThumbnailsImpl.kt | 29 ++- mobile/ios/Runner/Images/Thumbnails.g.swift | 6 +- mobile/ios/Runner/Images/ThumbnailsImpl.swift | 66 ++--- mobile/lib/platform/thumbnail_api.g.dart | 10 +- .../widgets/images/thumbnail.widget.dart | 225 +++++++++--------- .../widgets/images/thumbnail_tile.widget.dart | 10 +- mobile/pigeon/thumbnail_api.dart | 2 +- 8 files changed, 176 insertions(+), 179 deletions(-) diff --git a/mobile/android/app/src/main/kotlin/app/alextran/immich/images/Thumbnails.g.kt b/mobile/android/app/src/main/kotlin/app/alextran/immich/images/Thumbnails.g.kt index 54a30c6ba7..4b8553e2fc 100644 --- a/mobile/android/app/src/main/kotlin/app/alextran/immich/images/Thumbnails.g.kt +++ b/mobile/android/app/src/main/kotlin/app/alextran/immich/images/Thumbnails.g.kt @@ -59,7 +59,7 @@ private open class ThumbnailsPigeonCodec : StandardMessageCodec() { /** Generated interface from Pigeon that represents a handler of messages from Flutter. */ interface ThumbnailApi { - fun setThumbnailToBuffer(pointer: Long, assetId: String, width: Long, height: Long, callback: (Result) -> Unit) + fun setThumbnailToBuffer(pointer: Long, assetId: String, width: Long, height: Long, callback: (Result>) -> Unit) companion object { /** The codec used by ThumbnailApi. */ @@ -79,12 +79,13 @@ interface ThumbnailApi { val assetIdArg = args[1] as String val widthArg = args[2] as Long val heightArg = args[3] as Long - api.setThumbnailToBuffer(pointerArg, assetIdArg, widthArg, heightArg) { result: Result -> + api.setThumbnailToBuffer(pointerArg, assetIdArg, widthArg, heightArg) { result: Result> -> val error = result.exceptionOrNull() if (error != null) { reply.reply(ThumbnailsPigeonUtils.wrapError(error)) } else { - reply.reply(ThumbnailsPigeonUtils.wrapResult(null)) + val data = result.getOrNull() + reply.reply(ThumbnailsPigeonUtils.wrapResult(data)) } } } diff --git a/mobile/android/app/src/main/kotlin/app/alextran/immich/images/ThumbnailsImpl.kt b/mobile/android/app/src/main/kotlin/app/alextran/immich/images/ThumbnailsImpl.kt index d6aa1b3e0f..5f6ac3c0f1 100644 --- a/mobile/android/app/src/main/kotlin/app/alextran/immich/images/ThumbnailsImpl.kt +++ b/mobile/android/app/src/main/kotlin/app/alextran/immich/images/ThumbnailsImpl.kt @@ -96,16 +96,25 @@ class ThumbnailsImpl(context: Context) : ThumbnailApi { private fun decodeVideoThumbnail(assetId: String, targetWidth: Int, targetHeight: Int): Bitmap { val uri = ContentUris.withAppendedId(MediaStore.Video.Media.EXTERNAL_CONTENT_URI, assetId.toLong()) - return if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { - contentResolver.loadThumbnail(uri, Size(targetWidth, targetHeight), null) - } else { - val retriever = MediaMetadataRetriever() - try { - retriever.setDataSource(ctx, uri) - retriever.getFrameAtTime(0L) ?: throw RuntimeException("Failed to extract video frame") - } finally { - retriever.release() - } + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { + return contentResolver.loadThumbnail(uri, Size(targetWidth, targetHeight), null) + } + + val retriever = MediaMetadataRetriever() + try { + retriever.setDataSource(ctx, uri) + return if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) { + retriever.getScaledFrameAtTime( + 0L, + MediaMetadataRetriever.OPTION_NEXT_SYNC, + targetWidth, + targetHeight + ) + } else { + retriever.getFrameAtTime(0L) + } ?: throw RuntimeException("Failed to extract video frame") + } finally { + retriever.release() } } diff --git a/mobile/ios/Runner/Images/Thumbnails.g.swift b/mobile/ios/Runner/Images/Thumbnails.g.swift index 610aedf147..bac3626851 100644 --- a/mobile/ios/Runner/Images/Thumbnails.g.swift +++ b/mobile/ios/Runner/Images/Thumbnails.g.swift @@ -70,7 +70,7 @@ class ThumbnailsPigeonCodec: FlutterStandardMessageCodec, @unchecked Sendable { /// Generated protocol from Pigeon that represents a handler of messages from Flutter. protocol ThumbnailApi { - func setThumbnailToBuffer(pointer: Int64, assetId: String, width: Int64, height: Int64, completion: @escaping (Result) -> Void) + func setThumbnailToBuffer(pointer: Int64, assetId: String, width: Int64, height: Int64, completion: @escaping (Result<[String: Int64], Error>) -> Void) } /// Generated setup class from Pigeon to handle messages through the `binaryMessenger`. @@ -89,8 +89,8 @@ class ThumbnailApiSetup { let heightArg = args[3] as! Int64 api.setThumbnailToBuffer(pointer: pointerArg, assetId: assetIdArg, width: widthArg, height: heightArg) { result in switch result { - case .success: - reply(wrapResult(nil)) + case .success(let res): + reply(wrapResult(res)) case .failure(let error): reply(wrapError(error)) } diff --git a/mobile/ios/Runner/Images/ThumbnailsImpl.swift b/mobile/ios/Runner/Images/ThumbnailsImpl.swift index 84d4239a88..68877819d6 100644 --- a/mobile/ios/Runner/Images/ThumbnailsImpl.swift +++ b/mobile/ios/Runner/Images/ThumbnailsImpl.swift @@ -20,50 +20,36 @@ class ThumbnailApiImpl: ThumbnailApi { return requestOptions }() private static let processingQueue = DispatchQueue(label: "thumbnail.processing", qos: .userInteractive, attributes: .concurrent) + private static let rgbColorSpace = CGColorSpaceCreateDeviceRGB() + private static let bitmapInfo = CGBitmapInfo(rawValue: CGImageAlphaInfo.premultipliedLast.rawValue).rawValue - func setThumbnailToBuffer(pointer: Int64, assetId: String, width: Int64, height: Int64, completion: @escaping (Result) -> Void) { + func setThumbnailToBuffer(pointer: Int64, assetId: String, width: Int64, height: Int64, completion: @escaping (Result<[String: Int64], any Error>) -> Void) { guard let bufferPointer = UnsafeMutableRawPointer(bitPattern: Int(pointer)) else { completion(.failure(PigeonError(code: "", message: "Could not get buffer pointer for \(assetId)", details: nil))); return } Self.processingQueue.async { - do { - let asset = try self.getAsset(assetId: assetId) - Self.cacheManager.requestImage( - for: asset, - targetSize: CGSize(width: Double(width), height: Double(height)), - contentMode: .aspectFill, - options: Self.requestOptions, - resultHandler: { (image, info) -> Void in - guard let image = image, - let cgImage = image.cgImage, - let dataProvider = cgImage.dataProvider, - let pixelData = dataProvider.data - else { completion(.failure(PigeonError(code: "", message: "Could not get pixel data for \(assetId)", details: nil))); return } - - guard let sourceBuffer = CFDataGetBytePtr(pixelData) - else { completion(.failure(PigeonError(code: "", message: "Could not get pixel data buffer for \(assetId)", details: nil))); return } - let dataLength = CFDataGetLength(pixelData) - let bufferLength = width * height * 4 - guard dataLength <= bufferLength - else { completion(.failure(PigeonError(code: "", message: "Buffer is not large enough (\(bufferLength) vs \(dataLength) for \(assetId)", details: nil))); return } - - bufferPointer.copyMemory(from: sourceBuffer, byteCount: dataLength) - completion(.success(())) - } - ) - } catch { - completion( - .failure(PigeonError(code: "", message: "Could not get asset data for \(assetId)", details: nil))) - } + guard let asset = PHAsset.fetchAssets(withLocalIdentifiers: [assetId], options: Self.fetchOptions).firstObject + else { completion(.failure(PigeonError(code: "", message: "Could not get asset data for \(assetId)", details: nil))); return } + Self.cacheManager.requestImage( + for: asset, + targetSize: CGSize(width: Double(width), height: Double(height)), + contentMode: .aspectFill, + options: Self.requestOptions, + resultHandler: { (image, info) -> Void in + guard let image = image, + let cgImage = image.cgImage, + let context = CGContext( + data: bufferPointer, + width: cgImage.width, + height: cgImage.height, + bitsPerComponent: 8, + bytesPerRow: cgImage.width * 4, + space: Self.rgbColorSpace, + bitmapInfo: Self.bitmapInfo + ) else { completion(.failure(PigeonError(code: "", message: "Could not get pixel data for \(assetId)", details: nil))); return } + context.draw(cgImage, in: CGRect(x: 0, y: 0, width: cgImage.width, height: cgImage.height)) + completion(.success(["width": Int64(cgImage.width), "height": Int64(cgImage.height)])) + } + ) } } - - private func getAsset(assetId: String) throws -> PHAsset { - guard - let asset = PHAsset.fetchAssets(withLocalIdentifiers: [assetId], options: Self.fetchOptions) - .firstObject - else { - throw PigeonError(code: "", message: "Could not fetch asset", details: nil) - } - return asset - } } diff --git a/mobile/lib/platform/thumbnail_api.g.dart b/mobile/lib/platform/thumbnail_api.g.dart index e528590a82..cc6f6ef3bf 100644 --- a/mobile/lib/platform/thumbnail_api.g.dart +++ b/mobile/lib/platform/thumbnail_api.g.dart @@ -51,7 +51,7 @@ class ThumbnailApi { final String pigeonVar_messageChannelSuffix; - Future setThumbnailToBuffer( + Future> setThumbnailToBuffer( int pointer, String assetId, { required int width, @@ -77,8 +77,14 @@ class ThumbnailApi { message: pigeonVar_replyList[1] as String?, details: pigeonVar_replyList[2], ); + } else if (pigeonVar_replyList[0] == null) { + throw PlatformException( + code: 'null-error', + message: 'Host platform returned null value for non-null return value.', + ); } else { - return; + return (pigeonVar_replyList[0] as Map?)! + .cast(); } } } diff --git a/mobile/lib/presentation/widgets/images/thumbnail.widget.dart b/mobile/lib/presentation/widgets/images/thumbnail.widget.dart index a3940b4713..c4520ed106 100644 --- a/mobile/lib/presentation/widgets/images/thumbnail.widget.dart +++ b/mobile/lib/presentation/widgets/images/thumbnail.widget.dart @@ -92,7 +92,7 @@ class _ThumbnailState extends State { if (oldWidget.blurhash != widget.blurhash || oldWidget.localId != widget.localId || oldWidget.remoteId != widget.remoteId || - oldWidget.thumbhashOnly != widget.thumbhashOnly) { + (oldWidget.thumbhashOnly && !widget.thumbhashOnly)) { _decode(); } } @@ -104,18 +104,13 @@ class _ThumbnailState extends State { final thumbhashOnly = widget.thumbhashOnly; final blurhash = widget.blurhash; - final imageFuture = thumbhashOnly ? Future.value(null) : _decodeFromFile(); + final imageFuture = thumbhashOnly ? Future.value(null) : _decodeThumbnail(); - if (blurhash != null) { - final image = thumbhash.thumbHashToRGBA(base64.decode(blurhash)); + if (blurhash != null && _image == null) { try { - await _decodeThumbhash( - await ImmutableBuffer.fromUint8List(image.rgba), - image.width, - image.height, - ); + await _decodeThumbhash(); } catch (e) { - log.info('Error decoding thumbhash for ${widget.remoteId}: $e'); + log.severe('Error decoding thumbhash for ${widget.remoteId}: $e'); } } @@ -134,38 +129,32 @@ class _ThumbnailState extends State { _image = image; }); } catch (e) { - log.info('Error decoding thumbnail: $e'); + log.severe('Error decoding thumbnail: $e'); } } - Future _decodeThumbhash( - ImmutableBuffer buffer, - int width, - int height, - ) async { - if (!mounted) { + Future _decodeThumbhash() async { + final blurhash = widget.blurhash; + if (blurhash == null || !mounted || _image != null) { + return; + } + final image = thumbhash.thumbHashToRGBA(base64.decode(blurhash)); + final buffer = await ImmutableBuffer.fromUint8List(image.rgba); + if (!mounted || _image != null) { buffer.dispose(); return; } final descriptor = ImageDescriptor.raw( buffer, - width: width, - height: height, + width: image.width, + height: image.height, pixelFormat: PixelFormat.rgba8888, ); - if (!mounted) { - buffer.dispose(); - descriptor.dispose(); - return; - } - final codec = await descriptor.instantiateCodec( - targetWidth: width, - targetHeight: height, - ); + final codec = await descriptor.instantiateCodec(); - if (!mounted) { + if (!mounted || _image != null) { buffer.dispose(); descriptor.dispose(); codec.dispose(); @@ -176,7 +165,7 @@ class _ThumbnailState extends State { buffer.dispose(); descriptor.dispose(); codec.dispose(); - if (!mounted) { + if (!mounted || _image != null) { frame.dispose(); return; } @@ -185,112 +174,110 @@ class _ThumbnailState extends State { }); } - Future _decodeFromFile() async { - final buffer = await _getFile(); - if (buffer == null) { + Future _decodeThumbnail() async { + if (!mounted) { return null; } + final stopwatch = Stopwatch()..start(); - final thumb = await _decodeThumbnail(buffer, 256, 256); + final codec = await _decodeThumb(); + if (codec == null || !mounted) { + codec?.dispose(); + return null; + } + final image = (await codec.getNextFrame()).image; stopwatch.stop(); - return thumb; + log.info( + 'Decoded thumbnail for ${widget.remoteId ?? widget.localId} in ${stopwatch.elapsedMilliseconds} ms', + ); + return image; } - Future _decodeThumbnail( - ImmutableBuffer buffer, - int width, - int height, - ) async { - if (!mounted) { - buffer.dispose(); - return null; - } - - final descriptor = ImageDescriptor.raw( - buffer, - width: width, - height: height, - pixelFormat: PixelFormat.rgba8888, - ); - - if (!mounted) { - buffer.dispose(); - descriptor.dispose(); - return null; - } - - final codec = await descriptor.instantiateCodec( - targetWidth: width, - targetHeight: height, - ); - - if (!mounted) { - buffer.dispose(); - descriptor.dispose(); - codec.dispose(); - return null; - } - - final frame = (await codec.getNextFrame()).image; - buffer.dispose(); - descriptor.dispose(); - codec.dispose(); - if (!mounted) { - frame.dispose(); - return null; - } - - return frame; - } - - Future _getFile() async { - final stopwatch = Stopwatch()..start(); + Future _decodeThumb() { final localId = widget.localId; + if (!mounted) { + return Future.value(null); + } + if (localId != null) { - final size = 256 * 256 * 4; - final pointer = malloc(size); - try { - await thumbnailApi.setThumbnailToBuffer( - pointer.address, - localId, - width: 256, - height: 256, - ); - stopwatch.stop(); - log.info( - 'Retrieved local image $localId in ${stopwatch.elapsedMilliseconds.toStringAsFixed(2)} ms', - ); - return ImmutableBuffer.fromUint8List(pointer.asTypedList(size)); - } catch (e) { - log.warning('Failed to retrieve local image $localId: $e'); - } finally { - malloc.free(pointer); - } + final size = widget.size; + final width = size.width.toInt(); + final height = size.height.toInt(); + return _decodeLocal(localId, width, height); } final remoteId = widget.remoteId; if (remoteId != null) { - final uri = getThumbnailUrlForRemoteId(remoteId); - final headers = ApiService.getRequestHeaders(); - final stream = _imageCache.getFileStream( - uri, - key: uri, - withProgress: true, - headers: headers, - ); + return _decodeRemote(remoteId); + } - await for (final result in stream) { + return Future.value(null); + } + + Future _decodeLocal(String localId, int width, int height) async { + final pointer = malloc(width * height * 4); + + try { + final info = await thumbnailApi.setThumbnailToBuffer( + pointer.address, + localId, + width: width, + height: height, + ); + if (!mounted) { + return null; + } + final actualWidth = info['width']!; + final actualHeight = info['height']!; + final actualSize = actualWidth * actualHeight * 4; + final buffer = + await ImmutableBuffer.fromUint8List(pointer.asTypedList(actualSize)); + if (!mounted) { + buffer.dispose(); + return null; + } + final descriptor = ui.ImageDescriptor.raw( + buffer, + width: actualWidth, + height: actualHeight, + pixelFormat: ui.PixelFormat.rgba8888, + ); + return await descriptor.instantiateCodec(); + } catch (e) { + return null; + } finally { + malloc.free(pointer); + } + } + + Future _decodeRemote(String remoteId) async { + final uri = getThumbnailUrlForRemoteId(remoteId); + final headers = ApiService.getRequestHeaders(); + final stream = _imageCache.getFileStream( + uri, + key: uri, + withProgress: true, + headers: headers, + ); + + await for (final result in stream) { + if (!mounted) { + return null; + } + + if (result is FileInfo) { + final buffer = await ImmutableBuffer.fromFilePath(result.file.path); if (!mounted) { + buffer.dispose(); return null; } - - if (result is FileInfo) { - stopwatch.stop(); - log.info( - 'Retrieved remote image $remoteId in ${stopwatch.elapsedMilliseconds.toStringAsFixed(2)} ms', - ); - return ImmutableBuffer.fromFilePath(result.file.path); + final descriptor = await ImageDescriptor.encoded(buffer); + if (!mounted) { + buffer.dispose(); + descriptor.dispose(); + return null; } + return await descriptor.instantiateCodec(); } } diff --git a/mobile/lib/presentation/widgets/images/thumbnail_tile.widget.dart b/mobile/lib/presentation/widgets/images/thumbnail_tile.widget.dart index 176f89e5c9..13714e5cba 100644 --- a/mobile/lib/presentation/widgets/images/thumbnail_tile.widget.dart +++ b/mobile/lib/presentation/widgets/images/thumbnail_tile.widget.dart @@ -8,6 +8,7 @@ import 'package:immich_mobile/extensions/duration_extensions.dart'; import 'package:immich_mobile/extensions/theme_extensions.dart'; import 'package:immich_mobile/presentation/widgets/images/thumbnail.widget.dart'; import 'package:immich_mobile/providers/infrastructure/setting.provider.dart'; +import 'package:immich_mobile/presentation/widgets/timeline/timeline.state.dart'; import 'package:immich_mobile/providers/timeline/multiselect.provider.dart'; class ThumbnailTile extends ConsumerWidget { @@ -40,6 +41,8 @@ class ThumbnailTile extends ConsumerWidget { final isSelected = ref.watch( multiSelectProvider.select((multiselect) => multiselect.selectedAssets.contains(asset)), ); + final isScrubbing = + ref.watch(timelineStateProvider.select((state) => state.isScrubbing)); final borderStyle = lockSelection ? BoxDecoration( @@ -73,7 +76,12 @@ class ThumbnailTile extends ConsumerWidget { Positioned.fill( child: Hero( tag: '${asset?.heroTag ?? ''}_$heroIndex', - child: Thumbnail.fromBaseAsset(asset: asset, fit: fit, size: size), + child: Thumbnail.fromBaseAsset( + asset: asset, + fit: fit, + size: size, + thumbhashOnly: isScrubbing, + ), ), ), if (hasStack) diff --git a/mobile/pigeon/thumbnail_api.dart b/mobile/pigeon/thumbnail_api.dart index 545187ca92..26df871866 100644 --- a/mobile/pigeon/thumbnail_api.dart +++ b/mobile/pigeon/thumbnail_api.dart @@ -15,7 +15,7 @@ import 'package:pigeon/pigeon.dart'; @HostApi() abstract class ThumbnailApi { @async - void setThumbnailToBuffer( + Map setThumbnailToBuffer( int pointer, String assetId, { required int width,