draw to buffer

inline

scale video frame when possible

account for different dimensions

mobile/android/app/src/main/kotlin/app/alextran/immich/images/Thumbnails.g.kt

@@ -59,7 +59,7 @@ private open class ThumbnailsPigeonCodec : StandardMessageCodec() {
 
 /** Generated interface from Pigeon that represents a handler of messages from Flutter. */
 interface ThumbnailApi {
-  fun setThumbnailToBuffer(pointer: Long, assetId: String, width: Long, height: Long, callback: (Result<Unit>) -> Unit)
+  fun setThumbnailToBuffer(pointer: Long, assetId: String, width: Long, height: Long, callback: (Result<Map<String, Long>>) -> Unit)
 
   companion object {
     /** The codec used by ThumbnailApi. */
@@ -79,12 +79,13 @@ interface ThumbnailApi {
             val assetIdArg = args[1] as String
             val widthArg = args[2] as Long
             val heightArg = args[3] as Long
-            api.setThumbnailToBuffer(pointerArg, assetIdArg, widthArg, heightArg) { result: Result<Unit> ->
+            api.setThumbnailToBuffer(pointerArg, assetIdArg, widthArg, heightArg) { result: Result<Map<String, Long>> ->
               val error = result.exceptionOrNull()
               if (error != null) {
                 reply.reply(ThumbnailsPigeonUtils.wrapError(error))
               } else {
-                reply.reply(ThumbnailsPigeonUtils.wrapResult(null))
+                val data = result.getOrNull()
+                reply.reply(ThumbnailsPigeonUtils.wrapResult(data))
               }
             }
           }

mobile/android/app/src/main/kotlin/app/alextran/immich/images/ThumbnailsImpl.kt

@@ -96,16 +96,25 @@ class ThumbnailsImpl(context: Context) : ThumbnailApi {
   private fun decodeVideoThumbnail(assetId: String, targetWidth: Int, targetHeight: Int): Bitmap {
     val uri =
       ContentUris.withAppendedId(MediaStore.Video.Media.EXTERNAL_CONTENT_URI, assetId.toLong())
-    return if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
+    if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
-      contentResolver.loadThumbnail(uri, Size(targetWidth, targetHeight), null)
+      return contentResolver.loadThumbnail(uri, Size(targetWidth, targetHeight), null)
-    } else {
+    }
-      val retriever = MediaMetadataRetriever()
+
-      try {
+    val retriever = MediaMetadataRetriever()
-        retriever.setDataSource(ctx, uri)
+    try {
-        retriever.getFrameAtTime(0L) ?: throw RuntimeException("Failed to extract video frame")
+      retriever.setDataSource(ctx, uri)
-      } finally {
+      return if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) {
-        retriever.release()
+        retriever.getScaledFrameAtTime(
-      }
+          0L,
+          MediaMetadataRetriever.OPTION_NEXT_SYNC,
+          targetWidth,
+          targetHeight
+        )
+      } else {
+        retriever.getFrameAtTime(0L)
+      } ?: throw RuntimeException("Failed to extract video frame")
+    } finally {
+      retriever.release()
     }
   }
 

mobile/ios/Runner/Images/Thumbnails.g.swift

@@ -70,7 +70,7 @@ class ThumbnailsPigeonCodec: FlutterStandardMessageCodec, @unchecked Sendable {
 
 /// Generated protocol from Pigeon that represents a handler of messages from Flutter.
 protocol ThumbnailApi {
-  func setThumbnailToBuffer(pointer: Int64, assetId: String, width: Int64, height: Int64, completion: @escaping (Result<Void, Error>) -> Void)
+  func setThumbnailToBuffer(pointer: Int64, assetId: String, width: Int64, height: Int64, completion: @escaping (Result<[String: Int64], Error>) -> Void)
 }
 
 /// Generated setup class from Pigeon to handle messages through the `binaryMessenger`.
@@ -89,8 +89,8 @@ class ThumbnailApiSetup {
         let heightArg = args[3] as! Int64
         api.setThumbnailToBuffer(pointer: pointerArg, assetId: assetIdArg, width: widthArg, height: heightArg) { result in
           switch result {
-          case .success:
+          case .success(let res):
-            reply(wrapResult(nil))
+            reply(wrapResult(res))
           case .failure(let error):
             reply(wrapError(error))
           }

mobile/ios/Runner/Images/ThumbnailsImpl.swift

@@ -20,50 +20,36 @@ class ThumbnailApiImpl: ThumbnailApi {
     return requestOptions
   }()
   private static let processingQueue = DispatchQueue(label: "thumbnail.processing", qos: .userInteractive, attributes: .concurrent)
+  private static let rgbColorSpace = CGColorSpaceCreateDeviceRGB()
+  private static let bitmapInfo = CGBitmapInfo(rawValue: CGImageAlphaInfo.premultipliedLast.rawValue).rawValue
   
-  func setThumbnailToBuffer(pointer: Int64, assetId: String, width: Int64, height: Int64, completion: @escaping (Result<Void, any Error>) -> Void) {
+  func setThumbnailToBuffer(pointer: Int64, assetId: String, width: Int64, height: Int64, completion: @escaping (Result<[String: Int64], any Error>) -> Void) {
     guard let bufferPointer = UnsafeMutableRawPointer(bitPattern: Int(pointer))
     else { completion(.failure(PigeonError(code: "", message: "Could not get buffer pointer for \(assetId)", details: nil))); return }
     Self.processingQueue.async {
-      do {
+      guard let asset = PHAsset.fetchAssets(withLocalIdentifiers: [assetId], options: Self.fetchOptions).firstObject
-        let asset = try self.getAsset(assetId: assetId)
+      else { completion(.failure(PigeonError(code: "", message: "Could not get asset data for \(assetId)", details: nil))); return }
-        Self.cacheManager.requestImage(
+      Self.cacheManager.requestImage(
-          for: asset,
+        for: asset,
-          targetSize: CGSize(width: Double(width), height: Double(height)),
+        targetSize: CGSize(width: Double(width), height: Double(height)),
-          contentMode: .aspectFill,
+        contentMode: .aspectFill,
-          options: Self.requestOptions,
+        options: Self.requestOptions,
-          resultHandler: { (image, info) -> Void in
+        resultHandler: { (image, info) -> Void in
-            guard let image = image,
+          guard let image = image,
-                  let cgImage = image.cgImage,
+                let cgImage = image.cgImage,
-                  let dataProvider = cgImage.dataProvider,
+                let context = CGContext(
-                  let pixelData = dataProvider.data
+                  data: bufferPointer,
-            else { completion(.failure(PigeonError(code: "", message: "Could not get pixel data for \(assetId)", details: nil))); return }
+                  width: cgImage.width,
-            
+                  height: cgImage.height,
-            guard let sourceBuffer = CFDataGetBytePtr(pixelData)
+                  bitsPerComponent: 8,
-            else { completion(.failure(PigeonError(code: "", message: "Could not get pixel data buffer for \(assetId)", details: nil))); return }
+                  bytesPerRow: cgImage.width * 4,
-            let dataLength = CFDataGetLength(pixelData)
+                  space: Self.rgbColorSpace,
-            let bufferLength = width * height * 4
+                  bitmapInfo: Self.bitmapInfo
-            guard dataLength <= bufferLength
+                ) else { completion(.failure(PigeonError(code: "", message: "Could not get pixel data for \(assetId)", details: nil))); return }
-            else { completion(.failure(PigeonError(code: "", message: "Buffer is not large enough (\(bufferLength) vs \(dataLength) for \(assetId)", details: nil))); return }
+          context.draw(cgImage, in: CGRect(x: 0, y: 0, width: cgImage.width, height: cgImage.height))
-            
+          completion(.success(["width": Int64(cgImage.width), "height": Int64(cgImage.height)]))
-            bufferPointer.copyMemory(from: sourceBuffer, byteCount: dataLength)
+        }
-            completion(.success(()))
+      )
-          }
-        )
-      } catch {
-        completion(
-          .failure(PigeonError(code: "", message: "Could not get asset data for \(assetId)", details: nil)))
-      }
     }
   }
-  
-  private func getAsset(assetId: String) throws -> PHAsset {
-    guard
-      let asset = PHAsset.fetchAssets(withLocalIdentifiers: [assetId], options: Self.fetchOptions)
-        .firstObject
-    else {
-      throw PigeonError(code: "", message: "Could not fetch asset", details: nil)
-    }
-    return asset
-  }
 }

mobile/lib/platform/thumbnail_api.g.dart

@@ -51,7 +51,7 @@ class ThumbnailApi {
 
   final String pigeonVar_messageChannelSuffix;
 
-  Future<void> setThumbnailToBuffer(
+  Future<Map<String, int>> setThumbnailToBuffer(
     int pointer,
     String assetId, {
     required int width,
@@ -77,8 +77,14 @@ class ThumbnailApi {
         message: pigeonVar_replyList[1] as String?,
         details: pigeonVar_replyList[2],
       );
+    } else if (pigeonVar_replyList[0] == null) {
+      throw PlatformException(
+        code: 'null-error',
+        message: 'Host platform returned null value for non-null return value.',
+      );
     } else {
-      return;
+      return (pigeonVar_replyList[0] as Map<Object?, Object?>?)!
+          .cast<String, int>();
     }
   }
 }

mobile/lib/presentation/widgets/images/thumbnail.widget.dart

@@ -92,7 +92,7 @@ class _ThumbnailState extends State<Thumbnail> {
     if (oldWidget.blurhash != widget.blurhash ||
         oldWidget.localId != widget.localId ||
         oldWidget.remoteId != widget.remoteId ||
-        oldWidget.thumbhashOnly != widget.thumbhashOnly) {
+        (oldWidget.thumbhashOnly && !widget.thumbhashOnly)) {
       _decode();
     }
   }
@@ -104,18 +104,13 @@ class _ThumbnailState extends State<Thumbnail> {
 
     final thumbhashOnly = widget.thumbhashOnly;
     final blurhash = widget.blurhash;
-    final imageFuture = thumbhashOnly ? Future.value(null) : _decodeFromFile();
+    final imageFuture = thumbhashOnly ? Future.value(null) : _decodeThumbnail();
 
-    if (blurhash != null) {
+    if (blurhash != null && _image == null) {
-      final image = thumbhash.thumbHashToRGBA(base64.decode(blurhash));
       try {
-        await _decodeThumbhash(
+        await _decodeThumbhash();
-          await ImmutableBuffer.fromUint8List(image.rgba),
-          image.width,
-          image.height,
-        );
       } catch (e) {
-        log.info('Error decoding thumbhash for ${widget.remoteId}: $e');
+        log.severe('Error decoding thumbhash for ${widget.remoteId}: $e');
       }
     }
 
@@ -134,38 +129,32 @@ class _ThumbnailState extends State<Thumbnail> {
         _image = image;
       });
     } catch (e) {
-      log.info('Error decoding thumbnail: $e');
+      log.severe('Error decoding thumbnail: $e');
     }
   }
 
-  Future<void> _decodeThumbhash(
+  Future<void> _decodeThumbhash() async {
-    ImmutableBuffer buffer,
+    final blurhash = widget.blurhash;
-    int width,
+    if (blurhash == null || !mounted || _image != null) {
-    int height,
+      return;
-  ) async {
+    }
-    if (!mounted) {
+    final image = thumbhash.thumbHashToRGBA(base64.decode(blurhash));
+    final buffer = await ImmutableBuffer.fromUint8List(image.rgba);
+    if (!mounted || _image != null) {
       buffer.dispose();
       return;
     }
 
     final descriptor = ImageDescriptor.raw(
       buffer,
-      width: width,
+      width: image.width,
-      height: height,
+      height: image.height,
       pixelFormat: PixelFormat.rgba8888,
     );
-    if (!mounted) {
-      buffer.dispose();
-      descriptor.dispose();
-      return;
-    }
 
-    final codec = await descriptor.instantiateCodec(
+    final codec = await descriptor.instantiateCodec();
-      targetWidth: width,
-      targetHeight: height,
-    );
 
-    if (!mounted) {
+    if (!mounted || _image != null) {
       buffer.dispose();
       descriptor.dispose();
       codec.dispose();
@@ -176,7 +165,7 @@ class _ThumbnailState extends State<Thumbnail> {
     buffer.dispose();
     descriptor.dispose();
     codec.dispose();
-    if (!mounted) {
+    if (!mounted || _image != null) {
       frame.dispose();
       return;
     }
@@ -185,112 +174,110 @@ class _ThumbnailState extends State<Thumbnail> {
     });
   }
 
-  Future<ui.Image?> _decodeFromFile() async {
+  Future<ui.Image?> _decodeThumbnail() async {
-    final buffer = await _getFile();
+    if (!mounted) {
-    if (buffer == null) {
       return null;
     }
+
     final stopwatch = Stopwatch()..start();
-    final thumb = await _decodeThumbnail(buffer, 256, 256);
+    final codec = await _decodeThumb();
+    if (codec == null || !mounted) {
+      codec?.dispose();
+      return null;
+    }
+    final image = (await codec.getNextFrame()).image;
     stopwatch.stop();
-    return thumb;
+    log.info(
+      'Decoded thumbnail for ${widget.remoteId ?? widget.localId} in ${stopwatch.elapsedMilliseconds} ms',
+    );
+    return image;
   }
 
-  Future<ui.Image?> _decodeThumbnail(
+  Future<ui.Codec?> _decodeThumb() {
-    ImmutableBuffer buffer,
-    int width,
-    int height,
-  ) async {
-    if (!mounted) {
-      buffer.dispose();
-      return null;
-    }
-
-    final descriptor = ImageDescriptor.raw(
-      buffer,
-      width: width,
-      height: height,
-      pixelFormat: PixelFormat.rgba8888,
-    );
-
-    if (!mounted) {
-      buffer.dispose();
-      descriptor.dispose();
-      return null;
-    }
-
-    final codec = await descriptor.instantiateCodec(
-      targetWidth: width,
-      targetHeight: height,
-    );
-
-    if (!mounted) {
-      buffer.dispose();
-      descriptor.dispose();
-      codec.dispose();
-      return null;
-    }
-
-    final frame = (await codec.getNextFrame()).image;
-    buffer.dispose();
-    descriptor.dispose();
-    codec.dispose();
-    if (!mounted) {
-      frame.dispose();
-      return null;
-    }
-
-    return frame;
-  }
-
-  Future<ImmutableBuffer?> _getFile() async {
-    final stopwatch = Stopwatch()..start();
     final localId = widget.localId;
+    if (!mounted) {
+      return Future.value(null);
+    }
+    
     if (localId != null) {
-      final size = 256 * 256 * 4;
+      final size = widget.size;
-      final pointer = malloc<Uint8>(size);
+      final width = size.width.toInt();
-      try {
+      final height = size.height.toInt();
-        await thumbnailApi.setThumbnailToBuffer(
+      return _decodeLocal(localId, width, height);
-          pointer.address,
-          localId,
-          width: 256,
-          height: 256,
-        );
-        stopwatch.stop();
-        log.info(
-          'Retrieved local image $localId in ${stopwatch.elapsedMilliseconds.toStringAsFixed(2)} ms',
-        );
-        return ImmutableBuffer.fromUint8List(pointer.asTypedList(size));
-      } catch (e) {
-        log.warning('Failed to retrieve local image $localId: $e');
-      } finally {
-        malloc.free(pointer);
-      }
     }
 
     final remoteId = widget.remoteId;
     if (remoteId != null) {
-      final uri = getThumbnailUrlForRemoteId(remoteId);
+      return _decodeRemote(remoteId);
-      final headers = ApiService.getRequestHeaders();
+    }
-      final stream = _imageCache.getFileStream(
-        uri,
-        key: uri,
-        withProgress: true,
-        headers: headers,
-      );
 
-      await for (final result in stream) {
+    return Future.value(null);
+  }
+
+  Future<ui.Codec?> _decodeLocal(String localId, int width, int height) async {
+    final pointer = malloc<Uint8>(width * height * 4);
+
+    try {
+      final info = await thumbnailApi.setThumbnailToBuffer(
+        pointer.address,
+        localId,
+        width: width,
+        height: height,
+      );
+      if (!mounted) {
+        return null;
+      }
+      final actualWidth = info['width']!;
+      final actualHeight = info['height']!;
+      final actualSize = actualWidth * actualHeight * 4;
+      final buffer =
+          await ImmutableBuffer.fromUint8List(pointer.asTypedList(actualSize));
+      if (!mounted) {
+        buffer.dispose();
+        return null;
+      }
+      final descriptor = ui.ImageDescriptor.raw(
+        buffer,
+        width: actualWidth,
+        height: actualHeight,
+        pixelFormat: ui.PixelFormat.rgba8888,
+      );
+      return await descriptor.instantiateCodec();
+    } catch (e) {
+      return null;
+    } finally {
+      malloc.free(pointer);
+    }
+  }
+
+  Future<ui.Codec?> _decodeRemote(String remoteId) async {
+    final uri = getThumbnailUrlForRemoteId(remoteId);
+    final headers = ApiService.getRequestHeaders();
+    final stream = _imageCache.getFileStream(
+      uri,
+      key: uri,
+      withProgress: true,
+      headers: headers,
+    );
+
+    await for (final result in stream) {
+      if (!mounted) {
+        return null;
+      }
+
+      if (result is FileInfo) {
+        final buffer = await ImmutableBuffer.fromFilePath(result.file.path);
         if (!mounted) {
+          buffer.dispose();
           return null;
         }
-
+        final descriptor = await ImageDescriptor.encoded(buffer);
-        if (result is FileInfo) {
+        if (!mounted) {
-          stopwatch.stop();
+          buffer.dispose();
-          log.info(
+          descriptor.dispose();
-            'Retrieved remote image $remoteId in ${stopwatch.elapsedMilliseconds.toStringAsFixed(2)} ms',
+          return null;
-          );
-          return ImmutableBuffer.fromFilePath(result.file.path);
         }
+        return await descriptor.instantiateCodec();
       }
     }
 

mobile/lib/presentation/widgets/images/thumbnail_tile.widget.dart

@@ -8,6 +8,7 @@ import 'package:immich_mobile/extensions/duration_extensions.dart';
 import 'package:immich_mobile/extensions/theme_extensions.dart';
 import 'package:immich_mobile/presentation/widgets/images/thumbnail.widget.dart';
 import 'package:immich_mobile/providers/infrastructure/setting.provider.dart';
+import 'package:immich_mobile/presentation/widgets/timeline/timeline.state.dart';
 import 'package:immich_mobile/providers/timeline/multiselect.provider.dart';
 
 class ThumbnailTile extends ConsumerWidget {
@@ -40,6 +41,8 @@ class ThumbnailTile extends ConsumerWidget {
     final isSelected = ref.watch(
       multiSelectProvider.select((multiselect) => multiselect.selectedAssets.contains(asset)),
     );
+    final isScrubbing =
+        ref.watch(timelineStateProvider.select((state) => state.isScrubbing));
 
     final borderStyle = lockSelection
         ? BoxDecoration(
@@ -73,7 +76,12 @@ class ThumbnailTile extends ConsumerWidget {
                 Positioned.fill(
                   child: Hero(
                     tag: '${asset?.heroTag ?? ''}_$heroIndex',
-                    child: Thumbnail.fromBaseAsset(asset: asset, fit: fit, size: size),
+                    child: Thumbnail.fromBaseAsset(
+                      asset: asset,
+                      fit: fit,
+                      size: size,
+                      thumbhashOnly: isScrubbing,
+                    ),
                   ),
                 ),
                 if (hasStack)

mobile/pigeon/thumbnail_api.dart

@@ -15,7 +15,7 @@ import 'package:pigeon/pigeon.dart';
 @HostApi()
 abstract class ThumbnailApi {
   @async
-  void setThumbnailToBuffer(
+  Map<String, int> setThumbnailToBuffer(
     int pointer,
     String assetId, {
     required int width,