From 86ee4ff822aa82d9cc5b8a31f6695babb4fec7e6 Mon Sep 17 00:00:00 2001 From: mertalev <101130780+mertalev@users.noreply.github.com> Date: Wed, 6 Aug 2025 23:57:33 -0400 Subject: [PATCH] request cancellation --- .../app/alextran/immich/AppGlideModule.kt | 1 - .../alextran/immich/images/Thumbnails.g.kt | 32 ++- .../alextran/immich/images/ThumbnailsImpl.kt | 197 ++++++++++------ mobile/ios/Runner/Images/Thumbnails.g.swift | 33 ++- mobile/ios/Runner/Images/ThumbnailsImpl.swift | 68 +++++- mobile/lib/constants/constants.dart | 4 +- .../repositories/asset_media.repository.dart | 210 ++++++++++++++++-- mobile/lib/platform/thumbnail_api.g.dart | 36 ++- .../widgets/images/image_provider.dart | 20 +- .../widgets/images/local_image_provider.dart | 46 ++-- .../widgets/images/remote_image_provider.dart | 74 +++--- .../widgets/images/thumbnail.widget.dart | 5 + .../widgets/images/thumbnail_tile.widget.dart | 4 +- mobile/pigeon/thumbnail_api.dart | 8 +- 14 files changed, 558 insertions(+), 180 deletions(-) diff --git a/mobile/android/app/src/main/kotlin/app/alextran/immich/AppGlideModule.kt b/mobile/android/app/src/main/kotlin/app/alextran/immich/AppGlideModule.kt index d87ab3b63c..7b589cd80f 100644 --- a/mobile/android/app/src/main/kotlin/app/alextran/immich/AppGlideModule.kt +++ b/mobile/android/app/src/main/kotlin/app/alextran/immich/AppGlideModule.kt @@ -5,7 +5,6 @@ import com.bumptech.glide.GlideBuilder import com.bumptech.glide.annotation.GlideModule import com.bumptech.glide.load.engine.bitmap_recycle.BitmapPoolAdapter import com.bumptech.glide.load.engine.cache.DiskCacheAdapter -import com.bumptech.glide.load.engine.cache.MemoryCache import com.bumptech.glide.load.engine.cache.MemoryCacheAdapter import com.bumptech.glide.module.AppGlideModule diff --git a/mobile/android/app/src/main/kotlin/app/alextran/immich/images/Thumbnails.g.kt b/mobile/android/app/src/main/kotlin/app/alextran/immich/images/Thumbnails.g.kt index a70de4300b..4866fb648f 100644 --- a/mobile/android/app/src/main/kotlin/app/alextran/immich/images/Thumbnails.g.kt +++ b/mobile/android/app/src/main/kotlin/app/alextran/immich/images/Thumbnails.g.kt @@ -1,4 +1,4 @@ -// Autogenerated from Pigeon (v25.3.2), do not edit directly. +// Autogenerated from Pigeon (v26.0.0), do not edit directly. // See also: https://pub.dev/packages/pigeon @file:Suppress("UNCHECKED_CAST", "ArrayInDataClass") @@ -59,7 +59,8 @@ private open class ThumbnailsPigeonCodec : StandardMessageCodec() { /** Generated interface from Pigeon that represents a handler of messages from Flutter. */ interface ThumbnailApi { - fun getThumbnailBuffer(assetId: String, width: Long, height: Long, callback: (Result>) -> Unit) + fun requestImage(assetId: String, requestId: Long, width: Long, height: Long, callback: (Result>) -> Unit) + fun cancelImageRequest(requestId: Long) companion object { /** The codec used by ThumbnailApi. */ @@ -71,14 +72,15 @@ interface ThumbnailApi { fun setUp(binaryMessenger: BinaryMessenger, api: ThumbnailApi?, messageChannelSuffix: String = "") { val separatedMessageChannelSuffix = if (messageChannelSuffix.isNotEmpty()) ".$messageChannelSuffix" else "" run { - val channel = BasicMessageChannel(binaryMessenger, "dev.flutter.pigeon.immich_mobile.ThumbnailApi.getThumbnailBuffer$separatedMessageChannelSuffix", codec) + val channel = BasicMessageChannel(binaryMessenger, "dev.flutter.pigeon.immich_mobile.ThumbnailApi.requestImage$separatedMessageChannelSuffix", codec) if (api != null) { channel.setMessageHandler { message, reply -> val args = message as List val assetIdArg = args[0] as String - val widthArg = args[1] as Long - val heightArg = args[2] as Long - api.getThumbnailBuffer(assetIdArg, widthArg, heightArg) { result: Result> -> + val requestIdArg = args[1] as Long + val widthArg = args[2] as Long + val heightArg = args[3] as Long + api.requestImage(assetIdArg, requestIdArg, widthArg, heightArg) { result: Result> -> val error = result.exceptionOrNull() if (error != null) { reply.reply(ThumbnailsPigeonUtils.wrapError(error)) @@ -92,6 +94,24 @@ interface ThumbnailApi { channel.setMessageHandler(null) } } + run { + val channel = BasicMessageChannel(binaryMessenger, "dev.flutter.pigeon.immich_mobile.ThumbnailApi.cancelImageRequest$separatedMessageChannelSuffix", codec) + if (api != null) { + channel.setMessageHandler { message, reply -> + val args = message as List + val requestIdArg = args[0] as Long + val wrapped: List = try { + api.cancelImageRequest(requestIdArg) + listOf(null) + } catch (exception: Throwable) { + ThumbnailsPigeonUtils.wrapError(exception) + } + reply.reply(wrapped) + } + } else { + channel.setMessageHandler(null) + } + } } } } diff --git a/mobile/android/app/src/main/kotlin/app/alextran/immich/images/ThumbnailsImpl.kt b/mobile/android/app/src/main/kotlin/app/alextran/immich/images/ThumbnailsImpl.kt index 804cdcb3c1..168e504961 100644 --- a/mobile/android/app/src/main/kotlin/app/alextran/immich/images/ThumbnailsImpl.kt +++ b/mobile/android/app/src/main/kotlin/app/alextran/immich/images/ThumbnailsImpl.kt @@ -6,6 +6,8 @@ import android.content.Context import android.graphics.* import android.net.Uri import android.os.Build +import android.os.CancellationSignal +import android.os.OperationCanceledException import android.provider.MediaStore import android.provider.MediaStore.Images import android.provider.MediaStore.Video @@ -16,13 +18,23 @@ import java.util.concurrent.Executors import com.bumptech.glide.Glide import com.bumptech.glide.Priority import com.bumptech.glide.load.DecodeFormat -import kotlin.time.TimeSource +import java.util.HashMap +import java.util.concurrent.CancellationException +import java.util.concurrent.Future + +data class Request( + val requestId: Long, + val taskFuture: Future<*>, + val cancellationSignal: CancellationSignal, + val callback: (Result>) -> Unit +) class ThumbnailsImpl(context: Context) : ThumbnailApi { private val ctx: Context = context.applicationContext - private val contentResolver: ContentResolver = ctx.contentResolver + private val resolver: ContentResolver = ctx.contentResolver private val threadPool = - Executors.newFixedThreadPool(max(4, Runtime.getRuntime().availableProcessors())) + Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors() / 2 + 1) + private val requestMap = HashMap() companion object { val PROJECTION = arrayOf( @@ -34,6 +46,8 @@ class ThumbnailsImpl(context: Context) : ThumbnailApi { const val MEDIA_TYPE_IMAGE = MediaStore.Files.FileColumns.MEDIA_TYPE_IMAGE const val MEDIA_TYPE_VIDEO = MediaStore.Files.FileColumns.MEDIA_TYPE_VIDEO + val CANCELLED = Result.success>(mapOf()) + val OPTIONS = BitmapFactory.Options().apply { inPreferredConfig = Bitmap.Config.ARGB_8888 } init { System.loadLibrary("native_buffer") @@ -49,27 +63,55 @@ class ThumbnailsImpl(context: Context) : ThumbnailApi { external fun wrapAsBuffer(address: Long, capacity: Int): ByteBuffer } - override fun getThumbnailBuffer( - assetId: String, width: Long, height: Long, callback: (Result>) -> Unit + override fun requestImage( + assetId: String, + requestId: Long, + width: Long, + height: Long, + callback: (Result>) -> Unit ) { - threadPool.execute { + val signal = CancellationSignal() + val task = threadPool.submit { try { - getThumbnailBufferInternal(assetId, width, height, callback) + getThumbnailBufferInternal(assetId, width, height, callback, signal) } catch (e: Exception) { - callback(Result.failure(e)) + when (e) { + is OperationCanceledException -> callback(CANCELLED) + is CancellationException -> callback(CANCELLED) + else -> callback(Result.failure(e)) + } + } finally { + requestMap.remove(requestId) } } + requestMap[requestId] = Request(requestId, task, signal, callback) + } + + override fun cancelImageRequest(requestId: Long) { + val request = requestMap.remove(requestId) ?: return + request.taskFuture.cancel(false) + request.cancellationSignal.cancel() + if (request.taskFuture.isCancelled) { + request.callback(CANCELLED) + } } private fun getThumbnailBufferInternal( - assetId: String, width: Long, height: Long, callback: (Result>) -> Unit + assetId: String, + width: Long, + height: Long, + callback: (Result>) -> Unit, + signal: CancellationSignal ) { + signal.throwIfCanceled() val targetWidth = width.toInt() val targetHeight = height.toInt() + val id = assetId.toLong() - val cursor = contentResolver.query(URI, PROJECTION, SELECTION, arrayOf(assetId), null) + val cursor = resolver.query(URI, PROJECTION, SELECTION, arrayOf(assetId), null) ?: return callback(Result.failure(RuntimeException("Asset not found"))) + signal.throwIfCanceled() cursor.use { c -> if (!c.moveToNext()) { return callback(Result.failure(RuntimeException("Asset not found"))) @@ -77,77 +119,90 @@ class ThumbnailsImpl(context: Context) : ThumbnailApi { val mediaType = c.getInt(1) val bitmap = when (mediaType) { - MEDIA_TYPE_IMAGE -> decodeImage(assetId, targetWidth, targetHeight) - MEDIA_TYPE_VIDEO -> decodeVideoThumbnail(assetId, targetWidth, targetHeight) + MEDIA_TYPE_IMAGE -> decodeImage(id, targetWidth, targetHeight, signal) + MEDIA_TYPE_VIDEO -> decodeVideoThumbnail(id, targetWidth, targetHeight, signal) else -> return callback(Result.failure(RuntimeException("Unsupported media type"))) } - val actualWidth = bitmap.width - val actualHeight = bitmap.height - - val size = actualWidth * actualHeight * 4 - val pointer = allocateNative(size) - try { - val buffer = wrapAsBuffer(pointer, size) - bitmap.copyPixelsToBuffer(buffer) - bitmap.recycle() - callback( - Result.success( - mapOf( - "pointer" to pointer, - "width" to actualWidth.toLong(), - "height" to actualHeight.toLong() - ) - ) - ) - } catch (e: Exception) { - freeNative(pointer) - callback(Result.failure(e)) - } + processBitmap(bitmap, callback, signal) } } - private fun decodeImage(assetId: String, targetWidth: Int, targetHeight: Int): Bitmap { - val uri = ContentUris.withAppendedId(Images.Media.EXTERNAL_CONTENT_URI, assetId.toLong()) + private fun processBitmap( + bitmap: Bitmap, + callback: (Result>) -> Unit, + signal: CancellationSignal + ) { + signal.throwIfCanceled() + val actualWidth = bitmap.width + val actualHeight = bitmap.height - if (targetHeight <= 768 && targetWidth <= 768) { - return if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { - contentResolver.loadThumbnail(uri, Size(targetWidth, targetHeight), null) - } else { - Images.Thumbnails.getThumbnail( - contentResolver, - assetId.toLong(), - Images.Thumbnails.MINI_KIND, - BitmapFactory.Options().apply { - inPreferredConfig = Bitmap.Config.ARGB_8888 - } - ) - } - } + val size = actualWidth * actualHeight * 4 + val pointer = allocateNative(size) - return decodeSource(uri, targetWidth, targetHeight) - } - - private fun decodeVideoThumbnail(assetId: String, targetWidth: Int, targetHeight: Int): Bitmap { - return if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { - val uri = ContentUris.withAppendedId(Video.Media.EXTERNAL_CONTENT_URI, assetId.toLong()) - contentResolver.loadThumbnail(uri, Size(targetWidth, targetHeight), null) - } else { - Video.Thumbnails.getThumbnail( - contentResolver, - assetId.toLong(), - Video.Thumbnails.MINI_KIND, - BitmapFactory.Options().apply { - inPreferredConfig = Bitmap.Config.ARGB_8888 - } + try { + signal.throwIfCanceled() + val buffer = wrapAsBuffer(pointer, size) + bitmap.copyPixelsToBuffer(buffer) + signal.throwIfCanceled() + val res = mapOf( + "pointer" to pointer, + "width" to actualWidth.toLong(), + "height" to actualHeight.toLong() ) + callback(Result.success(res)) + } catch (e: Exception) { + freeNative(pointer) + callback(if (e is OperationCanceledException) CANCELLED else Result.failure(e)) } } - private fun decodeSource(uri: Uri, targetWidth: Int, targetHeight: Int): Bitmap { - return if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { - val source = ImageDecoder.createSource(contentResolver, uri) + private fun decodeImage( + id: Long, + targetWidth: Int, + targetHeight: Int, + signal: CancellationSignal + ): Bitmap { + signal.throwIfCanceled() + val uri = ContentUris.withAppendedId(Images.Media.EXTERNAL_CONTENT_URI, id) + if (targetHeight > 768 || targetWidth > 768) { + return decodeSource(uri, targetWidth, targetHeight, signal) + } + return if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { + resolver.loadThumbnail(uri, Size(targetWidth, targetHeight), signal) + } else { + signal.setOnCancelListener { Images.Thumbnails.cancelThumbnailRequest(resolver, id) } + Images.Thumbnails.getThumbnail(resolver, id, Images.Thumbnails.MINI_KIND, OPTIONS) + } + } + + private fun decodeVideoThumbnail( + id: Long, + targetWidth: Int, + targetHeight: Int, + signal: CancellationSignal + ): Bitmap { + signal.throwIfCanceled() + return if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { + val uri = ContentUris.withAppendedId(Video.Media.EXTERNAL_CONTENT_URI, id) + resolver.loadThumbnail(uri, Size(targetWidth, targetHeight), signal) + } else { + signal.setOnCancelListener { Video.Thumbnails.cancelThumbnailRequest(resolver, id) } + Video.Thumbnails.getThumbnail(resolver, id, Video.Thumbnails.MINI_KIND, OPTIONS) + } + } + + private fun decodeSource( + uri: Uri, + targetWidth: Int, + targetHeight: Int, + signal: CancellationSignal + ): Bitmap { + signal.throwIfCanceled() + return if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { + val source = ImageDecoder.createSource(resolver, uri) + signal.throwIfCanceled() ImageDecoder.decodeBitmap(source) { decoder, info, _ -> val sampleSize = getSampleSize(info.size.width, info.size.height, targetWidth, targetHeight) @@ -156,13 +211,15 @@ class ThumbnailsImpl(context: Context) : ThumbnailApi { decoder.setTargetColorSpace(ColorSpace.get(ColorSpace.Named.SRGB)) } } else { - Glide.with(ctx) + val ref = Glide.with(ctx) .asBitmap() .priority(Priority.IMMEDIATE) .load(uri) .disallowHardwareConfig() .format(DecodeFormat.PREFER_ARGB_8888) - .submit(targetWidth, targetHeight).get() + .submit(targetWidth, targetHeight) + signal.setOnCancelListener { Glide.with(ctx).clear(ref) } + ref.get() } } diff --git a/mobile/ios/Runner/Images/Thumbnails.g.swift b/mobile/ios/Runner/Images/Thumbnails.g.swift index 6fbf28a474..f35ded0775 100644 --- a/mobile/ios/Runner/Images/Thumbnails.g.swift +++ b/mobile/ios/Runner/Images/Thumbnails.g.swift @@ -1,4 +1,4 @@ -// Autogenerated from Pigeon (v25.3.2), do not edit directly. +// Autogenerated from Pigeon (v26.0.0), do not edit directly. // See also: https://pub.dev/packages/pigeon import Foundation @@ -70,7 +70,8 @@ class ThumbnailsPigeonCodec: FlutterStandardMessageCodec, @unchecked Sendable { /// Generated protocol from Pigeon that represents a handler of messages from Flutter. protocol ThumbnailApi { - func getThumbnailBuffer(assetId: String, width: Int64, height: Int64, completion: @escaping (Result<[String: Int64], Error>) -> Void) + func requestImage(assetId: String, requestId: Int64, width: Int64, height: Int64, completion: @escaping (Result<[String: Int64], Error>) -> Void) + func cancelImageRequest(requestId: Int64) throws } /// Generated setup class from Pigeon to handle messages through the `binaryMessenger`. @@ -79,14 +80,15 @@ class ThumbnailApiSetup { /// Sets up an instance of `ThumbnailApi` to handle messages through the `binaryMessenger`. static func setUp(binaryMessenger: FlutterBinaryMessenger, api: ThumbnailApi?, messageChannelSuffix: String = "") { let channelSuffix = messageChannelSuffix.count > 0 ? ".\(messageChannelSuffix)" : "" - let getThumbnailBufferChannel = FlutterBasicMessageChannel(name: "dev.flutter.pigeon.immich_mobile.ThumbnailApi.getThumbnailBuffer\(channelSuffix)", binaryMessenger: binaryMessenger, codec: codec) + let requestImageChannel = FlutterBasicMessageChannel(name: "dev.flutter.pigeon.immich_mobile.ThumbnailApi.requestImage\(channelSuffix)", binaryMessenger: binaryMessenger, codec: codec) if let api = api { - getThumbnailBufferChannel.setMessageHandler { message, reply in + requestImageChannel.setMessageHandler { message, reply in let args = message as! [Any?] let assetIdArg = args[0] as! String - let widthArg = args[1] as! Int64 - let heightArg = args[2] as! Int64 - api.getThumbnailBuffer(assetId: assetIdArg, width: widthArg, height: heightArg) { result in + let requestIdArg = args[1] as! Int64 + let widthArg = args[2] as! Int64 + let heightArg = args[3] as! Int64 + api.requestImage(assetId: assetIdArg, requestId: requestIdArg, width: widthArg, height: heightArg) { result in switch result { case .success(let res): reply(wrapResult(res)) @@ -96,7 +98,22 @@ class ThumbnailApiSetup { } } } else { - getThumbnailBufferChannel.setMessageHandler(nil) + requestImageChannel.setMessageHandler(nil) + } + let cancelImageRequestChannel = FlutterBasicMessageChannel(name: "dev.flutter.pigeon.immich_mobile.ThumbnailApi.cancelImageRequest\(channelSuffix)", binaryMessenger: binaryMessenger, codec: codec) + if let api = api { + cancelImageRequestChannel.setMessageHandler { message, reply in + let args = message as! [Any?] + let requestIdArg = args[0] as! Int64 + do { + try api.cancelImageRequest(requestId: requestIdArg) + reply(wrapResult(nil)) + } catch { + reply(wrapError(error)) + } + } + } else { + cancelImageRequestChannel.setMessageHandler(nil) } } } diff --git a/mobile/ios/Runner/Images/ThumbnailsImpl.swift b/mobile/ios/Runner/Images/ThumbnailsImpl.swift index b774572268..c99103381b 100644 --- a/mobile/ios/Runner/Images/ThumbnailsImpl.swift +++ b/mobile/ios/Runner/Images/ThumbnailsImpl.swift @@ -3,6 +3,13 @@ import Flutter import MobileCoreServices import Photos +struct Request { + var managerId: Int32? + var workItem: DispatchWorkItem? + var isCancelled = false + let callback: (Result<[String: Int64], any Error>) -> Void +} + class ThumbnailApiImpl: ThumbnailApi { private static let cacheManager = PHImageManager.default() private static let fetchOptions = { @@ -22,29 +29,45 @@ class ThumbnailApiImpl: ThumbnailApi { private static let processingQueue = DispatchQueue(label: "thumbnail.processing", qos: .userInteractive, attributes: .concurrent) private static let rgbColorSpace = CGColorSpaceCreateDeviceRGB() private static let bitmapInfo = CGBitmapInfo(rawValue: CGImageAlphaInfo.premultipliedLast.rawValue).rawValue + private static var requests = [Int64: Request]() + private static let cancelledResult = Result<[String: Int64], any Error>.success([:]) - func getThumbnailBuffer(assetId: String, width: Int64, height: Int64, completion: @escaping (Result<[String: Int64], any Error>) -> Void) { - Self.processingQueue.async { + func requestImage(assetId: String, requestId: Int64, width: Int64, height: Int64, completion: @escaping (Result<[String: Int64], any Error>) -> Void) { + var request = Request(callback: completion) + let item = DispatchWorkItem { guard let asset = PHAsset.fetchAssets(withLocalIdentifiers: [assetId], options: Self.fetchOptions).firstObject - else { completion(.failure(PigeonError(code: "", message: "Could not get asset data for \(assetId)", details: nil))); return } + else { + Self.requests[requestId] = nil + completion(.failure(PigeonError(code: "", message: "Could not get asset data for \(assetId)", details: nil))) + return + } - Self.cacheManager.requestImage( + request.managerId = Self.cacheManager.requestImage( for: asset, targetSize: CGSize(width: Double(width), height: Double(height)), contentMode: .aspectFit, options: Self.requestOptions, resultHandler: { (image, info) -> Void in + defer { Self.requests[requestId] = nil } guard let image = image, let cgImage = image.cgImage else { - completion(.failure(PigeonError(code: "", message: "Could not get pixel data for \(assetId)", details: nil))) - return + return completion(.failure(PigeonError(code: "", message: "Could not get pixel data for \(assetId)", details: nil))) } - + + if request.isCancelled { + return completion(Self.cancelledResult) + } + let pointer = UnsafeMutableRawPointer.allocate( byteCount: Int(cgImage.width) * Int(cgImage.height) * 4, alignment: MemoryLayout.alignment ) + if request.isCancelled { + pointer.deallocate() + return completion(Self.cancelledResult) + } + guard let context = CGContext( data: pointer, width: cgImage.width, @@ -55,14 +78,41 @@ class ThumbnailApiImpl: ThumbnailApi { bitmapInfo: Self.bitmapInfo ) else { pointer.deallocate() - completion(.failure(PigeonError(code: "", message: "Could not create context for \(assetId)", details: nil))) - return + return completion(.failure(PigeonError(code: "", message: "Could not create context for \(assetId)", details: nil))) } + + if request.isCancelled { + pointer.deallocate() + return completion(Self.cancelledResult) + } + context.interpolationQuality = .none context.draw(cgImage, in: CGRect(x: 0, y: 0, width: cgImage.width, height: cgImage.height)) + + if request.isCancelled { + pointer.deallocate() + return completion(Self.cancelledResult) + } + completion(.success(["pointer": Int64(Int(bitPattern: pointer)), "width": Int64(cgImage.width), "height": Int64(cgImage.height)])) } ) } + + request.workItem = item + Self.requests[requestId] = request + Self.processingQueue.async(execute: item) + } + + func cancelImageRequest(requestId: Int64) { + guard var request = Self.requests.removeValue(forKey: requestId) else { return } + request.isCancelled = true + guard let item = request.workItem else { return } + item.cancel() + if item.isCancelled { + request.callback(Self.cancelledResult) + } else if let managerId = request.managerId { + Self.cacheManager.cancelImageRequest(managerId) + } } } diff --git a/mobile/lib/constants/constants.dart b/mobile/lib/constants/constants.dart index 52d4100dc1..75e3cf207d 100644 --- a/mobile/lib/constants/constants.dart +++ b/mobile/lib/constants/constants.dart @@ -31,8 +31,8 @@ const int kTimelineNoneSegmentSize = 120; const int kTimelineAssetLoadBatchSize = 1024; const int kTimelineAssetLoadOppositeSize = 64; const Size kTimelineThumbnailTileSize = Size.square(256.0); -const Size kTimelineThumbnailSize = Size.square(384.0); -const int kTimelineImageCacheMemory = 250 * 1024 * 1024; +const Size kTimelineThumbnailSize = Size.square(256.0); +const int kTimelineImageCacheMemory = 200 * 1024 * 1024; // Widget keys const String appShareGroupId = "group.app.immich.share"; diff --git a/mobile/lib/infrastructure/repositories/asset_media.repository.dart b/mobile/lib/infrastructure/repositories/asset_media.repository.dart index dd8cc244c8..2cace0d7ea 100644 --- a/mobile/lib/infrastructure/repositories/asset_media.repository.dart +++ b/mobile/lib/infrastructure/repositories/asset_media.repository.dart @@ -1,45 +1,211 @@ +import 'dart:async'; import 'dart:ffi'; -import 'dart:typed_data'; -import 'dart:ui'; +import 'dart:io'; import 'dart:ui' as ui; +import 'package:flutter/material.dart'; +import 'package:flutter/services.dart'; +import 'package:flutter/foundation.dart'; +import 'package:immich_mobile/providers/image/cache/remote_image_cache_manager.dart'; import 'package:immich_mobile/providers/infrastructure/platform.provider.dart'; -import 'package:photo_manager/photo_manager.dart'; import 'package:ffi/ffi.dart'; +import 'package:immich_mobile/services/api.service.dart'; +import 'package:logging/logging.dart'; -class AssetMediaRepository { - const AssetMediaRepository(); +abstract class ImageRequest { + static int _nextRequestId = 0; - Future getLocalThumbnail(String localId, ui.Size size) async { - final info = await thumbnailApi.getThumbnailBuffer(localId, width: size.width.toInt(), height: size.height.toInt()); + final int requestId = _nextRequestId++; + bool _isCancelled = false; - final pointer = Pointer.fromAddress(info['pointer']!); - final actualWidth = info['width']!; - final actualHeight = info['height']!; - final actualSize = actualWidth * actualHeight * 4; + get isCancelled => _isCancelled; + ImageRequest(); + + Future load(ImageDecoderCallback decode, {double scale = 1.0}); + + void cancel() { + if (isCancelled) { + return; + } + _isCancelled = true; + return _onCancelled(); + } + + void _onCancelled(); +} + +class LocalImageRequest extends ImageRequest { + final String localId; + final int width; + final int height; + + LocalImageRequest({required this.localId, required ui.Size size}) + : width = size.width.toInt(), + height = size.height.toInt(); + + @override + Future load(ImageDecoderCallback decode, {double scale = 1.0}) async { + if (_isCancelled) { + return null; + } + + final Map info = await thumbnailApi.requestImage( + localId, + requestId: requestId, + width: width, + height: height, + ); + + final address = info['pointer']; + if (address == null) { + return null; + } + + final pointer = Pointer.fromAddress(address); try { + if (_isCancelled) { + return null; + } + + final actualWidth = info['width']!; + final actualHeight = info['height']!; + final actualSize = actualWidth * actualHeight * 4; + final buffer = await ImmutableBuffer.fromUint8List(pointer.asTypedList(actualSize)); + if (_isCancelled) { + return null; + } + final descriptor = ui.ImageDescriptor.raw( buffer, width: actualWidth, height: actualHeight, pixelFormat: ui.PixelFormat.rgba8888, ); - return await descriptor.instantiateCodec(); + final codec = await descriptor.instantiateCodec(); + if (_isCancelled) { + return null; + } + + final frame = await codec.getNextFrame(); + return ImageInfo(image: frame.image, scale: scale); } finally { malloc.free(pointer); } } - Future getThumbnail(String id, {int quality = 80, ui.Size size = const ui.Size.square(256)}) => - AssetEntity( - id: id, - // The below fields are not used in thumbnailDataWithSize but are required - // to create an AssetEntity instance. It is faster to create a dummy AssetEntity - // instance than to fetch the asset from the device first. - typeInt: AssetType.image.index, - width: size.width.toInt(), - height: size.height.toInt(), - ).thumbnailDataWithSize(ThumbnailSize(size.width.toInt(), size.height.toInt()), quality: quality); + @override + Future _onCancelled() { + return thumbnailApi.cancelImageRequest(requestId); + } +} + +class RemoteImageRequest extends ImageRequest { + static final log = Logger('RemoteImageRequest'); + static final cacheManager = RemoteImageCacheManager(); + static final client = HttpClient(); + String uri; + Map headers; + HttpClientRequest? _request; + + RemoteImageRequest({required this.uri, required this.headers}); + + @override + Future load(ImageDecoderCallback decode, {double scale = 1.0}) async { + if (_isCancelled) { + return null; + } + + try { + // The DB calls made by the cache manager are a *massive* bottleneck (6+ seconds) with high concurrency. + // Since it isn't possible to cancel these operations, we only prefer the cache when they can be avoided. + // The DB hit is left as a fallback for offline use. + final cachedFileBuffer = await _loadCachedFile(uri, inMemoryOnly: true); + if (cachedFileBuffer != null) { + return _decodeBuffer(cachedFileBuffer, decode, scale); + } + + final buffer = await _downloadImage(uri); + if (buffer == null || _isCancelled) { + return null; + } + return await _decodeBuffer(buffer, decode, scale); + } catch (e) { + if (e is HttpException && (e.message.endsWith('aborted') || e.message.startsWith('Connection closed'))) { + return null; + } + log.severe('Failed to load remote image', e); + final buffer = await _loadCachedFile(uri, inMemoryOnly: false); + if (buffer != null) { + return _decodeBuffer(buffer, decode, scale); + } + rethrow; + } finally { + _request = null; + } + } + + Future _downloadImage(String url) async { + final request = _request = await client.getUrl(Uri.parse(url)); + if (_isCancelled) { + return null; + } + + final headers = ApiService.getRequestHeaders(); + for (final entry in headers.entries) { + request.headers.set(entry.key, entry.value); + } + final response = await request.close(); + if (_isCancelled) { + return null; + } + + final bytes = await consolidateHttpClientResponseBytes(response); + _cacheFile(url, bytes); + if (_isCancelled) { + return null; + } + return await ImmutableBuffer.fromUint8List(bytes); + } + + Future _cacheFile(String url, Uint8List bytes) async { + try { + await cacheManager.putFile(url, bytes); + } catch (e) { + log.severe('Failed to cache image', e); + } + } + + Future _loadCachedFile(String url, {required bool inMemoryOnly}) async { + if (_isCancelled) { + return null; + } + final file = await (inMemoryOnly ? cacheManager.getFileFromMemory(url) : cacheManager.getFileFromCache(url)); + if (_isCancelled || file == null) { + return null; + } + return await ImmutableBuffer.fromFilePath(file.file.path); + } + + Future _decodeBuffer(ImmutableBuffer buffer, ImageDecoderCallback decode, scale) async { + if (_isCancelled) { + buffer.dispose(); + return null; + } + final codec = await decode(buffer); + if (_isCancelled) { + buffer.dispose(); + codec.dispose(); + return null; + } + final frame = await codec.getNextFrame(); + return ImageInfo(image: frame.image, scale: scale); + } + + @override + void _onCancelled() { + _request?.abort(); + _request = null; + } } diff --git a/mobile/lib/platform/thumbnail_api.g.dart b/mobile/lib/platform/thumbnail_api.g.dart index de598d3411..7a4fee9ba6 100644 --- a/mobile/lib/platform/thumbnail_api.g.dart +++ b/mobile/lib/platform/thumbnail_api.g.dart @@ -1,4 +1,4 @@ -// Autogenerated from Pigeon (v25.3.2), do not edit directly. +// Autogenerated from Pigeon (v26.0.0), do not edit directly. // See also: https://pub.dev/packages/pigeon // ignore_for_file: public_member_api_docs, non_constant_identifier_names, avoid_as, unused_import, unnecessary_parenthesis, prefer_null_aware_operators, omit_local_variable_types, unused_shown_name, unnecessary_import, no_leading_underscores_for_local_identifiers @@ -49,15 +49,20 @@ class ThumbnailApi { final String pigeonVar_messageChannelSuffix; - Future> getThumbnailBuffer(String assetId, {required int width, required int height}) async { + Future> requestImage( + String assetId, { + required int requestId, + required int width, + required int height, + }) async { final String pigeonVar_channelName = - 'dev.flutter.pigeon.immich_mobile.ThumbnailApi.getThumbnailBuffer$pigeonVar_messageChannelSuffix'; + 'dev.flutter.pigeon.immich_mobile.ThumbnailApi.requestImage$pigeonVar_messageChannelSuffix'; final BasicMessageChannel pigeonVar_channel = BasicMessageChannel( pigeonVar_channelName, pigeonChannelCodec, binaryMessenger: pigeonVar_binaryMessenger, ); - final Future pigeonVar_sendFuture = pigeonVar_channel.send([assetId, width, height]); + final Future pigeonVar_sendFuture = pigeonVar_channel.send([assetId, requestId, width, height]); final List? pigeonVar_replyList = await pigeonVar_sendFuture as List?; if (pigeonVar_replyList == null) { throw _createConnectionError(pigeonVar_channelName); @@ -76,4 +81,27 @@ class ThumbnailApi { return (pigeonVar_replyList[0] as Map?)!.cast(); } } + + Future cancelImageRequest(int requestId) async { + final String pigeonVar_channelName = + 'dev.flutter.pigeon.immich_mobile.ThumbnailApi.cancelImageRequest$pigeonVar_messageChannelSuffix'; + final BasicMessageChannel pigeonVar_channel = BasicMessageChannel( + pigeonVar_channelName, + pigeonChannelCodec, + binaryMessenger: pigeonVar_binaryMessenger, + ); + final Future pigeonVar_sendFuture = pigeonVar_channel.send([requestId]); + final List? pigeonVar_replyList = await pigeonVar_sendFuture as List?; + if (pigeonVar_replyList == null) { + throw _createConnectionError(pigeonVar_channelName); + } else if (pigeonVar_replyList.length > 1) { + throw PlatformException( + code: pigeonVar_replyList[0]! as String, + message: pigeonVar_replyList[1] as String?, + details: pigeonVar_replyList[2], + ); + } else { + return; + } + } } diff --git a/mobile/lib/presentation/widgets/images/image_provider.dart b/mobile/lib/presentation/widgets/images/image_provider.dart index 0b45e53405..b428937bc0 100644 --- a/mobile/lib/presentation/widgets/images/image_provider.dart +++ b/mobile/lib/presentation/widgets/images/image_provider.dart @@ -1,11 +1,29 @@ import 'package:flutter/widgets.dart'; -import 'package:immich_mobile/constants/constants.dart'; import 'package:immich_mobile/domain/models/asset/base_asset.model.dart'; import 'package:immich_mobile/domain/models/setting.model.dart'; import 'package:immich_mobile/domain/services/setting.service.dart'; import 'package:immich_mobile/presentation/widgets/images/local_image_provider.dart'; import 'package:immich_mobile/presentation/widgets/images/remote_image_provider.dart'; import 'package:immich_mobile/presentation/widgets/timeline/constants.dart'; +import 'package:immich_mobile/infrastructure/repositories/asset_media.repository.dart'; + +abstract class CancellableImageProvider { + void cancel(); +} + +mixin class CancellableImageProviderMixin implements CancellableImageProvider { + ImageRequest? request; + + @override + void cancel() { + final request = this.request; + if (request == null) { + return; + } + this.request = null; + return request.cancel(); + } +} ImageProvider getFullImageProvider(BaseAsset asset, {Size size = const Size(1080, 1920)}) { // Create new provider and cache it diff --git a/mobile/lib/presentation/widgets/images/local_image_provider.dart b/mobile/lib/presentation/widgets/images/local_image_provider.dart index 32a79be616..2671c11f75 100644 --- a/mobile/lib/presentation/widgets/images/local_image_provider.dart +++ b/mobile/lib/presentation/widgets/images/local_image_provider.dart @@ -8,13 +8,11 @@ import 'package:immich_mobile/infrastructure/repositories/asset_media.repository import 'package:immich_mobile/presentation/widgets/images/image_provider.dart'; import 'package:immich_mobile/presentation/widgets/images/one_frame_multi_image_stream_completer.dart'; -class LocalThumbProvider extends ImageProvider { - static const _assetMediaRepository = AssetMediaRepository(); - +class LocalThumbProvider extends ImageProvider with CancellableImageProviderMixin { final String id; final Size size; - const LocalThumbProvider({required this.id, this.size = kTimelineThumbnailSize}); + LocalThumbProvider({required this.id, this.size = kTimelineThumbnailSize}); @override Future obtainKey(ImageConfiguration configuration) { @@ -23,8 +21,8 @@ class LocalThumbProvider extends ImageProvider { @override ImageStreamCompleter loadImage(LocalThumbProvider key, ImageDecoderCallback decode) { - return OneFrameImageStreamCompleter( - _codec(key), + return OneFramePlaceholderImageStreamCompleter( + _codec(key, decode), informationCollector: () => [ DiagnosticsProperty('Id', key.id), DiagnosticsProperty('Size', key.size), @@ -32,9 +30,16 @@ class LocalThumbProvider extends ImageProvider { ); } - Future _codec(LocalThumbProvider key) async { - final codec = await _assetMediaRepository.getLocalThumbnail(key.id, key.size); - return ImageInfo(image: (await codec.getNextFrame()).image, scale: 1.0); + Stream _codec(LocalThumbProvider key, ImageDecoderCallback decode) async* { + final request = this.request = LocalImageRequest(localId: key.id, size: size); + try { + final image = await request.load(decode); + if (image != null) { + yield image; + } + } finally { + this.request = null; + } } @override @@ -50,13 +55,11 @@ class LocalThumbProvider extends ImageProvider { int get hashCode => id.hashCode ^ size.hashCode; } -class LocalFullImageProvider extends ImageProvider { - static const _assetMediaRepository = AssetMediaRepository(); - +class LocalFullImageProvider extends ImageProvider with CancellableImageProviderMixin { final String id; final Size size; - const LocalFullImageProvider({required this.id, required this.size}); + LocalFullImageProvider({required this.id, required this.size}); @override Future obtainKey(ImageConfiguration configuration) { @@ -78,12 +81,19 @@ class LocalFullImageProvider extends ImageProvider { Stream _codec(LocalFullImageProvider key, ImageDecoderCallback decode) async* { final devicePixelRatio = PlatformDispatcher.instance.views.first.devicePixelRatio; - final codec = await _assetMediaRepository.getLocalThumbnail( - key.id, - Size(size.width * devicePixelRatio, size.height * devicePixelRatio), + final request = this.request = LocalImageRequest( + localId: key.id, + size: Size(size.width * devicePixelRatio, size.height * devicePixelRatio), ); - final frame = await codec.getNextFrame(); - yield ImageInfo(image: frame.image, scale: 1.0); + + try { + final image = await request.load(decode); + if (image != null) { + yield image; + } + } finally { + this.request = null; + } } @override diff --git a/mobile/lib/presentation/widgets/images/remote_image_provider.dart b/mobile/lib/presentation/widgets/images/remote_image_provider.dart index d9ab78053f..8a51e0ad77 100644 --- a/mobile/lib/presentation/widgets/images/remote_image_provider.dart +++ b/mobile/lib/presentation/widgets/images/remote_image_provider.dart @@ -1,22 +1,22 @@ import 'dart:async'; -import 'dart:ui'; import 'package:flutter/foundation.dart'; import 'package:flutter/painting.dart'; import 'package:flutter_cache_manager/flutter_cache_manager.dart'; import 'package:immich_mobile/domain/models/setting.model.dart'; import 'package:immich_mobile/domain/services/setting.service.dart'; +import 'package:immich_mobile/infrastructure/repositories/asset_media.repository.dart'; import 'package:immich_mobile/presentation/widgets/images/image_provider.dart'; import 'package:immich_mobile/presentation/widgets/images/one_frame_multi_image_stream_completer.dart'; -import 'package:immich_mobile/providers/image/cache/image_loader.dart'; import 'package:immich_mobile/providers/image/cache/remote_image_cache_manager.dart'; +import 'package:immich_mobile/services/api.service.dart'; import 'package:immich_mobile/utils/image_url_builder.dart'; -class RemoteThumbProvider extends ImageProvider { +class RemoteThumbProvider extends ImageProvider with CancellableImageProviderMixin { final String assetId; final CacheManager? cacheManager; - const RemoteThumbProvider({required this.assetId, this.cacheManager}); + RemoteThumbProvider({required this.assetId, this.cacheManager}); @override Future obtainKey(ImageConfiguration configuration) { @@ -26,9 +26,8 @@ class RemoteThumbProvider extends ImageProvider { @override ImageStreamCompleter loadImage(RemoteThumbProvider key, ImageDecoderCallback decode) { final cache = cacheManager ?? RemoteImageCacheManager(); - return MultiFrameImageStreamCompleter( - codec: _codec(key, cache, decode), - scale: 1.0, + return OneFramePlaceholderImageStreamCompleter( + _codec(key, cache, decode), informationCollector: () => [ DiagnosticsProperty('Image provider', this), DiagnosticsProperty('Asset Id', key.assetId), @@ -36,10 +35,17 @@ class RemoteThumbProvider extends ImageProvider { ); } - Future _codec(RemoteThumbProvider key, CacheManager cache, ImageDecoderCallback decode) async { + Stream _codec(RemoteThumbProvider key, CacheManager cache, ImageDecoderCallback decode) async* { final preview = getThumbnailUrlForRemoteId(key.assetId); - - return ImageLoader.loadImageFromCache(preview, cache: cache, decode: decode); + final request = this.request = RemoteImageRequest(uri: preview, headers: ApiService.getRequestHeaders()); + try { + final image = await request.load(decode); + if (image != null) { + yield image; + } + } finally { + this.request = null; + } } @override @@ -56,11 +62,11 @@ class RemoteThumbProvider extends ImageProvider { int get hashCode => assetId.hashCode; } -class RemoteFullImageProvider extends ImageProvider { +class RemoteFullImageProvider extends ImageProvider with CancellableImageProviderMixin { final String assetId; final CacheManager? cacheManager; - const RemoteFullImageProvider({required this.assetId, this.cacheManager}); + RemoteFullImageProvider({required this.assetId, this.cacheManager}); @override Future obtainKey(ImageConfiguration configuration) { @@ -81,27 +87,33 @@ class RemoteFullImageProvider extends ImageProvider { } Stream _codec(RemoteFullImageProvider key, CacheManager cache, ImageDecoderCallback decode) async* { - ImageInfo? imageInfo; - final originalImageFuture = AppSetting.get(Setting.loadOriginal) - ? ImageLoader.loadImageFromCache( - getOriginalUrlForRemoteId(key.assetId), - cache: cache, - decode: decode, - ).then((image) => image.getNextFrame()).then((frame) => imageInfo = ImageInfo(image: frame.image, scale: 1.0)) - : null; - - final previewImageFuture = - ImageLoader.loadImageFromCache(getPreviewUrlForRemoteId(key.assetId), cache: cache, decode: decode) - .then((image) async => imageInfo == null ? await image.getNextFrame() : null) - .then((frame) => imageInfo == null ? ImageInfo(image: frame!.image, scale: 1.0) : null); - - final previewImage = await previewImageFuture; - if (previewImage != null) { - yield previewImage; + try { + final request = this.request = RemoteImageRequest( + uri: getPreviewUrlForRemoteId(key.assetId), + headers: ApiService.getRequestHeaders(), + ); + final image = await request.load(decode); + if (image == null) { + return; + } + yield image; + } finally { + request = null; } - if (originalImageFuture != null) { - yield await originalImageFuture; + if (AppSetting.get(Setting.loadOriginal)) { + try { + final request = this.request = RemoteImageRequest( + uri: getOriginalUrlForRemoteId(key.assetId), + headers: ApiService.getRequestHeaders(), + ); + final image = await request.load(decode); + if (image != null) { + yield image; + } + } finally { + request = null; + } } } diff --git a/mobile/lib/presentation/widgets/images/thumbnail.widget.dart b/mobile/lib/presentation/widgets/images/thumbnail.widget.dart index b6724c0a00..7ffbb6a9c8 100644 --- a/mobile/lib/presentation/widgets/images/thumbnail.widget.dart +++ b/mobile/lib/presentation/widgets/images/thumbnail.widget.dart @@ -10,6 +10,7 @@ import 'package:immich_mobile/domain/models/asset/base_asset.model.dart'; import 'package:immich_mobile/entities/asset.entity.dart'; import 'package:immich_mobile/extensions/build_context_extensions.dart'; import 'package:immich_mobile/extensions/theme_extensions.dart'; +import 'package:immich_mobile/presentation/widgets/images/image_provider.dart'; import 'package:immich_mobile/presentation/widgets/images/local_image_provider.dart'; import 'package:immich_mobile/presentation/widgets/images/remote_image_provider.dart'; import 'package:logging/logging.dart'; @@ -226,6 +227,10 @@ class _ThumbnailState extends State { void dispose() { _stopListeningToStream(); _providerImage?.dispose(); + final imageProvider = widget.imageProvider; + if (imageProvider is CancellableImageProvider) { + (imageProvider as CancellableImageProvider).cancel(); + } super.dispose(); } } diff --git a/mobile/lib/presentation/widgets/images/thumbnail_tile.widget.dart b/mobile/lib/presentation/widgets/images/thumbnail_tile.widget.dart index 11ec6ca430..1634a1df5d 100644 --- a/mobile/lib/presentation/widgets/images/thumbnail_tile.widget.dart +++ b/mobile/lib/presentation/widgets/images/thumbnail_tile.widget.dart @@ -56,8 +56,6 @@ class ThumbnailTile extends ConsumerWidget { ) : const BoxDecoration(); - final hasStack = asset is RemoteAsset && (asset as RemoteAsset).stackId != null; - final bool storageIndicator = showStorageIndicator ?? ref.watch(settingsProvider.select((s) => s.get(Setting.showStorageIndicator))); @@ -86,7 +84,7 @@ class ThumbnailTile extends ConsumerWidget { ), ), ), - if (hasStack) + if (asset is RemoteAsset && asset.stackId != null) asset.isVideo ? const Align( alignment: Alignment.topRight, diff --git a/mobile/pigeon/thumbnail_api.dart b/mobile/pigeon/thumbnail_api.dart index 2072c14b11..45b475ec0c 100644 --- a/mobile/pigeon/thumbnail_api.dart +++ b/mobile/pigeon/thumbnail_api.dart @@ -15,9 +15,7 @@ import 'package:pigeon/pigeon.dart'; @HostApi() abstract class ThumbnailApi { @async - Map getThumbnailBuffer( - String assetId, { - required int width, - required int height, - }); + Map requestImage(String assetId, {required int requestId, required int width, required int height}); + + void cancelImageRequest(int requestId); }