From b13bd8df98aabdd525ee37f5f7a934c049affde2 Mon Sep 17 00:00:00 2001 From: mertalev <101130780+mertalev@users.noreply.github.com> Date: Wed, 23 Jul 2025 20:08:48 +0300 Subject: [PATCH] light at the end of the tunnel --- .../android/app/src/main/cpp/native_buffer.c | 47 ++- .../app/alextran/immich/AppGlideModule.kt | 16 +- .../alextran/immich/images/Thumbnails.g.kt | 11 +- .../alextran/immich/images/ThumbnailsImpl.kt | 268 +++++++------ mobile/ios/Runner/Images/Thumbnails.g.swift | 11 +- mobile/ios/Runner/Images/ThumbnailsImpl.swift | 45 ++- mobile/lib/constants/constants.dart | 3 + .../repositories/asset_media.repository.dart | 57 ++- mobile/lib/main.dart | 3 + .../backup/drift_upload_detail.page.dart | 2 +- mobile/lib/platform/thumbnail_api.g.dart | 3 +- .../presentation/pages/drift_place.page.dart | 11 +- .../asset_viewer/asset_viewer.page.dart | 42 ++- .../asset_viewer/asset_viewer.state.dart | 27 +- .../widgets/images/image_provider.dart | 20 +- .../widgets/images/local_image_provider.dart | 175 ++------- .../widgets/images/thumbnail.widget.dart | 354 +++++++----------- .../widgets/images/thumbnail_tile.widget.dart | 9 +- .../widgets/timeline/fixed/segment.model.dart | 10 + .../widgets/timeline/segment_builder.dart | 18 - .../widgets/timeline/timeline.widget.dart | 1 + .../photo_view/src/core/photo_view_core.dart | 1 + mobile/pigeon/thumbnail_api.dart | 1 - 23 files changed, 563 insertions(+), 572 deletions(-) diff --git a/mobile/android/app/src/main/cpp/native_buffer.c b/mobile/android/app/src/main/cpp/native_buffer.c index a0551ad26c..2f1d76d187 100644 --- a/mobile/android/app/src/main/cpp/native_buffer.c +++ b/mobile/android/app/src/main/cpp/native_buffer.c @@ -1,13 +1,52 @@ #include +#include + +JNIEXPORT jlong JNICALL +Java_app_alextran_immich_images_ThumbnailsImpl_00024Companion_allocateNative( + JNIEnv *env, jclass clazz, jint size) +{ + void *ptr = malloc(size); + return (jlong)ptr; +} + +JNIEXPORT jlong JNICALL +Java_app_alextran_immich_images_ThumbnailsImpl_allocateNative( + JNIEnv *env, jclass clazz, jint size) +{ + void *ptr = malloc(size); + return (jlong)ptr; +} + +JNIEXPORT void JNICALL +Java_app_alextran_immich_images_ThumbnailsImpl_00024Companion_freeNative( + JNIEnv *env, jclass clazz, jlong address) +{ + if (address != 0) + { + free((void *)address); + } +} + +JNIEXPORT void JNICALL +Java_app_alextran_immich_images_ThumbnailsImpl_freeNative( + JNIEnv *env, jclass clazz, jlong address) +{ + if (address != 0) + { + free((void *)address); + } +} JNIEXPORT jobject JNICALL -Java_app_alextran_immich_images_ThumbnailsImpl_00024Companion_wrapPointer( - JNIEnv *env, jclass clazz, jlong address, jint capacity) { +Java_app_alextran_immich_images_ThumbnailsImpl_00024Companion_wrapAsBuffer( + JNIEnv *env, jclass clazz, jlong address, jint capacity) +{ return (*env)->NewDirectByteBuffer(env, (void*)address, capacity); } JNIEXPORT jobject JNICALL -Java_app_alextran_immich_images_ThumbnailsImpl_wrapPointer( - JNIEnv *env, jclass clazz, jlong address, jint capacity) { +Java_app_alextran_immich_images_ThumbnailsImpl_wrapAsBuffer( + JNIEnv *env, jclass clazz, jlong address, jint capacity) +{ return (*env)->NewDirectByteBuffer(env, (void*)address, capacity); } diff --git a/mobile/android/app/src/main/kotlin/app/alextran/immich/AppGlideModule.kt b/mobile/android/app/src/main/kotlin/app/alextran/immich/AppGlideModule.kt index f969b9576f..d87ab3b63c 100644 --- a/mobile/android/app/src/main/kotlin/app/alextran/immich/AppGlideModule.kt +++ b/mobile/android/app/src/main/kotlin/app/alextran/immich/AppGlideModule.kt @@ -1,7 +1,21 @@ package app.alextran.immich +import android.content.Context +import com.bumptech.glide.GlideBuilder import com.bumptech.glide.annotation.GlideModule +import com.bumptech.glide.load.engine.bitmap_recycle.BitmapPoolAdapter +import com.bumptech.glide.load.engine.cache.DiskCacheAdapter +import com.bumptech.glide.load.engine.cache.MemoryCache +import com.bumptech.glide.load.engine.cache.MemoryCacheAdapter import com.bumptech.glide.module.AppGlideModule @GlideModule -class AppGlideModule : AppGlideModule() \ No newline at end of file +class AppGlideModule : AppGlideModule() { + override fun applyOptions(context: Context, builder: GlideBuilder) { + super.applyOptions(context, builder) + // disable caching as this is already done on the Flutter side + builder.setMemoryCache(MemoryCacheAdapter()) + builder.setDiskCache(DiskCacheAdapter.Factory()) + builder.setBitmapPool(BitmapPoolAdapter()) + } +} diff --git a/mobile/android/app/src/main/kotlin/app/alextran/immich/images/Thumbnails.g.kt b/mobile/android/app/src/main/kotlin/app/alextran/immich/images/Thumbnails.g.kt index 4b8553e2fc..3222fc6f2c 100644 --- a/mobile/android/app/src/main/kotlin/app/alextran/immich/images/Thumbnails.g.kt +++ b/mobile/android/app/src/main/kotlin/app/alextran/immich/images/Thumbnails.g.kt @@ -59,7 +59,7 @@ private open class ThumbnailsPigeonCodec : StandardMessageCodec() { /** Generated interface from Pigeon that represents a handler of messages from Flutter. */ interface ThumbnailApi { - fun setThumbnailToBuffer(pointer: Long, assetId: String, width: Long, height: Long, callback: (Result>) -> Unit) + fun setThumbnailToBuffer(assetId: String, width: Long, height: Long, callback: (Result>) -> Unit) companion object { /** The codec used by ThumbnailApi. */ @@ -75,11 +75,10 @@ interface ThumbnailApi { if (api != null) { channel.setMessageHandler { message, reply -> val args = message as List - val pointerArg = args[0] as Long - val assetIdArg = args[1] as String - val widthArg = args[2] as Long - val heightArg = args[3] as Long - api.setThumbnailToBuffer(pointerArg, assetIdArg, widthArg, heightArg) { result: Result> -> + val assetIdArg = args[0] as String + val widthArg = args[1] as Long + val heightArg = args[2] as Long + api.setThumbnailToBuffer(assetIdArg, widthArg, heightArg) { result: Result> -> val error = result.exceptionOrNull() if (error != null) { reply.reply(ThumbnailsPigeonUtils.wrapError(error)) diff --git a/mobile/android/app/src/main/kotlin/app/alextran/immich/images/ThumbnailsImpl.kt b/mobile/android/app/src/main/kotlin/app/alextran/immich/images/ThumbnailsImpl.kt index 5f6ac3c0f1..ccf0995150 100644 --- a/mobile/android/app/src/main/kotlin/app/alextran/immich/images/ThumbnailsImpl.kt +++ b/mobile/android/app/src/main/kotlin/app/alextran/immich/images/ThumbnailsImpl.kt @@ -4,154 +4,176 @@ import android.content.ContentResolver import android.content.ContentUris import android.content.Context import android.graphics.* -import android.media.MediaMetadataRetriever -import android.media.ThumbnailUtils import android.net.Uri import android.os.Build import android.provider.MediaStore +import android.provider.MediaStore.Images +import android.provider.MediaStore.Video import android.util.Size import java.nio.ByteBuffer -import kotlin.math.max +import kotlin.math.* import java.util.concurrent.Executors +import com.bumptech.glide.Glide +import com.bumptech.glide.Priority +import com.bumptech.glide.load.DecodeFormat +import kotlin.time.TimeSource class ThumbnailsImpl(context: Context) : ThumbnailApi { - private val ctx: Context = context.applicationContext - private val contentResolver: ContentResolver = ctx.contentResolver - private val threadPool = - Executors.newFixedThreadPool(max(4, Runtime.getRuntime().availableProcessors())) + private val ctx: Context = context.applicationContext + private val contentResolver: ContentResolver = ctx.contentResolver + private val threadPool = + Executors.newFixedThreadPool(max(4, Runtime.getRuntime().availableProcessors())) - companion object { - val PROJECTION = arrayOf( - MediaStore.MediaColumns.DATE_MODIFIED, - MediaStore.Files.FileColumns.MEDIA_TYPE, - ) - const val SELECTION = "${MediaStore.MediaColumns._ID} = ?" - val URI: Uri = MediaStore.Files.getContentUri("external") + companion object { + val PROJECTION = arrayOf( + MediaStore.MediaColumns.DATE_MODIFIED, + MediaStore.Files.FileColumns.MEDIA_TYPE, + ) + const val SELECTION = "${MediaStore.MediaColumns._ID} = ?" + val URI: Uri = MediaStore.Files.getContentUri("external") - const val MEDIA_TYPE_IMAGE = MediaStore.Files.FileColumns.MEDIA_TYPE_IMAGE - const val MEDIA_TYPE_VIDEO = MediaStore.Files.FileColumns.MEDIA_TYPE_VIDEO + const val MEDIA_TYPE_IMAGE = MediaStore.Files.FileColumns.MEDIA_TYPE_IMAGE + const val MEDIA_TYPE_VIDEO = MediaStore.Files.FileColumns.MEDIA_TYPE_VIDEO - init { - System.loadLibrary("native_buffer") + init { + System.loadLibrary("native_buffer") + } + + @JvmStatic + external fun allocateNative(size: Int): Long + + @JvmStatic + external fun freeNative(pointer: Long) + + @JvmStatic + external fun wrapAsBuffer(address: Long, capacity: Int): ByteBuffer } - @JvmStatic - external fun wrapPointer(address: Long, capacity: Int): ByteBuffer - } + override fun setThumbnailToBuffer( + assetId: String, width: Long, height: Long, callback: (Result>) -> Unit + ) { + threadPool.execute { + try { + setThumbnailToBufferInternal(assetId, width, height, callback) + } catch (e: Exception) { + callback(Result.failure(e)) + } + } + } - override fun setThumbnailToBuffer( - pointer: Long, - assetId: String, - width: Long, - height: Long, - callback: (Result) -> Unit - ) { - threadPool.execute { - try { + private fun setThumbnailToBufferInternal( + assetId: String, width: Long, height: Long, callback: (Result>) -> Unit + ) { val targetWidth = width.toInt() val targetHeight = height.toInt() val cursor = contentResolver.query(URI, PROJECTION, SELECTION, arrayOf(assetId), null) - ?: return@execute callback(Result.failure(RuntimeException("Asset not found"))) + ?: return callback(Result.failure(RuntimeException("Asset not found"))) cursor.use { c -> - if (!c.moveToNext()) { - return@execute callback(Result.failure(RuntimeException("Asset not found"))) - } + if (!c.moveToNext()) { + return callback(Result.failure(RuntimeException("Asset not found"))) + } - val mediaType = c.getInt(1) - val bitmap = when (mediaType) { - MEDIA_TYPE_IMAGE -> decodeImageThumbnail(assetId, targetWidth, targetHeight) - MEDIA_TYPE_VIDEO -> decodeVideoThumbnail(assetId, targetWidth, targetHeight) - else -> return@execute callback(Result.failure(RuntimeException("Unsupported media type"))) - } + val mediaType = c.getInt(1) + val bitmap = when (mediaType) { + MEDIA_TYPE_IMAGE -> decodeImage(assetId, targetWidth, targetHeight) + MEDIA_TYPE_VIDEO -> decodeVideoThumbnail(assetId, targetWidth, targetHeight) + else -> return callback(Result.failure(RuntimeException("Unsupported media type"))) + } - val croppedBitmap = ThumbnailUtils.extractThumbnail( - bitmap, - targetWidth, - targetHeight, - ThumbnailUtils.OPTIONS_RECYCLE_INPUT - ) - val buffer = wrapPointer(pointer, (width * height * 4).toInt()) - croppedBitmap.copyPixelsToBuffer(buffer) - croppedBitmap.recycle() - callback(Result.success(Unit)) + val actualWidth = bitmap.width + val actualHeight = bitmap.height + + val size = actualWidth * actualHeight * 4 + val pointer = allocateNative(size) + try { + val buffer = wrapAsBuffer(pointer, size) + bitmap.copyPixelsToBuffer(buffer) + bitmap.recycle() + callback( + Result.success( + mapOf( + "pointer" to pointer, + "width" to actualWidth.toLong(), + "height" to actualHeight.toLong() + ) + ) + ) + } catch (e: Exception) { + freeNative(pointer) + callback(Result.failure(e)) + } } - } catch (e: Exception) { - callback(Result.failure(e)) - } - } - } - - private fun decodeImageThumbnail(assetId: String, targetWidth: Int, targetHeight: Int): Bitmap { - val uri = - ContentUris.withAppendedId(MediaStore.Images.Media.EXTERNAL_CONTENT_URI, assetId.toLong()) - return if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { - contentResolver.loadThumbnail(uri, Size(targetWidth, targetHeight), null) - } else { - decodeSampledBitmap(uri, targetWidth, targetHeight) - } - } - - private fun decodeVideoThumbnail(assetId: String, targetWidth: Int, targetHeight: Int): Bitmap { - val uri = - ContentUris.withAppendedId(MediaStore.Video.Media.EXTERNAL_CONTENT_URI, assetId.toLong()) - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { - return contentResolver.loadThumbnail(uri, Size(targetWidth, targetHeight), null) } - val retriever = MediaMetadataRetriever() - try { - retriever.setDataSource(ctx, uri) - return if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) { - retriever.getScaledFrameAtTime( - 0L, - MediaMetadataRetriever.OPTION_NEXT_SYNC, - targetWidth, - targetHeight + private fun decodeImage(assetId: String, targetWidth: Int, targetHeight: Int): Bitmap { + val uri = ContentUris.withAppendedId(Images.Media.EXTERNAL_CONTENT_URI, assetId.toLong()) + + if (targetHeight <= 768 && targetWidth <= 768) { + return if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { + contentResolver.loadThumbnail(uri, Size(targetWidth, targetHeight), null) + } else { + Images.Thumbnails.getThumbnail( + contentResolver, + assetId.toLong(), + Images.Thumbnails.MINI_KIND, + BitmapFactory.Options().apply { + inPreferredConfig = Bitmap.Config.ARGB_8888 + } + ) + } + } + + return decodeSource(uri, targetWidth, targetHeight) + } + + private fun decodeVideoThumbnail(assetId: String, targetWidth: Int, targetHeight: Int): Bitmap { + return if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { + val uri = ContentUris.withAppendedId(Video.Media.EXTERNAL_CONTENT_URI, assetId.toLong()) + contentResolver.loadThumbnail(uri, Size(targetWidth, targetHeight), null) + } else { + Video.Thumbnails.getThumbnail( + contentResolver, + assetId.toLong(), + Video.Thumbnails.MINI_KIND, + BitmapFactory.Options().apply { + inPreferredConfig = Bitmap.Config.ARGB_8888 + } + ) + } + } + + private fun decodeSource(uri: Uri, targetWidth: Int, targetHeight: Int): Bitmap { + return if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { + val source = ImageDecoder.createSource(contentResolver, uri) + + ImageDecoder.decodeBitmap(source) { decoder, info, _ -> + val sampleSize = + getSampleSize(info.size.width, info.size.height, targetWidth, targetHeight) + decoder.setTargetSampleSize(sampleSize) + decoder.allocator = ImageDecoder.ALLOCATOR_SOFTWARE + decoder.setTargetColorSpace(ColorSpace.get(ColorSpace.Named.SRGB)) + } + } else { + Glide.with(ctx) + .asBitmap() + .priority(Priority.IMMEDIATE) + .load(uri) + .disallowHardwareConfig() + .format(DecodeFormat.PREFER_ARGB_8888) + .submit(targetWidth, targetHeight).get() + } + } + + private fun getSampleSize(fullWidth: Int, fullHeight: Int, reqWidth: Int, reqHeight: Int): Int { + return 1 shl max( + 0, floor( + min( + log2(fullWidth / (2.0 * reqWidth)), + log2(fullHeight / (2.0 * reqHeight)), + ) + ).toInt() ) - } else { - retriever.getFrameAtTime(0L) - } ?: throw RuntimeException("Failed to extract video frame") - } finally { - retriever.release() } - } - - private fun decodeSampledBitmap(uri: Uri, targetWidth: Int, targetHeight: Int): Bitmap { - val options = BitmapFactory.Options().apply { - inJustDecodeBounds = true - } - - contentResolver.openInputStream(uri)?.use { stream -> - BitmapFactory.decodeStream(stream, null, options) - } - - options.apply { - inSampleSize = getSampleSize(this, targetWidth, targetHeight) - inJustDecodeBounds = false - inPreferredConfig = Bitmap.Config.ARGB_8888 - } - - return contentResolver.openInputStream(uri)?.use { stream -> - BitmapFactory.decodeStream(stream, null, options) - } ?: throw RuntimeException("Failed to decode bitmap") - } - - private fun getSampleSize(options: BitmapFactory.Options, reqWidth: Int, reqHeight: Int): Int { - val height = options.outHeight - val width = options.outWidth - var inSampleSize = 1 - - if (height > reqHeight || width > reqWidth) { - val halfHeight = height / 2 - val halfWidth = width / 2 - - while ((halfHeight / inSampleSize) >= reqHeight && (halfWidth / inSampleSize) >= reqWidth) { - inSampleSize *= 2 - } - } - - return inSampleSize - } } diff --git a/mobile/ios/Runner/Images/Thumbnails.g.swift b/mobile/ios/Runner/Images/Thumbnails.g.swift index bac3626851..3f8b1e693b 100644 --- a/mobile/ios/Runner/Images/Thumbnails.g.swift +++ b/mobile/ios/Runner/Images/Thumbnails.g.swift @@ -70,7 +70,7 @@ class ThumbnailsPigeonCodec: FlutterStandardMessageCodec, @unchecked Sendable { /// Generated protocol from Pigeon that represents a handler of messages from Flutter. protocol ThumbnailApi { - func setThumbnailToBuffer(pointer: Int64, assetId: String, width: Int64, height: Int64, completion: @escaping (Result<[String: Int64], Error>) -> Void) + func setThumbnailToBuffer(assetId: String, width: Int64, height: Int64, completion: @escaping (Result<[String: Int64], Error>) -> Void) } /// Generated setup class from Pigeon to handle messages through the `binaryMessenger`. @@ -83,11 +83,10 @@ class ThumbnailApiSetup { if let api = api { setThumbnailToBufferChannel.setMessageHandler { message, reply in let args = message as! [Any?] - let pointerArg = args[0] as! Int64 - let assetIdArg = args[1] as! String - let widthArg = args[2] as! Int64 - let heightArg = args[3] as! Int64 - api.setThumbnailToBuffer(pointer: pointerArg, assetId: assetIdArg, width: widthArg, height: heightArg) { result in + let assetIdArg = args[0] as! String + let widthArg = args[1] as! Int64 + let heightArg = args[2] as! Int64 + api.setThumbnailToBuffer(assetId: assetIdArg, width: widthArg, height: heightArg) { result in switch result { case .success(let res): reply(wrapResult(res)) diff --git a/mobile/ios/Runner/Images/ThumbnailsImpl.swift b/mobile/ios/Runner/Images/ThumbnailsImpl.swift index 68877819d6..af7d508046 100644 --- a/mobile/ios/Runner/Images/ThumbnailsImpl.swift +++ b/mobile/ios/Runner/Images/ThumbnailsImpl.swift @@ -14,7 +14,7 @@ class ThumbnailApiImpl: ThumbnailApi { let requestOptions = PHImageRequestOptions() requestOptions.isNetworkAccessAllowed = true requestOptions.deliveryMode = .highQualityFormat - requestOptions.resizeMode = .exact + requestOptions.resizeMode = .fast requestOptions.isSynchronous = true requestOptions.version = .current return requestOptions @@ -23,31 +23,44 @@ class ThumbnailApiImpl: ThumbnailApi { private static let rgbColorSpace = CGColorSpaceCreateDeviceRGB() private static let bitmapInfo = CGBitmapInfo(rawValue: CGImageAlphaInfo.premultipliedLast.rawValue).rawValue - func setThumbnailToBuffer(pointer: Int64, assetId: String, width: Int64, height: Int64, completion: @escaping (Result<[String: Int64], any Error>) -> Void) { - guard let bufferPointer = UnsafeMutableRawPointer(bitPattern: Int(pointer)) - else { completion(.failure(PigeonError(code: "", message: "Could not get buffer pointer for \(assetId)", details: nil))); return } + func setThumbnailToBuffer(assetId: String, width: Int64, height: Int64, completion: @escaping (Result<[String: Int64], any Error>) -> Void) { Self.processingQueue.async { guard let asset = PHAsset.fetchAssets(withLocalIdentifiers: [assetId], options: Self.fetchOptions).firstObject else { completion(.failure(PigeonError(code: "", message: "Could not get asset data for \(assetId)", details: nil))); return } + Self.cacheManager.requestImage( for: asset, targetSize: CGSize(width: Double(width), height: Double(height)), - contentMode: .aspectFill, + contentMode: .aspectFit, options: Self.requestOptions, resultHandler: { (image, info) -> Void in guard let image = image, - let cgImage = image.cgImage, - let context = CGContext( - data: bufferPointer, - width: cgImage.width, - height: cgImage.height, - bitsPerComponent: 8, - bytesPerRow: cgImage.width * 4, - space: Self.rgbColorSpace, - bitmapInfo: Self.bitmapInfo - ) else { completion(.failure(PigeonError(code: "", message: "Could not get pixel data for \(assetId)", details: nil))); return } + let cgImage = image.cgImage else { + completion(.failure(PigeonError(code: "", message: "Could not get pixel data for \(assetId)", details: nil))) + return + } + + let pointer = UnsafeMutableRawPointer.allocate( + byteCount: Int(cgImage.width) * Int(cgImage.height) * 4, + alignment: MemoryLayout.alignment + ) + + guard let context = CGContext( + data: pointer, + width: cgImage.width, + height: cgImage.height, + bitsPerComponent: 8, + bytesPerRow: cgImage.width * 4, + space: Self.rgbColorSpace, + bitmapInfo: Self.bitmapInfo + ) else { + pointer.deallocate() + completion(.failure(PigeonError(code: "", message: "Could not create context for \(assetId)", details: nil))) + return + } + context.interpolationQuality = .none context.draw(cgImage, in: CGRect(x: 0, y: 0, width: cgImage.width, height: cgImage.height)) - completion(.success(["width": Int64(cgImage.width), "height": Int64(cgImage.height)])) + completion(.success(["pointer": Int64(Int(bitPattern: pointer)), "width": Int64(cgImage.width), "height": Int64(cgImage.height)])) } ) } diff --git a/mobile/lib/constants/constants.dart b/mobile/lib/constants/constants.dart index 158bfa7147..609ac711fd 100644 --- a/mobile/lib/constants/constants.dart +++ b/mobile/lib/constants/constants.dart @@ -28,6 +28,9 @@ const String kDownloadGroupLivePhoto = 'group_livephoto'; const int kTimelineNoneSegmentSize = 120; const int kTimelineAssetLoadBatchSize = 1024; const int kTimelineAssetLoadOppositeSize = 64; +const double kTimelineThumbnailTileSize = 256.0; +const double kTimelineThumbnailSize = 384.0; +const int kTimelineImageCacheMemory = 250 * 1024 * 1024; // Widget keys const String kWidgetAuthToken = "widget_auth_token"; diff --git a/mobile/lib/infrastructure/repositories/asset_media.repository.dart b/mobile/lib/infrastructure/repositories/asset_media.repository.dart index 6c81c7ff7f..d640d47910 100644 --- a/mobile/lib/infrastructure/repositories/asset_media.repository.dart +++ b/mobile/lib/infrastructure/repositories/asset_media.repository.dart @@ -1,18 +1,57 @@ +import 'dart:ffi'; import 'dart:typed_data'; import 'dart:ui'; +import 'dart:ui' as ui; +import 'package:immich_mobile/providers/infrastructure/platform.provider.dart'; import 'package:photo_manager/photo_manager.dart'; +import 'package:ffi/ffi.dart'; class AssetMediaRepository { const AssetMediaRepository(); - Future getThumbnail(String id, {int quality = 80, Size size = const Size.square(256)}) => AssetEntity( - id: id, - // The below fields are not used in thumbnailDataWithSize but are required - // to create an AssetEntity instance. It is faster to create a dummy AssetEntity - // instance than to fetch the asset from the device first. - typeInt: AssetType.image.index, - width: size.width.toInt(), - height: size.height.toInt(), - ).thumbnailDataWithSize(ThumbnailSize(size.width.toInt(), size.height.toInt()), quality: quality); + Future getLocalThumbnail(String localId, ui.Size size) async { + final info = await thumbnailApi.setThumbnailToBuffer( + localId, + width: size.width.toInt(), + height: size.height.toInt(), + ); + + final pointer = Pointer.fromAddress(info['pointer']!); + final actualWidth = info['width']!; + final actualHeight = info['height']!; + final actualSize = actualWidth * actualHeight * 4; + + try { + final buffer = + await ImmutableBuffer.fromUint8List(pointer.asTypedList(actualSize)); + final descriptor = ui.ImageDescriptor.raw( + buffer, + width: actualWidth, + height: actualHeight, + pixelFormat: ui.PixelFormat.rgba8888, + ); + return await descriptor.instantiateCodec(); + } finally { + malloc.free(pointer); + } + } + + Future getThumbnail( + String id, { + int quality = 80, + ui.Size size = const ui.Size.square(256), + }) => + AssetEntity( + id: id, + // The below fields are not used in thumbnailDataWithSize but are required + // to create an AssetEntity instance. It is faster to create a dummy AssetEntity + // instance than to fetch the asset from the device first. + typeInt: AssetType.image.index, + width: size.width.toInt(), + height: size.height.toInt(), + ).thumbnailDataWithSize( + ThumbnailSize(size.width.toInt(), size.height.toInt()), + quality: quality, + ); } diff --git a/mobile/lib/main.dart b/mobile/lib/main.dart index d1f415a304..bada3b6d1d 100644 --- a/mobile/lib/main.dart +++ b/mobile/lib/main.dart @@ -69,6 +69,9 @@ Future initApp() async { } } + PaintingBinding.instance.imageCache.maximumSizeBytes = + kTimelineImageCacheMemory; + await DynamicTheme.fetchSystemPalette(); final log = Logger("ImmichErrorLogger"); diff --git a/mobile/lib/pages/backup/drift_upload_detail.page.dart b/mobile/lib/pages/backup/drift_upload_detail.page.dart index 36dbe4e128..41d25fa3ac 100644 --- a/mobile/lib/pages/backup/drift_upload_detail.page.dart +++ b/mobile/lib/pages/backup/drift_upload_detail.page.dart @@ -224,7 +224,7 @@ class FileDetailDialog extends ConsumerWidget { borderRadius: const BorderRadius.all(Radius.circular(12)), ), child: asset != null - ? Thumbnail(asset: asset, size: const Size(512, 512), fit: BoxFit.cover) + ? Thumbnail.fromBaseAsset(asset: asset, size: const Size(512, 512), fit: BoxFit.cover) : null, ), ), diff --git a/mobile/lib/platform/thumbnail_api.g.dart b/mobile/lib/platform/thumbnail_api.g.dart index cc6f6ef3bf..ac5249d6c7 100644 --- a/mobile/lib/platform/thumbnail_api.g.dart +++ b/mobile/lib/platform/thumbnail_api.g.dart @@ -52,7 +52,6 @@ class ThumbnailApi { final String pigeonVar_messageChannelSuffix; Future> setThumbnailToBuffer( - int pointer, String assetId, { required int width, required int height, @@ -66,7 +65,7 @@ class ThumbnailApi { binaryMessenger: pigeonVar_binaryMessenger, ); final Future pigeonVar_sendFuture = - pigeonVar_channel.send([pointer, assetId, width, height]); + pigeonVar_channel.send([assetId, width, height]); final List? pigeonVar_replyList = await pigeonVar_sendFuture as List?; if (pigeonVar_replyList == null) { diff --git a/mobile/lib/presentation/pages/drift_place.page.dart b/mobile/lib/presentation/pages/drift_place.page.dart index e85bb90d54..bda0c43c0d 100644 --- a/mobile/lib/presentation/pages/drift_place.page.dart +++ b/mobile/lib/presentation/pages/drift_place.page.dart @@ -4,6 +4,7 @@ import 'package:hooks_riverpod/hooks_riverpod.dart'; import 'package:immich_mobile/extensions/build_context_extensions.dart'; import 'package:immich_mobile/extensions/translate_extensions.dart'; import 'package:immich_mobile/pages/common/large_leading_tile.dart'; +import 'package:immich_mobile/presentation/widgets/images/remote_image_provider.dart'; import 'package:immich_mobile/presentation/widgets/images/thumbnail.widget.dart'; import 'package:immich_mobile/providers/infrastructure/asset.provider.dart'; import 'package:immich_mobile/routing/router.dart'; @@ -163,8 +164,14 @@ class _PlaceTile extends StatelessWidget { onTap: () => context.pushRoute(DriftPlaceDetailRoute(place: place.$1)), title: Text(place.$1, style: context.textTheme.titleMedium?.copyWith(fontWeight: FontWeight.w500)), leading: ClipRRect( - borderRadius: const BorderRadius.all(Radius.circular(20)), - child: Thumbnail(size: const Size(80, 80), fit: BoxFit.cover, remoteId: place.$2), + borderRadius: const BorderRadius.all( + Radius.circular(20), + ), + child: Thumbnail( + imageProvider: RemoteThumbProvider(assetId: place.$2), + size: const Size(80, 80), + fit: BoxFit.cover, + ), ), ); } diff --git a/mobile/lib/presentation/widgets/asset_viewer/asset_viewer.page.dart b/mobile/lib/presentation/widgets/asset_viewer/asset_viewer.page.dart index 7e99e15851..2b24775fa8 100644 --- a/mobile/lib/presentation/widgets/asset_viewer/asset_viewer.page.dart +++ b/mobile/lib/presentation/widgets/asset_viewer/asset_viewer.page.dart @@ -114,10 +114,10 @@ class _AssetViewerState extends ConsumerState { super.dispose(); } - bool get showingBottomSheet => ref.read(assetViewerProvider.select((s) => s.showingBottomSheet)); + bool get showingBottomSheet => ref.read(assetViewerProvider).showingBottomSheet; Color get backgroundColor { - final opacity = ref.read(assetViewerProvider.select((s) => s.backgroundOpacity)); + final opacity = ref.read(assetViewerProvider).backgroundOpacity; return Colors.black.withAlpha(opacity); } @@ -148,7 +148,7 @@ class _AssetViewerState extends ConsumerState { unawaited( Future.wait([ precacheImage( - getThumbnailImageProvider(asset: asset, size: screenSize), + getThumbnailImageProvider(asset: asset), context, onError: (_, __) {}, ), @@ -177,9 +177,8 @@ class _AssetViewerState extends ConsumerState { // Check if widget is still mounted before proceeding if (!mounted) return; - for (final offset in [-1, 1]) { - unawaited(_precacheImage(index + offset)); - } + unawaited(_precacheImage(index - 1)); + unawaited(_precacheImage(index + 1)); }); _delayedOperations.add(timer); @@ -482,7 +481,10 @@ class _AssetViewerState extends ConsumerState { width: double.infinity, height: double.infinity, color: backgroundColor, - child: Thumbnail.fromBaseAsset(asset: asset, fit: BoxFit.contain, size: Size(ctx.width, ctx.height)), + child: Thumbnail.fromBaseAsset( + asset: asset, + fit: BoxFit.contain, + ), ); } @@ -501,7 +503,7 @@ class _AssetViewerState extends ConsumerState { BaseAsset asset = ref.read(timelineServiceProvider).getAsset(index); final stackChildren = ref.read(stackChildrenNotifier(asset)).valueOrNull; if (stackChildren != null && stackChildren.isNotEmpty) { - asset = stackChildren.elementAt(ref.read(assetViewerProvider.select((s) => s.stackIndex))); + asset = stackChildren.elementAt(ref.read(assetViewerProvider).stackIndex); } final isPlayingMotionVideo = ref.read(isPlayingMotionVideoProvider); @@ -520,8 +522,8 @@ class _AssetViewerState extends ConsumerState { heroAttributes: PhotoViewHeroAttributes(tag: '${asset.heroTag}_$heroOffset'), filterQuality: FilterQuality.high, tightMode: true, - initialScale: PhotoViewComputedScale.contained * 0.999, - minScale: PhotoViewComputedScale.contained * 0.999, + initialScale: PhotoViewComputedScale.contained, + minScale: PhotoViewComputedScale.contained, disableScaleGestures: showingBottomSheet, onDragStart: _onDragStart, onDragUpdate: _onDragUpdate, @@ -532,7 +534,10 @@ class _AssetViewerState extends ConsumerState { width: ctx.width, height: ctx.height, color: backgroundColor, - child: Thumbnail.fromBaseAsset(asset: asset, fit: BoxFit.contain, size: size), + child: Thumbnail.fromBaseAsset( + asset: asset, + fit: BoxFit.contain, + ), ), ); } @@ -550,9 +555,9 @@ class _AssetViewerState extends ConsumerState { onTapDown: _onTapDown, heroAttributes: PhotoViewHeroAttributes(tag: '${asset.heroTag}_$heroOffset'), filterQuality: FilterQuality.high, - initialScale: PhotoViewComputedScale.contained * 0.99, + initialScale: PhotoViewComputedScale.contained, maxScale: 1.0, - minScale: PhotoViewComputedScale.contained * 0.99, + minScale: PhotoViewComputedScale.contained, basePosition: Alignment.center, child: SizedBox( width: ctx.width, @@ -581,9 +586,14 @@ class _AssetViewerState extends ConsumerState { Widget build(BuildContext context) { // Rebuild the widget when the asset viewer state changes // Using multiple selectors to avoid unnecessary rebuilds for other state changes - ref.watch(assetViewerProvider.select((s) => s.showingBottomSheet)); - ref.watch(assetViewerProvider.select((s) => s.backgroundOpacity)); - ref.watch(assetViewerProvider.select((s) => s.stackIndex)); + ref.watch( + assetViewerProvider.select( + (s) => + s.showingBottomSheet.hashCode ^ + s.backgroundOpacity.hashCode ^ + s.stackIndex.hashCode, + ), + ); ref.watch(isPlayingMotionVideoProvider); // Listen for casting changes and send initial asset to the cast provider diff --git a/mobile/lib/presentation/widgets/asset_viewer/asset_viewer.state.dart b/mobile/lib/presentation/widgets/asset_viewer/asset_viewer.state.dart index 88513516eb..096d5b1782 100644 --- a/mobile/lib/presentation/widgets/asset_viewer/asset_viewer.state.dart +++ b/mobile/lib/presentation/widgets/asset_viewer/asset_viewer.state.dart @@ -75,22 +75,37 @@ class AssetViewerStateNotifier extends AutoDisposeNotifier { } void setAsset(BaseAsset? asset) { - state = state.copyWith(currentAsset: asset, stackIndex: 0); + if (asset != state.currentAsset) { + state = state.copyWith(currentAsset: asset, stackIndex: 0); + } } void setOpacity(int opacity) { - state = state.copyWith(backgroundOpacity: opacity, showingControls: opacity == 255 ? true : state.showingControls); + if (opacity != state.backgroundOpacity) { + state = state.copyWith( + backgroundOpacity: opacity, + showingControls: opacity == 255 ? true : state.showingControls, + ); + } } void setBottomSheet(bool showing) { - state = state.copyWith(showingBottomSheet: showing, showingControls: showing ? true : state.showingControls); + if (showing == state.showingBottomSheet) { + return; + } + state = state.copyWith( + showingBottomSheet: showing, + showingControls: showing || state.showingControls, + ); if (showing) { ref.read(videoPlayerControlsProvider.notifier).pause(); } } void setControls(bool isShowing) { - state = state.copyWith(showingControls: isShowing); + if (isShowing != state.showingControls) { + state = state.copyWith(showingControls: isShowing); + } } void toggleControls() { @@ -98,7 +113,9 @@ class AssetViewerStateNotifier extends AutoDisposeNotifier { } void setStackIndex(int index) { - state = state.copyWith(stackIndex: index); + if (index != state.stackIndex) { + state = state.copyWith(stackIndex: index); + } } } diff --git a/mobile/lib/presentation/widgets/images/image_provider.dart b/mobile/lib/presentation/widgets/images/image_provider.dart index d94480b434..a1bfac4657 100644 --- a/mobile/lib/presentation/widgets/images/image_provider.dart +++ b/mobile/lib/presentation/widgets/images/image_provider.dart @@ -1,4 +1,5 @@ import 'package:flutter/widgets.dart'; +import 'package:immich_mobile/constants/constants.dart'; import 'package:immich_mobile/domain/models/asset/base_asset.model.dart'; import 'package:immich_mobile/domain/models/setting.model.dart'; import 'package:immich_mobile/domain/services/setting.service.dart'; @@ -10,7 +11,7 @@ ImageProvider getFullImageProvider(BaseAsset asset, {Size size = const Size(1080 final ImageProvider provider; if (_shouldUseLocalAsset(asset)) { final id = asset is LocalAsset ? asset.id : (asset as RemoteAsset).localId!; - provider = LocalFullImageProvider(id: id, name: asset.name, size: size, type: asset.type); + provider = LocalFullImageProvider(id: id, size: size); } else { final String assetId; if (asset is LocalAsset && asset.hasRemote) { @@ -26,8 +27,15 @@ ImageProvider getFullImageProvider(BaseAsset asset, {Size size = const Size(1080 return provider; } -ImageProvider getThumbnailImageProvider({BaseAsset? asset, String? remoteId, Size size = const Size.square(256)}) { - assert(asset != null || remoteId != null, 'Either asset or remoteId must be provided'); +ImageProvider getThumbnailImageProvider({ + BaseAsset? asset, + String? remoteId, + Size size = const Size.square(kTimelineThumbnailSize), +}) { + assert( + asset != null || remoteId != null, + 'Either asset or remoteId must be provided', + ); if (remoteId != null) { return RemoteThumbProvider(assetId: remoteId); @@ -35,7 +43,11 @@ ImageProvider getThumbnailImageProvider({BaseAsset? asset, String? remoteId, Siz if (_shouldUseLocalAsset(asset!)) { final id = asset is LocalAsset ? asset.id : (asset as RemoteAsset).localId!; - return LocalThumbProvider(id: id, updatedAt: asset.updatedAt, name: asset.name, size: size); + return LocalThumbProvider( + id: id, + // updatedAt: asset.updatedAt, TODO + size: size, + ); } final String assetId; diff --git a/mobile/lib/presentation/widgets/images/local_image_provider.dart b/mobile/lib/presentation/widgets/images/local_image_provider.dart index 350bcbb8fb..5dedd50fe7 100644 --- a/mobile/lib/presentation/widgets/images/local_image_provider.dart +++ b/mobile/lib/presentation/widgets/images/local_image_provider.dart @@ -1,36 +1,21 @@ import 'dart:async'; -import 'dart:io'; import 'dart:ui'; -import 'package:cached_network_image/cached_network_image.dart'; import 'package:flutter/foundation.dart'; import 'package:flutter/widgets.dart'; -import 'package:flutter_cache_manager/flutter_cache_manager.dart'; -import 'package:immich_mobile/domain/models/asset/base_asset.model.dart'; -import 'package:immich_mobile/domain/models/setting.model.dart'; -import 'package:immich_mobile/domain/services/setting.service.dart'; import 'package:immich_mobile/infrastructure/repositories/asset_media.repository.dart'; -import 'package:immich_mobile/infrastructure/repositories/storage.repository.dart'; -import 'package:immich_mobile/presentation/widgets/timeline/constants.dart'; -import 'package:immich_mobile/providers/image/cache/thumbnail_image_cache_manager.dart'; -import 'package:immich_mobile/providers/image/exceptions/image_loading_exception.dart'; -import 'package:logging/logging.dart'; class LocalThumbProvider extends ImageProvider { - final AssetMediaRepository _assetMediaRepository = const AssetMediaRepository(); - final CacheManager? cacheManager; + static const _assetMediaRepository = AssetMediaRepository(); final String id; - final DateTime updatedAt; - final String name; final Size size; + final DateTime? updatedAt; const LocalThumbProvider({ required this.id, - required this.updatedAt, - required this.name, - this.size = const Size.square(kTimelineFixedTileExtent), - this.cacheManager, + required this.size, + this.updatedAt, }); @override @@ -39,48 +24,34 @@ class LocalThumbProvider extends ImageProvider { } @override - ImageStreamCompleter loadImage(LocalThumbProvider key, ImageDecoderCallback decode) { - final cache = cacheManager ?? ThumbnailImageCacheManager(); - return MultiFrameImageStreamCompleter( - codec: _codec(key, cache, decode), - scale: 1.0, + ImageStreamCompleter loadImage( + LocalThumbProvider key, + ImageDecoderCallback decode, + ) { + return OneFrameImageStreamCompleter( + _codec(key), informationCollector: () => [ DiagnosticsProperty('Image provider', this), DiagnosticsProperty('Id', key.id), DiagnosticsProperty('Updated at', key.updatedAt), - DiagnosticsProperty('Name', key.name), DiagnosticsProperty('Size', key.size), ], ); } - Future _codec(LocalThumbProvider key, CacheManager cache, ImageDecoderCallback decode) async { - final cacheKey = '${key.id}-${key.updatedAt}-${key.size.width}x${key.size.height}'; - - final fileFromCache = await cache.getFileFromCache(cacheKey); - if (fileFromCache != null) { - try { - final buffer = await ImmutableBuffer.fromFilePath(fileFromCache.file.path); - return decode(buffer); - } catch (_) {} - } - - final thumbnailBytes = await _assetMediaRepository.getThumbnail(key.id, size: key.size); - if (thumbnailBytes == null) { - PaintingBinding.instance.imageCache.evict(key); - throw StateError("Loading thumb for local photo ${key.name} failed"); - } - - final buffer = await ImmutableBuffer.fromUint8List(thumbnailBytes); - unawaited(cache.putFile(cacheKey, thumbnailBytes)); - return decode(buffer); + Future _codec(LocalThumbProvider key) async { + final codec = + await _assetMediaRepository.getLocalThumbnail(key.id, key.size); + return ImageInfo(image: (await codec.getNextFrame()).image, scale: 1.0); } @override bool operator ==(Object other) { if (identical(this, other)) return true; if (other is LocalThumbProvider) { - return id == other.id && updatedAt == other.updatedAt; + return id == other.id && + size == other.size && + updatedAt == other.updatedAt; } return false; } @@ -90,15 +61,12 @@ class LocalThumbProvider extends ImageProvider { } class LocalFullImageProvider extends ImageProvider { - final AssetMediaRepository _assetMediaRepository = const AssetMediaRepository(); - final StorageRepository _storageRepository = const StorageRepository(); + static const _assetMediaRepository = AssetMediaRepository(); final String id; - final String name; final Size size; - final AssetType type; - const LocalFullImageProvider({required this.id, required this.name, required this.size, required this.type}); + const LocalFullImageProvider({required this.id, required this.size}); @override Future obtainKey(ImageConfiguration configuration) { @@ -106,105 +74,32 @@ class LocalFullImageProvider extends ImageProvider { } @override - ImageStreamCompleter loadImage(LocalFullImageProvider key, ImageDecoderCallback decode) { - return MultiImageStreamCompleter( - codec: _codec(key, decode), - scale: 1.0, - informationCollector: () sync* { - yield ErrorDescription(name); - }, + ImageStreamCompleter loadImage( + LocalFullImageProvider key, + ImageDecoderCallback decode, + ) { + return OneFrameImageStreamCompleter(_codec(key)); + } + + Future _codec(LocalFullImageProvider key) async { + final devicePixelRatio = + PlatformDispatcher.instance.views.first.devicePixelRatio; + final codec = await _assetMediaRepository.getLocalThumbnail( + key.id, + Size(size.width * devicePixelRatio, size.height * devicePixelRatio), ); - } - - // Streams in each stage of the image as we ask for it - Stream _codec(LocalFullImageProvider key, ImageDecoderCallback decode) async* { - try { - switch (key.type) { - case AssetType.image: - yield* _decodeProgressive(key, decode); - break; - case AssetType.video: - final codec = await _getThumbnailCodec(key, decode); - if (codec == null) { - throw StateError("Failed to load preview for ${key.name}"); - } - yield codec; - break; - case AssetType.other: - case AssetType.audio: - throw StateError('Unsupported asset type ${key.type}'); - } - } catch (error, stack) { - Logger('ImmichLocalImageProvider').severe('Error loading local image ${key.name}', error, stack); - throw const ImageLoadingException('Could not load image from local storage'); - } - } - - Future _getThumbnailCodec(LocalFullImageProvider key, ImageDecoderCallback decode) async { - final thumbBytes = await _assetMediaRepository.getThumbnail(key.id, size: key.size); - if (thumbBytes == null) { - return null; - } - final buffer = await ImmutableBuffer.fromUint8List(thumbBytes); - return decode(buffer); - } - - Stream _decodeProgressive(LocalFullImageProvider key, ImageDecoderCallback decode) async* { - final file = await _storageRepository.getFileForAsset(key.id); - if (file == null) { - throw StateError("Opening file for asset ${key.name} failed"); - } - - final fileSize = await file.length(); - final devicePixelRatio = PlatformDispatcher.instance.views.first.devicePixelRatio; - final isLargeFile = fileSize > 20 * 1024 * 1024; // 20MB - final isHEIC = file.path.toLowerCase().contains(RegExp(r'\.(heic|heif)$')); - final isProgressive = isLargeFile || (isHEIC && !Platform.isIOS); - - if (isProgressive) { - try { - final progressiveMultiplier = devicePixelRatio >= 3.0 ? 1.3 : 1.2; - final size = Size( - (key.size.width * progressiveMultiplier).clamp(256, 1024), - (key.size.height * progressiveMultiplier).clamp(256, 1024), - ); - final mediumThumb = await _assetMediaRepository.getThumbnail(key.id, size: size); - if (mediumThumb != null) { - final mediumBuffer = await ImmutableBuffer.fromUint8List(mediumThumb); - yield await decode(mediumBuffer); - } - } catch (_) {} - } - - // Load original only when the file is smaller or if the user wants to load original images - // Or load a slightly larger image for progressive loading - if (isProgressive && !(AppSetting.get(Setting.loadOriginal))) { - final progressiveMultiplier = devicePixelRatio >= 3.0 ? 2.0 : 1.6; - final size = Size( - (key.size.width * progressiveMultiplier).clamp(512, 2048), - (key.size.height * progressiveMultiplier).clamp(512, 2048), - ); - final highThumb = await _assetMediaRepository.getThumbnail(key.id, size: size); - if (highThumb != null) { - final highBuffer = await ImmutableBuffer.fromUint8List(highThumb); - yield await decode(highBuffer); - } - return; - } - - final buffer = await ImmutableBuffer.fromFilePath(file.path); - yield await decode(buffer); + return ImageInfo(image: (await codec.getNextFrame()).image, scale: 1.0); } @override bool operator ==(Object other) { if (identical(this, other)) return true; if (other is LocalFullImageProvider) { - return id == other.id && size == other.size && type == other.type && name == other.name; + return id == other.id && size == other.size; } return false; } @override - int get hashCode => id.hashCode ^ size.hashCode ^ type.hashCode ^ name.hashCode; + int get hashCode => id.hashCode ^ size.hashCode; } diff --git a/mobile/lib/presentation/widgets/images/thumbnail.widget.dart b/mobile/lib/presentation/widgets/images/thumbnail.widget.dart index c4520ed106..afb54a577b 100644 --- a/mobile/lib/presentation/widgets/images/thumbnail.widget.dart +++ b/mobile/lib/presentation/widgets/images/thumbnail.widget.dart @@ -1,289 +1,214 @@ -import 'dart:ffi'; - -import 'package:flutter/material.dart'; import 'dart:async'; import 'dart:convert'; import 'dart:ui' as ui; import 'dart:ui'; -import 'package:flutter_cache_manager/flutter_cache_manager.dart'; +import 'package:flutter/material.dart'; +import 'package:immich_mobile/constants/constants.dart'; import 'package:immich_mobile/domain/models/asset/base_asset.model.dart'; import 'package:immich_mobile/entities/asset.entity.dart'; import 'package:immich_mobile/extensions/build_context_extensions.dart'; import 'package:immich_mobile/extensions/theme_extensions.dart'; -import 'package:immich_mobile/providers/image/cache/thumbnail_image_cache_manager.dart'; -import 'package:immich_mobile/providers/infrastructure/platform.provider.dart'; -import 'package:immich_mobile/services/api.service.dart'; -import 'package:immich_mobile/utils/image_url_builder.dart'; +import 'package:immich_mobile/presentation/widgets/images/local_image_provider.dart'; +import 'package:immich_mobile/presentation/widgets/images/remote_image_provider.dart'; import 'package:logging/logging.dart'; import 'package:thumbhash/thumbhash.dart' as thumbhash; -import 'package:ffi/ffi.dart'; final log = Logger('ThumbnailWidget'); +enum ThumbhashMode { enabled, disabled, only } + class Thumbnail extends StatefulWidget { + final ImageProvider? imageProvider; final BoxFit fit; final ui.Size size; final String? blurhash; - final String? localId; - final String? remoteId; - final bool thumbhashOnly; + final ThumbhashMode thumbhashMode; const Thumbnail({ + this.imageProvider, this.fit = BoxFit.cover, - this.size = const ui.Size.square(256), + this.size = const ui.Size.square(kTimelineThumbnailSize), this.blurhash, - this.localId, - this.remoteId, - this.thumbhashOnly = false, + this.thumbhashMode = ThumbhashMode.enabled, super.key, }); Thumbnail.fromAsset({ required Asset asset, this.fit = BoxFit.cover, - this.size = const ui.Size.square(256), - this.thumbhashOnly = false, + this.size = const ui.Size.square(kTimelineThumbnailSize), + this.thumbhashMode = ThumbhashMode.enabled, super.key, }) : blurhash = asset.thumbhash, - localId = asset.localId, - remoteId = asset.remoteId; + imageProvider = _getImageProviderFromAsset(asset, size); Thumbnail.fromBaseAsset({ required BaseAsset? asset, this.fit = BoxFit.cover, - this.size = const ui.Size.square(256), - this.thumbhashOnly = false, + this.size = const ui.Size.square(kTimelineThumbnailSize), + this.thumbhashMode = ThumbhashMode.enabled, super.key, }) : blurhash = switch (asset) { RemoteAsset() => asset.thumbHash, _ => null, }, - localId = switch (asset) { - RemoteAsset() => asset.localId, - LocalAsset() => asset.id, - _ => null, - }, - remoteId = switch (asset) { - RemoteAsset() => asset.id, - LocalAsset() => asset.remoteId, - _ => null, - }; + imageProvider = _getImageProviderFromBaseAsset(asset, size); + + static ImageProvider? _getImageProviderFromAsset(Asset asset, ui.Size size) { + if (asset.localId != null) { + return LocalThumbProvider(id: asset.localId!, size: size); + } else if (asset.remoteId != null) { + return RemoteThumbProvider(assetId: asset.remoteId!); + } + return null; + } + + static ImageProvider? _getImageProviderFromBaseAsset( + BaseAsset? asset, + ui.Size size, + ) { + switch (asset) { + case RemoteAsset(): + if (asset.localId != null) { + return LocalThumbProvider(id: asset.localId!, size: size); + } else { + return RemoteThumbProvider(assetId: asset.id); + } + case LocalAsset(): + return LocalThumbProvider(id: asset.id, size: size); + case null: + return null; + } + } @override State createState() => _ThumbnailState(); } class _ThumbnailState extends State { - ui.Image? _image; + ui.Image? _thumbhashImage; + ui.Image? _providerImage; + ImageStream? _imageStream; + ImageStreamListener? _imageStreamListener; static final _gradientCache = {}; - static final _imageCache = ThumbnailImageCacheManager(); @override void initState() { super.initState(); - _decode(); + _loadImage(); } @override void didUpdateWidget(Thumbnail oldWidget) { super.didUpdateWidget(oldWidget); - if (oldWidget.blurhash != widget.blurhash || - oldWidget.localId != widget.localId || - oldWidget.remoteId != widget.remoteId || - (oldWidget.thumbhashOnly && !widget.thumbhashOnly)) { - _decode(); + if (oldWidget.imageProvider != widget.imageProvider || + oldWidget.blurhash != widget.blurhash || + (oldWidget.thumbhashMode == ThumbhashMode.disabled && + oldWidget.thumbhashMode != ThumbhashMode.disabled)) { + _loadImage(); } } - Future _decode() async { - if (!mounted) { - return; + @override + void reassemble() { + super.reassemble(); + _loadImage(); + } + + void _loadImage() { + _stopListeningToStream(); + if (widget.thumbhashMode != ThumbhashMode.disabled && + widget.blurhash != null) { + _decodeThumbhash(); } - - final thumbhashOnly = widget.thumbhashOnly; - final blurhash = widget.blurhash; - final imageFuture = thumbhashOnly ? Future.value(null) : _decodeThumbnail(); - - if (blurhash != null && _image == null) { - try { - await _decodeThumbhash(); - } catch (e) { - log.severe('Error decoding thumbhash for ${widget.remoteId}: $e'); - } + + if (widget.thumbhashMode != ThumbhashMode.only && + widget.imageProvider != null) { + _loadFromProvider(); } + } - if (!mounted || thumbhashOnly) { - return; - } - - try { - final image = await imageFuture; - if (!mounted || image == null) { - return; - } - - _image?.dispose(); - setState(() { - _image = image; - }); - } catch (e) { - log.severe('Error decoding thumbnail: $e'); + void _loadFromProvider() { + final imageProvider = widget.imageProvider; + if (imageProvider == null) return; + + _imageStream = imageProvider.resolve(ImageConfiguration.empty); + _imageStreamListener = ImageStreamListener( + (ImageInfo imageInfo, bool synchronousCall) { + if (!mounted) return; + + _thumbhashImage?.dispose(); + if (_providerImage != imageInfo.image) { + setState(() { + _providerImage = imageInfo.image; + }); + } + }, + onError: (exception, stackTrace) { + log.severe('Error loading image: $exception', exception, stackTrace); + }, + ); + _imageStream?.addListener(_imageStreamListener!); + } + + void _stopListeningToStream() { + if (_imageStreamListener != null && _imageStream != null) { + _imageStream!.removeListener(_imageStreamListener!); } + _imageStream = null; + _imageStreamListener = null; } Future _decodeThumbhash() async { final blurhash = widget.blurhash; - if (blurhash == null || !mounted || _image != null) { - return; - } - final image = thumbhash.thumbHashToRGBA(base64.decode(blurhash)); - final buffer = await ImmutableBuffer.fromUint8List(image.rgba); - if (!mounted || _image != null) { - buffer.dispose(); + if (blurhash == null || !mounted || _providerImage != null) { return; } - final descriptor = ImageDescriptor.raw( - buffer, - width: image.width, - height: image.height, - pixelFormat: PixelFormat.rgba8888, - ); + try { + final image = thumbhash.thumbHashToRGBA(base64.decode(blurhash)); + final buffer = await ImmutableBuffer.fromUint8List(image.rgba); + if (!mounted || _providerImage != null) { + buffer.dispose(); + return; + } - final codec = await descriptor.instantiateCodec(); + final descriptor = ImageDescriptor.raw( + buffer, + width: image.width, + height: image.height, + pixelFormat: PixelFormat.rgba8888, + ); - if (!mounted || _image != null) { + final codec = await descriptor.instantiateCodec(); + + if (!mounted || _providerImage != null) { + buffer.dispose(); + descriptor.dispose(); + codec.dispose(); + return; + } + + final frame = (await codec.getNextFrame()).image; buffer.dispose(); descriptor.dispose(); codec.dispose(); - return; - } - final frame = (await codec.getNextFrame()).image; - buffer.dispose(); - descriptor.dispose(); - codec.dispose(); - if (!mounted || _image != null) { - frame.dispose(); - return; - } - setState(() { - _image = frame; - }); - } - - Future _decodeThumbnail() async { - if (!mounted) { - return null; - } - - final stopwatch = Stopwatch()..start(); - final codec = await _decodeThumb(); - if (codec == null || !mounted) { - codec?.dispose(); - return null; - } - final image = (await codec.getNextFrame()).image; - stopwatch.stop(); - log.info( - 'Decoded thumbnail for ${widget.remoteId ?? widget.localId} in ${stopwatch.elapsedMilliseconds} ms', - ); - return image; - } - - Future _decodeThumb() { - final localId = widget.localId; - if (!mounted) { - return Future.value(null); - } - - if (localId != null) { - final size = widget.size; - final width = size.width.toInt(); - final height = size.height.toInt(); - return _decodeLocal(localId, width, height); - } - - final remoteId = widget.remoteId; - if (remoteId != null) { - return _decodeRemote(remoteId); - } - - return Future.value(null); - } - - Future _decodeLocal(String localId, int width, int height) async { - final pointer = malloc(width * height * 4); - - try { - final info = await thumbnailApi.setThumbnailToBuffer( - pointer.address, - localId, - width: width, - height: height, - ); - if (!mounted) { - return null; + if (!mounted || _providerImage != null) { + frame.dispose(); + return; } - final actualWidth = info['width']!; - final actualHeight = info['height']!; - final actualSize = actualWidth * actualHeight * 4; - final buffer = - await ImmutableBuffer.fromUint8List(pointer.asTypedList(actualSize)); - if (!mounted) { - buffer.dispose(); - return null; - } - final descriptor = ui.ImageDescriptor.raw( - buffer, - width: actualWidth, - height: actualHeight, - pixelFormat: ui.PixelFormat.rgba8888, - ); - return await descriptor.instantiateCodec(); + + setState(() { + _providerImage = frame; + }); } catch (e) { - return null; - } finally { - malloc.free(pointer); + log.severe('Error decoding thumbhash: $e'); } } - Future _decodeRemote(String remoteId) async { - final uri = getThumbnailUrlForRemoteId(remoteId); - final headers = ApiService.getRequestHeaders(); - final stream = _imageCache.getFileStream( - uri, - key: uri, - withProgress: true, - headers: headers, - ); - - await for (final result in stream) { - if (!mounted) { - return null; - } - - if (result is FileInfo) { - final buffer = await ImmutableBuffer.fromFilePath(result.file.path); - if (!mounted) { - buffer.dispose(); - return null; - } - final descriptor = await ImageDescriptor.encoded(buffer); - if (!mounted) { - buffer.dispose(); - descriptor.dispose(); - return null; - } - return await descriptor.instantiateCodec(); - } - } - - return null; - } - @override Widget build(BuildContext context) { final colorScheme = context.colorScheme; @@ -296,8 +221,8 @@ class _ThumbnailState extends State { end: Alignment.bottomCenter, ); - return _ThumbhashLeaf( - image: _image, + return _ThumbnailLeaf( + image: _providerImage ?? _thumbhashImage, fit: widget.fit, placeholderGradient: gradient, ); @@ -305,17 +230,18 @@ class _ThumbnailState extends State { @override void dispose() { - _image?.dispose(); + _stopListeningToStream(); + _thumbhashImage?.dispose(); super.dispose(); } } -class _ThumbhashLeaf extends LeafRenderObjectWidget { +class _ThumbnailLeaf extends LeafRenderObjectWidget { final ui.Image? image; final BoxFit fit; final Gradient placeholderGradient; - const _ThumbhashLeaf({ + const _ThumbnailLeaf({ required this.image, required this.fit, required this.placeholderGradient, @@ -323,7 +249,7 @@ class _ThumbhashLeaf extends LeafRenderObjectWidget { @override RenderObject createRenderObject(BuildContext context) { - return _ThumbhashRenderBox( + return _ThumbnailRenderBox( image: image, fit: fit, placeholderGradient: placeholderGradient, @@ -333,7 +259,7 @@ class _ThumbhashLeaf extends LeafRenderObjectWidget { @override void updateRenderObject( BuildContext context, - _ThumbhashRenderBox renderObject, + _ThumbnailRenderBox renderObject, ) { renderObject.fit = fit; renderObject.image = image; @@ -341,7 +267,7 @@ class _ThumbhashLeaf extends LeafRenderObjectWidget { } } -class _ThumbhashRenderBox extends RenderBox { +class _ThumbnailRenderBox extends RenderBox { ui.Image? _image; BoxFit _fit; Gradient _placeholderGradient; @@ -349,7 +275,7 @@ class _ThumbhashRenderBox extends RenderBox { @override bool isRepaintBoundary = true; - _ThumbhashRenderBox({ + _ThumbnailRenderBox({ required ui.Image? image, required BoxFit fit, required Gradient placeholderGradient, diff --git a/mobile/lib/presentation/widgets/images/thumbnail_tile.widget.dart b/mobile/lib/presentation/widgets/images/thumbnail_tile.widget.dart index 9e5dabcb01..7ce119e349 100644 --- a/mobile/lib/presentation/widgets/images/thumbnail_tile.widget.dart +++ b/mobile/lib/presentation/widgets/images/thumbnail_tile.widget.dart @@ -1,6 +1,7 @@ import 'package:auto_route/auto_route.dart'; import 'package:flutter/material.dart'; import 'package:hooks_riverpod/hooks_riverpod.dart'; +import 'package:immich_mobile/constants/constants.dart'; import 'package:immich_mobile/domain/models/asset/base_asset.model.dart'; import 'package:immich_mobile/extensions/build_context_extensions.dart'; import 'package:immich_mobile/extensions/duration_extensions.dart'; @@ -12,7 +13,7 @@ import 'package:immich_mobile/providers/timeline/multiselect.provider.dart'; class ThumbnailTile extends ConsumerWidget { const ThumbnailTile( this.asset, { - this.size = const Size.square(256), + this.size = const Size.square(kTimelineThumbnailTileSize), this.fit = BoxFit.cover, this.showStorageIndicator = true, this.lockSelection = false, @@ -73,9 +74,9 @@ class ThumbnailTile extends ConsumerWidget { tag: '${asset?.heroTag ?? ''}_$heroIndex', child: Thumbnail.fromBaseAsset( asset: asset, - fit: fit, - size: size, - thumbhashOnly: isScrubbing, + thumbhashMode: isScrubbing + ? ThumbhashMode.only + : ThumbhashMode.enabled, ), ), ), diff --git a/mobile/lib/presentation/widgets/timeline/fixed/segment.model.dart b/mobile/lib/presentation/widgets/timeline/fixed/segment.model.dart index 06bd1975b0..0ffe8e46a5 100644 --- a/mobile/lib/presentation/widgets/timeline/fixed/segment.model.dart +++ b/mobile/lib/presentation/widgets/timeline/fixed/segment.model.dart @@ -6,12 +6,16 @@ import 'package:hooks_riverpod/hooks_riverpod.dart'; import 'package:immich_mobile/domain/models/asset/base_asset.model.dart'; import 'package:immich_mobile/domain/models/timeline.model.dart'; import 'package:immich_mobile/domain/services/timeline.service.dart'; +import 'package:immich_mobile/presentation/widgets/asset_viewer/asset_viewer.state.dart'; import 'package:immich_mobile/presentation/widgets/images/thumbnail_tile.widget.dart'; import 'package:immich_mobile/presentation/widgets/timeline/fixed/row.dart'; import 'package:immich_mobile/presentation/widgets/timeline/header.widget.dart'; import 'package:immich_mobile/presentation/widgets/timeline/segment.model.dart'; import 'package:immich_mobile/providers/asset_viewer/is_motion_video_playing.provider.dart'; +import 'package:immich_mobile/providers/asset_viewer/video_player_controls_provider.dart'; +import 'package:immich_mobile/providers/asset_viewer/video_player_value_provider.dart'; import 'package:immich_mobile/providers/haptic_feedback.provider.dart'; +import 'package:immich_mobile/providers/infrastructure/asset_viewer/current_asset.provider.dart'; import 'package:immich_mobile/providers/infrastructure/timeline.provider.dart'; import 'package:immich_mobile/providers/timeline/multiselect.provider.dart'; import 'package:immich_mobile/routing/router.dart'; @@ -162,6 +166,12 @@ class _AssetTileWidget extends ConsumerWidget { } else { await ref.read(timelineServiceProvider).loadAssets(assetIndex, 1); ref.read(isPlayingMotionVideoProvider.notifier).playing = false; + ref.read(assetViewerProvider.notifier).setAsset(asset); + ref.read(currentAssetNotifier.notifier).setAsset(asset); + if (asset.isVideo || asset.isMotionPhoto) { + ref.read(videoPlaybackValueProvider.notifier).reset(); + ref.read(videoPlayerControlsProvider.notifier).pause(); + } ctx.pushRoute( AssetViewerRoute( initialIndex: assetIndex, diff --git a/mobile/lib/presentation/widgets/timeline/segment_builder.dart b/mobile/lib/presentation/widgets/timeline/segment_builder.dart index 7c4c87a503..ced969b849 100644 --- a/mobile/lib/presentation/widgets/timeline/segment_builder.dart +++ b/mobile/lib/presentation/widgets/timeline/segment_builder.dart @@ -1,8 +1,5 @@ -import 'package:flutter/widgets.dart'; import 'package:immich_mobile/domain/models/timeline.model.dart'; -import 'package:immich_mobile/presentation/widgets/images/thumbnail.widget.dart'; import 'package:immich_mobile/presentation/widgets/timeline/constants.dart'; -import 'package:immich_mobile/presentation/widgets/timeline/fixed/row.dart'; abstract class SegmentBuilder { final List buckets; @@ -17,19 +14,4 @@ abstract class SegmentBuilder { HeaderType.monthAndDay => kTimelineHeaderExtent * 1.6, HeaderType.none => 0.0, }; - - static Widget buildPlaceholder( - BuildContext context, - int count, { - Size size = const Size.square(kTimelineFixedTileExtent), - double spacing = kTimelineSpacing, - }) => - RepaintBoundary( - child: FixedTimelineRow( - dimension: size.height, - spacing: spacing, - textDirection: Directionality.of(context), - children: List.filled(count, const Thumbnail()), - ), - ); } diff --git a/mobile/lib/presentation/widgets/timeline/timeline.widget.dart b/mobile/lib/presentation/widgets/timeline/timeline.widget.dart index d946872781..d1557370f9 100644 --- a/mobile/lib/presentation/widgets/timeline/timeline.widget.dart +++ b/mobile/lib/presentation/widgets/timeline/timeline.widget.dart @@ -87,6 +87,7 @@ class _SliverTimeline extends ConsumerStatefulWidget { class _SliverTimelineState extends ConsumerState<_SliverTimeline> { final _scrollController = ScrollController(); StreamSubscription? _eventSubscription; + // late final KeepAliveLink asyncSegmentsLink; @override void initState() { diff --git a/mobile/lib/widgets/photo_view/src/core/photo_view_core.dart b/mobile/lib/widgets/photo_view/src/core/photo_view_core.dart index 944e5ba7e6..8a76e9b5cf 100644 --- a/mobile/lib/widgets/photo_view/src/core/photo_view_core.dart +++ b/mobile/lib/widgets/photo_view/src/core/photo_view_core.dart @@ -421,6 +421,7 @@ class PhotoViewCoreState extends State filterQuality: widget.filterQuality, width: scaleBoundaries.childSize.width * scale, fit: BoxFit.cover, + isAntiAlias: widget.filterQuality == FilterQuality.high, ); } } diff --git a/mobile/pigeon/thumbnail_api.dart b/mobile/pigeon/thumbnail_api.dart index 26df871866..04a56b3a14 100644 --- a/mobile/pigeon/thumbnail_api.dart +++ b/mobile/pigeon/thumbnail_api.dart @@ -16,7 +16,6 @@ import 'package:pigeon/pigeon.dart'; abstract class ThumbnailApi { @async Map setThumbnailToBuffer( - int pointer, String assetId, { required int width, required int height,