kotlin impl, avoid message passing overhead

This commit is contained in:
mertalev 2025-07-21 13:27:10 +03:00
parent 8d163ec932
commit edb024e88c
No known key found for this signature in database
GPG Key ID: DF6ABC77AAD98C95
11 changed files with 264 additions and 132 deletions

View File

@ -0,0 +1,10 @@
cmake_minimum_required(VERSION 3.10.2)
project("native_buffer")
add_library(native_buffer SHARED
src/main/cpp/native_buffer.c)
find_library(log-lib log)
target_link_libraries(native_buffer ${log-lib})

View File

@ -83,6 +83,12 @@ android {
}
}
namespace 'app.alextran.immich'
externalNativeBuild {
cmake {
path "CMakeLists.txt"
}
}
}
flutter {

View File

@ -0,0 +1,13 @@
#include <jni.h>
JNIEXPORT jobject JNICALL
Java_app_alextran_immich_images_ThumbnailsImpl_00024Companion_wrapPointer(
JNIEnv *env, jclass clazz, jlong address, jint capacity) {
return (*env)->NewDirectByteBuffer(env, (void*)address, capacity);
}
JNIEXPORT jobject JNICALL
Java_app_alextran_immich_images_ThumbnailsImpl_wrapPointer(
JNIEnv *env, jclass clazz, jlong address, jint capacity) {
return (*env)->NewDirectByteBuffer(env, (void*)address, capacity);
}

View File

@ -2,7 +2,8 @@ package app.alextran.immich
import android.os.Build
import android.os.ext.SdkExtensions
import androidx.annotation.NonNull
import app.alextran.immich.images.ThumbnailApi
import app.alextran.immich.images.ThumbnailsImpl
import app.alextran.immich.sync.NativeSyncApi
import app.alextran.immich.sync.NativeSyncApiImpl26
import app.alextran.immich.sync.NativeSyncApiImpl30
@ -10,7 +11,7 @@ import io.flutter.embedding.android.FlutterFragmentActivity
import io.flutter.embedding.engine.FlutterEngine
class MainActivity : FlutterFragmentActivity() {
override fun configureFlutterEngine(@NonNull flutterEngine: FlutterEngine) {
override fun configureFlutterEngine(flutterEngine: FlutterEngine) {
super.configureFlutterEngine(flutterEngine)
flutterEngine.plugins.add(BackgroundServicePlugin())
flutterEngine.plugins.add(HttpSSLOptionsPlugin())
@ -23,5 +24,6 @@ class MainActivity : FlutterFragmentActivity() {
NativeSyncApiImpl30(this)
}
NativeSyncApi.setUp(flutterEngine.dartExecutor.binaryMessenger, nativeSyncApiImpl)
ThumbnailApi.setUp(flutterEngine.dartExecutor.binaryMessenger, ThumbnailsImpl(this))
}
}

View File

@ -59,7 +59,7 @@ private open class ThumbnailsPigeonCodec : StandardMessageCodec() {
/** Generated interface from Pigeon that represents a handler of messages from Flutter. */
interface ThumbnailApi {
fun getThumbnail(assetId: String, width: Long, height: Long, callback: (Result<ByteArray>) -> Unit)
fun setThumbnailToBuffer(pointer: Long, assetId: String, width: Long, height: Long, callback: (Result<Unit>) -> Unit)
companion object {
/** The codec used by ThumbnailApi. */
@ -71,20 +71,20 @@ interface ThumbnailApi {
fun setUp(binaryMessenger: BinaryMessenger, api: ThumbnailApi?, messageChannelSuffix: String = "") {
val separatedMessageChannelSuffix = if (messageChannelSuffix.isNotEmpty()) ".$messageChannelSuffix" else ""
run {
val channel = BasicMessageChannel<Any?>(binaryMessenger, "dev.flutter.pigeon.immich_mobile.ThumbnailApi.getThumbnail$separatedMessageChannelSuffix", codec)
val channel = BasicMessageChannel<Any?>(binaryMessenger, "dev.flutter.pigeon.immich_mobile.ThumbnailApi.setThumbnailToBuffer$separatedMessageChannelSuffix", codec)
if (api != null) {
channel.setMessageHandler { message, reply ->
val args = message as List<Any?>
val assetIdArg = args[0] as String
val widthArg = args[1] as Long
val heightArg = args[2] as Long
api.getThumbnail(assetIdArg, widthArg, heightArg) { result: Result<ByteArray> ->
val pointerArg = args[0] as Long
val assetIdArg = args[1] as String
val widthArg = args[2] as Long
val heightArg = args[3] as Long
api.setThumbnailToBuffer(pointerArg, assetIdArg, widthArg, heightArg) { result: Result<Unit> ->
val error = result.exceptionOrNull()
if (error != null) {
reply.reply(ThumbnailsPigeonUtils.wrapError(error))
} else {
val data = result.getOrNull()
reply.reply(ThumbnailsPigeonUtils.wrapResult(data))
reply.reply(ThumbnailsPigeonUtils.wrapResult(null))
}
}
}

View File

@ -0,0 +1,148 @@
package app.alextran.immich.images
import android.content.ContentResolver
import android.content.ContentUris
import android.content.Context
import android.graphics.*
import android.media.MediaMetadataRetriever
import android.media.ThumbnailUtils
import android.net.Uri
import android.os.Build
import android.provider.MediaStore
import android.util.Size
import java.nio.ByteBuffer
import kotlin.math.max
import java.util.concurrent.Executors
class ThumbnailsImpl(context: Context) : ThumbnailApi {
private val ctx: Context = context.applicationContext
private val contentResolver: ContentResolver = ctx.contentResolver
private val threadPool =
Executors.newFixedThreadPool(max(4, Runtime.getRuntime().availableProcessors()))
companion object {
val PROJECTION = arrayOf(
MediaStore.MediaColumns.DATE_MODIFIED,
MediaStore.Files.FileColumns.MEDIA_TYPE,
)
const val SELECTION = "${MediaStore.MediaColumns._ID} = ?"
val URI: Uri = MediaStore.Files.getContentUri("external")
const val MEDIA_TYPE_IMAGE = MediaStore.Files.FileColumns.MEDIA_TYPE_IMAGE
const val MEDIA_TYPE_VIDEO = MediaStore.Files.FileColumns.MEDIA_TYPE_VIDEO
init {
System.loadLibrary("native_buffer")
}
@JvmStatic
external fun wrapPointer(address: Long, capacity: Int): ByteBuffer
}
override fun setThumbnailToBuffer(
pointer: Long,
assetId: String,
width: Long,
height: Long,
callback: (Result<Unit>) -> Unit
) {
threadPool.execute {
try {
val targetWidth = width.toInt()
val targetHeight = height.toInt()
val cursor = contentResolver.query(URI, PROJECTION, SELECTION, arrayOf(assetId), null)
?: return@execute callback(Result.failure(RuntimeException("Asset not found")))
cursor.use { c ->
if (!c.moveToNext()) {
return@execute callback(Result.failure(RuntimeException("Asset not found")))
}
val mediaType = c.getInt(1)
val bitmap = when (mediaType) {
MEDIA_TYPE_IMAGE -> decodeImageThumbnail(assetId, targetWidth, targetHeight)
MEDIA_TYPE_VIDEO -> decodeVideoThumbnail(assetId, targetWidth, targetHeight)
else -> return@execute callback(Result.failure(RuntimeException("Unsupported media type")))
}
val croppedBitmap = ThumbnailUtils.extractThumbnail(
bitmap,
targetWidth,
targetHeight,
ThumbnailUtils.OPTIONS_RECYCLE_INPUT
)
val buffer = wrapPointer(pointer, (width * height * 4).toInt())
croppedBitmap.copyPixelsToBuffer(buffer)
croppedBitmap.recycle()
callback(Result.success(Unit))
}
} catch (e: Exception) {
callback(Result.failure(e))
}
}
}
private fun decodeImageThumbnail(assetId: String, targetWidth: Int, targetHeight: Int): Bitmap {
val uri =
ContentUris.withAppendedId(MediaStore.Images.Media.EXTERNAL_CONTENT_URI, assetId.toLong())
return if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
contentResolver.loadThumbnail(uri, Size(targetWidth, targetHeight), null)
} else {
decodeSampledBitmap(uri, targetWidth, targetHeight)
}
}
private fun decodeVideoThumbnail(assetId: String, targetWidth: Int, targetHeight: Int): Bitmap {
val uri =
ContentUris.withAppendedId(MediaStore.Video.Media.EXTERNAL_CONTENT_URI, assetId.toLong())
return if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
contentResolver.loadThumbnail(uri, Size(targetWidth, targetHeight), null)
} else {
val retriever = MediaMetadataRetriever()
try {
retriever.setDataSource(ctx, uri)
retriever.getFrameAtTime(0L) ?: throw RuntimeException("Failed to extract video frame")
} finally {
retriever.release()
}
}
}
private fun decodeSampledBitmap(uri: Uri, targetWidth: Int, targetHeight: Int): Bitmap {
val options = BitmapFactory.Options().apply {
inJustDecodeBounds = true
}
contentResolver.openInputStream(uri)?.use { stream ->
BitmapFactory.decodeStream(stream, null, options)
}
options.apply {
inSampleSize = getSampleSize(this, targetWidth, targetHeight)
inJustDecodeBounds = false
inPreferredConfig = Bitmap.Config.ARGB_8888
}
return contentResolver.openInputStream(uri)?.use { stream ->
BitmapFactory.decodeStream(stream, null, options)
} ?: throw RuntimeException("Failed to decode bitmap")
}
private fun getSampleSize(options: BitmapFactory.Options, reqWidth: Int, reqHeight: Int): Int {
val height = options.outHeight
val width = options.outWidth
var inSampleSize = 1
if (height > reqHeight || width > reqWidth) {
val halfHeight = height / 2
val halfWidth = width / 2
while ((halfHeight / inSampleSize) >= reqHeight && (halfWidth / inSampleSize) >= reqWidth) {
inSampleSize *= 2
}
}
return inSampleSize
}
}

View File

@ -70,7 +70,7 @@ class ThumbnailsPigeonCodec: FlutterStandardMessageCodec, @unchecked Sendable {
/// Generated protocol from Pigeon that represents a handler of messages from Flutter.
protocol ThumbnailApi {
func getThumbnail(assetId: String, width: Int64, height: Int64, completion: @escaping (Result<FlutterStandardTypedData, Error>) -> Void)
func setThumbnailToBuffer(pointer: Int64, assetId: String, width: Int64, height: Int64, completion: @escaping (Result<Void, Error>) -> Void)
}
/// Generated setup class from Pigeon to handle messages through the `binaryMessenger`.
@ -79,24 +79,25 @@ class ThumbnailApiSetup {
/// Sets up an instance of `ThumbnailApi` to handle messages through the `binaryMessenger`.
static func setUp(binaryMessenger: FlutterBinaryMessenger, api: ThumbnailApi?, messageChannelSuffix: String = "") {
let channelSuffix = messageChannelSuffix.count > 0 ? ".\(messageChannelSuffix)" : ""
let getThumbnailChannel = FlutterBasicMessageChannel(name: "dev.flutter.pigeon.immich_mobile.ThumbnailApi.getThumbnail\(channelSuffix)", binaryMessenger: binaryMessenger, codec: codec)
let setThumbnailToBufferChannel = FlutterBasicMessageChannel(name: "dev.flutter.pigeon.immich_mobile.ThumbnailApi.setThumbnailToBuffer\(channelSuffix)", binaryMessenger: binaryMessenger, codec: codec)
if let api = api {
getThumbnailChannel.setMessageHandler { message, reply in
setThumbnailToBufferChannel.setMessageHandler { message, reply in
let args = message as! [Any?]
let assetIdArg = args[0] as! String
let widthArg = args[1] as! Int64
let heightArg = args[2] as! Int64
api.getThumbnail(assetId: assetIdArg, width: widthArg, height: heightArg) { result in
let pointerArg = args[0] as! Int64
let assetIdArg = args[1] as! String
let widthArg = args[2] as! Int64
let heightArg = args[3] as! Int64
api.setThumbnailToBuffer(pointer: pointerArg, assetId: assetIdArg, width: widthArg, height: heightArg) { result in
switch result {
case .success(let res):
reply(wrapResult(res))
case .success:
reply(wrapResult(nil))
case .failure(let error):
reply(wrapError(error))
}
}
}
} else {
getThumbnailChannel.setMessageHandler(nil)
setThumbnailToBufferChannel.setMessageHandler(nil)
}
}
}

View File

@ -3,64 +3,6 @@ import Flutter
import MobileCoreServices
import Photos
// https://stackoverflow.com/a/55839062
extension UIImage {
func toData(options: NSDictionary?, type: ImageType) -> Data? {
guard cgImage != nil else { return nil }
return toData(options: options, type: type.value)
}
// about properties: https://developer.apple.com/documentation/imageio/1464962-cgimagedestinationaddimage
func toData(options: NSDictionary?, type: CFString) -> Data? {
guard let cgImage = cgImage else { return nil }
return autoreleasepool { () -> Data? in
let data = NSMutableData()
guard
let imageDestination = CGImageDestinationCreateWithData(data as CFMutableData, type, 1, nil)
else { return nil }
CGImageDestinationAddImage(imageDestination, cgImage, options)
CGImageDestinationFinalize(imageDestination)
return data as Data
}
}
enum ImageType {
case image // abstract image data
case jpeg // JPEG image
case jpeg2000 // JPEG-2000 image
case tiff // TIFF image
case pict // Quickdraw PICT format
case gif // GIF image
case png // PNG image
case quickTimeImage // QuickTime image format (OSType 'qtif')
case appleICNS // Apple icon data
case bmp // Windows bitmap
case ico // Windows icon data
case rawImage // base type for raw image data (.raw)
case scalableVectorGraphics // SVG image
case livePhoto // Live Photo
var value: CFString {
switch self {
case .image: return kUTTypeImage
case .jpeg: return kUTTypeJPEG
case .jpeg2000: return kUTTypeJPEG2000
case .tiff: return kUTTypeTIFF
case .pict: return kUTTypePICT
case .gif: return kUTTypeGIF
case .png: return kUTTypePNG
case .quickTimeImage: return kUTTypeQuickTimeImage
case .appleICNS: return kUTTypeAppleICNS
case .bmp: return kUTTypeBMP
case .ico: return kUTTypeICO
case .rawImage: return kUTTypeRawImage
case .scalableVectorGraphics: return kUTTypeScalableVectorGraphics
case .livePhoto: return kUTTypeLivePhoto
}
}
}
}
class ThumbnailApiImpl: ThumbnailApi {
private static let cacheManager = PHImageManager.default()
private static let fetchOptions = {
@ -77,36 +19,44 @@ class ThumbnailApiImpl: ThumbnailApi {
requestOptions.version = .current
return requestOptions
}()
private static let processingQueue = DispatchQueue(
label: "thumbnail.processing", qos: .userInteractive, attributes: .concurrent)
func getThumbnail(
assetId: String,
width: Int64,
height: Int64,
completion: @escaping (Result<FlutterStandardTypedData, Error>) -> Void
) {
private static let processingQueue = DispatchQueue(label: "thumbnail.processing", qos: .userInteractive, attributes: .concurrent)
func setThumbnailToBuffer(pointer: Int64, assetId: String, width: Int64, height: Int64, completion: @escaping (Result<Void, any Error>) -> Void) {
guard let bufferPointer = UnsafeMutableRawPointer(bitPattern: Int(pointer))
else { completion(.failure(PigeonError(code: "", message: "Could not get buffer pointer for \(assetId)", details: nil))); return }
Self.processingQueue.async {
do {
let asset = try self.getAsset(assetId: assetId)
Self.cacheManager.requestImage(
for: asset,
targetSize: CGSize(width: Double(width), height: Double(height)),
contentMode: .aspectFill,
options: Self.requestOptions,
resultHandler: { (image, info) -> Void in
guard let data = image?.toData(options: nil, type: .bmp) else { return }
completion(.success(FlutterStandardTypedData(bytes: data)))
guard let image = image,
let cgImage = image.cgImage,
let dataProvider = cgImage.dataProvider,
let pixelData = dataProvider.data
else { completion(.failure(PigeonError(code: "", message: "Could not get pixel data for \(assetId)", details: nil))); return }
guard let sourceBuffer = CFDataGetBytePtr(pixelData)
else { completion(.failure(PigeonError(code: "", message: "Could not get pixel data buffer for \(assetId)", details: nil))); return }
let dataLength = CFDataGetLength(pixelData)
let bufferLength = width * height * 4
guard dataLength <= bufferLength
else { completion(.failure(PigeonError(code: "", message: "Buffer is not large enough (\(bufferLength) vs \(dataLength) for \(assetId)", details: nil))); return }
bufferPointer.copyMemory(from: sourceBuffer, byteCount: dataLength)
completion(.success(()))
}
)
} catch {
completion(
.failure(PigeonError(code: "", message: "Could not get asset data", details: nil)))
.failure(PigeonError(code: "", message: "Could not get asset data for \(assetId)", details: nil)))
}
}
}
private func getAsset(assetId: String) throws -> PHAsset {
guard
let asset = PHAsset.fetchAssets(withLocalIdentifiers: [assetId], options: Self.fetchOptions)
@ -116,9 +66,4 @@ class ThumbnailApiImpl: ThumbnailApi {
}
return asset
}
// func cancel(requestId: Int32) {
// Self.cacheManager.cancelImageRequest(requestId as PHImageRequestID)
// }
}

View File

@ -51,13 +51,14 @@ class ThumbnailApi {
final String pigeonVar_messageChannelSuffix;
Future<Uint8List> getThumbnail(
Future<void> setThumbnailToBuffer(
int pointer,
String assetId, {
required int width,
required int height,
}) async {
final String pigeonVar_channelName =
'dev.flutter.pigeon.immich_mobile.ThumbnailApi.getThumbnail$pigeonVar_messageChannelSuffix';
'dev.flutter.pigeon.immich_mobile.ThumbnailApi.setThumbnailToBuffer$pigeonVar_messageChannelSuffix';
final BasicMessageChannel<Object?> pigeonVar_channel =
BasicMessageChannel<Object?>(
pigeonVar_channelName,
@ -65,7 +66,7 @@ class ThumbnailApi {
binaryMessenger: pigeonVar_binaryMessenger,
);
final Future<Object?> pigeonVar_sendFuture =
pigeonVar_channel.send(<Object?>[assetId, width, height]);
pigeonVar_channel.send(<Object?>[pointer, assetId, width, height]);
final List<Object?>? pigeonVar_replyList =
await pigeonVar_sendFuture as List<Object?>?;
if (pigeonVar_replyList == null) {
@ -76,13 +77,8 @@ class ThumbnailApi {
message: pigeonVar_replyList[1] as String?,
details: pigeonVar_replyList[2],
);
} else if (pigeonVar_replyList[0] == null) {
throw PlatformException(
code: 'null-error',
message: 'Host platform returned null value for non-null return value.',
);
} else {
return (pigeonVar_replyList[0] as Uint8List?)!;
return;
}
}
}

View File

@ -1,3 +1,5 @@
import 'dart:ffi';
import 'package:flutter/material.dart';
import 'dart:async';
import 'dart:convert';
@ -15,12 +17,13 @@ import 'package:immich_mobile/services/api.service.dart';
import 'package:immich_mobile/utils/image_url_builder.dart';
import 'package:logging/logging.dart';
import 'package:thumbhash/thumbhash.dart' as thumbhash;
import 'package:ffi/ffi.dart';
final log = Logger('ThumbnailWidget');
class Thumbnail extends StatefulWidget {
final BoxFit fit;
final Size size;
final ui.Size size;
final String? blurhash;
final String? localId;
final String? remoteId;
@ -28,7 +31,7 @@ class Thumbnail extends StatefulWidget {
const Thumbnail({
this.fit = BoxFit.cover,
this.size = const Size.square(256),
this.size = const ui.Size.square(256),
this.blurhash,
this.localId,
this.remoteId,
@ -39,7 +42,7 @@ class Thumbnail extends StatefulWidget {
Thumbnail.fromAsset({
required Asset asset,
this.fit = BoxFit.cover,
this.size = const Size.square(256),
this.size = const ui.Size.square(256),
this.thumbhashOnly = false,
super.key,
}) : blurhash = asset.thumbhash,
@ -49,7 +52,7 @@ class Thumbnail extends StatefulWidget {
Thumbnail.fromBaseAsset({
required BaseAsset? asset,
this.fit = BoxFit.cover,
this.size = const Size.square(256),
this.size = const ui.Size.square(256),
this.thumbhashOnly = false,
super.key,
}) : blurhash = switch (asset) {
@ -188,25 +191,38 @@ class _ThumbnailState extends State<Thumbnail> {
return null;
}
final stopwatch = Stopwatch()..start();
final thumb = await _decodeThumbnail(buffer, 256);
final thumb = await _decodeThumbnail(buffer, 256, 256);
stopwatch.stop();
return thumb;
}
Future<ui.Image?> _decodeThumbnail(ImmutableBuffer buffer, int height) async {
Future<ui.Image?> _decodeThumbnail(
ImmutableBuffer buffer,
int width,
int height,
) async {
if (!mounted) {
buffer.dispose();
return null;
}
final descriptor = await ImageDescriptor.encoded(buffer);
final descriptor = ImageDescriptor.raw(
buffer,
width: width,
height: height,
pixelFormat: PixelFormat.rgba8888,
);
if (!mounted) {
buffer.dispose();
descriptor.dispose();
return null;
}
final codec = await descriptor.instantiateCodec(targetHeight: height);
final codec = await descriptor.instantiateCodec(
targetWidth: width,
targetHeight: height,
);
if (!mounted) {
buffer.dispose();
@ -231,16 +247,24 @@ class _ThumbnailState extends State<Thumbnail> {
final stopwatch = Stopwatch()..start();
final localId = widget.localId;
if (localId != null) {
final size = 256 * 256 * 4;
final pointer = malloc<Uint8>(size);
try {
final data =
await thumbnailApi.getThumbnail(localId, width: 256, height: 256);
await thumbnailApi.setThumbnailToBuffer(
pointer.address,
localId,
width: 256,
height: 256,
);
stopwatch.stop();
log.info(
'Retrieved local image $localId in ${stopwatch.elapsedMilliseconds.toStringAsFixed(2)} ms',
);
return ImmutableBuffer.fromUint8List(data);
return ImmutableBuffer.fromUint8List(pointer.asTypedList(size));
} catch (e) {
log.warning('Failed to retrieve local image $localId: $e');
} finally {
malloc.free(pointer);
}
}

View File

@ -15,23 +15,10 @@ import 'package:pigeon/pigeon.dart';
@HostApi()
abstract class ThumbnailApi {
@async
Uint8List getThumbnail(
void setThumbnailToBuffer(
int pointer,
String assetId, {
required int width,
required int height,
});
// @async
// int requestThumbnail(
// String assetId, {
// required int width,
// required int height,
// void Function(int requestId) onDone,
// });
}
// @FlutterApi()
// abstract class PlatformThumbnailApi {
// @async
// Uint8List? getThumbnail(String assetId, int width, int height);
// }