diff --git a/machine-learning/Dockerfile b/machine-learning/Dockerfile index 25f3c44d9e..9f70998a45 100644 --- a/machine-learning/Dockerfile +++ b/machine-learning/Dockerfile @@ -65,7 +65,8 @@ RUN if [ "$DEVICE" = "rocm" ]; then \ FROM python:3.11-slim-bookworm@sha256:49d73c49616929b0a4f37c50fee0056eb4b0f15de624591e8d9bf84b4dfdd3ce AS prod-cpu -ENV LD_PRELOAD=/usr/lib/libmimalloc.so.2 +ENV LD_PRELOAD=/usr/lib/libmimalloc.so.2 \ + MACHINE_LEARNING_MODEL_ARENA=false FROM python:3.11-slim-bookworm@sha256:49d73c49616929b0a4f37c50fee0056eb4b0f15de624591e8d9bf84b4dfdd3ce AS prod-openvino @@ -82,7 +83,8 @@ RUN apt-get update && \ FROM nvidia/cuda:12.2.2-runtime-ubuntu22.04@sha256:94c1577b2cd9dd6c0312dc04dff9cb2fdce2b268018abc3d7c2dbcacf1155000 AS prod-cuda -ENV LD_PRELOAD=/usr/lib/libmimalloc.so.2 +ENV LD_PRELOAD=/usr/lib/libmimalloc.so.2 \ + MACHINE_LEARNING_MODEL_ARENA=false RUN apt-get update && \ apt-get install --no-install-recommends -yqq libcudnn9-cuda-12 && \ @@ -98,7 +100,8 @@ FROM rocm/dev-ubuntu-22.04:6.3.4-complete@sha256:1f7e92ca7e3a3785680473329ed1091 FROM prod-cpu AS prod-armnn ENV LD_LIBRARY_PATH=/opt/armnn \ - LD_PRELOAD=/usr/lib/libmimalloc.so.2 + LD_PRELOAD=/usr/lib/libmimalloc.so.2 \ + MACHINE_LEARNING_MODEL_ARENA=false RUN apt-get update && apt-get install -y --no-install-recommends ocl-icd-libopencl1 mesa-opencl-icd libgomp1 && \ rm -rf /var/lib/apt/lists/* && \ @@ -118,7 +121,8 @@ COPY --from=builder-armnn \ FROM prod-cpu AS prod-rknn -ENV LD_PRELOAD=/usr/lib/libmimalloc.so.2 +ENV LD_PRELOAD=/usr/lib/libmimalloc.so.2 \ + MACHINE_LEARNING_MODEL_ARENA=false ADD --checksum=sha256:73993ed4b440460825f21611731564503cc1d5a0c123746477da6cd574f34885 https://github.com/airockchip/rknn-toolkit2/raw/refs/tags/v2.3.0/rknpu2/runtime/Linux/librknn_api/aarch64/librknnrt.so /usr/lib/ diff --git a/machine-learning/immich_ml/config.py b/machine-learning/immich_ml/config.py index 939afbc98b..d9edd88a8c 100644 --- a/machine-learning/immich_ml/config.py +++ b/machine-learning/immich_ml/config.py @@ -61,6 +61,7 @@ class Settings(BaseSettings): request_threads: int = os.cpu_count() or 4 model_inter_op_threads: int = 0 model_intra_op_threads: int = 0 + model_arena: bool = True ann: bool = True ann_fp16_turbo: bool = False ann_tuning_level: int = 2 diff --git a/machine-learning/immich_ml/sessions/ort.py b/machine-learning/immich_ml/sessions/ort.py index 42ceddf4bf..d18aae751a 100644 --- a/machine-learning/immich_ml/sessions/ort.py +++ b/machine-learning/immich_ml/sessions/ort.py @@ -123,7 +123,7 @@ class OrtSession: @property def _sess_options_default(self) -> ort.SessionOptions: sess_options = ort.SessionOptions() - sess_options.enable_cpu_mem_arena = False + sess_options.enable_cpu_mem_arena = settings.model_arena # avoid thread contention between models if settings.model_inter_op_threads > 0: diff --git a/machine-learning/test_main.py b/machine-learning/test_main.py index f8027f477a..0d02ef7466 100644 --- a/machine-learning/test_main.py +++ b/machine-learning/test_main.py @@ -309,6 +309,22 @@ class TestOrtSession: assert session.sess_options.inter_op_num_threads == 2 assert session.sess_options.intra_op_num_threads == 4 + def test_uses_arena_if_enabled(self, mocker: MockerFixture) -> None: + mock_settings = mocker.patch("immich_ml.sessions.ort.settings", autospec=True) + mock_settings.model_arena = True + + session = OrtSession("ViT-B-32__openai", providers=["CPUExecutionProvider"]) + + assert session.sess_options.enable_cpu_mem_arena + + def test_does_not_use_arena_if_disabled(self, mocker: MockerFixture) -> None: + mock_settings = mocker.patch("immich_ml.sessions.ort.settings", autospec=True) + mock_settings.model_arena = False + + session = OrtSession("ViT-B-32__openai", providers=["CPUExecutionProvider"]) + + assert not session.sess_options.enable_cpu_mem_arena + def test_sets_sess_options_kwarg(self) -> None: sess_options = ort.SessionOptions() session = OrtSession(