mirror of
https://github.com/immich-app/immich.git
synced 2025-05-24 01:12:58 -04:00
4bfef2460a
* model benchmarks * minor fixes * formatting * docs build * maybe fix reference * clarify optimal * use emojis * wording * wording * clarify optimal wording * bolding * more detailed instructions * clarify edge case fix * early exit in dim loop
172 lines
4.6 KiB
Python
172 lines
4.6 KiB
Python
import subprocess
|
|
from pathlib import Path
|
|
|
|
from exporters.constants import ModelSource
|
|
|
|
from immich_model_exporter import clean_name
|
|
from immich_model_exporter.exporters.constants import SOURCE_TO_TASK
|
|
|
|
mclip = [
|
|
"M-CLIP/LABSE-Vit-L-14",
|
|
"M-CLIP/XLM-Roberta-Large-Vit-B-16Plus",
|
|
"M-CLIP/XLM-Roberta-Large-Vit-B-32",
|
|
"M-CLIP/XLM-Roberta-Large-Vit-L-14",
|
|
]
|
|
|
|
openclip = [
|
|
"RN101__openai",
|
|
"RN101__yfcc15m",
|
|
"RN50__cc12m",
|
|
"RN50__openai",
|
|
"RN50__yfcc15m",
|
|
"RN50x16__openai",
|
|
"RN50x4__openai",
|
|
"RN50x64__openai",
|
|
"ViT-B-16-SigLIP-256__webli",
|
|
"ViT-B-16-SigLIP-384__webli",
|
|
"ViT-B-16-SigLIP-512__webli",
|
|
"ViT-B-16-SigLIP-i18n-256__webli",
|
|
"ViT-B-16-SigLIP2__webli",
|
|
"ViT-B-16-SigLIP__webli",
|
|
"ViT-B-16-plus-240__laion400m_e31",
|
|
"ViT-B-16-plus-240__laion400m_e32",
|
|
"ViT-B-16__laion400m_e31",
|
|
"ViT-B-16__laion400m_e32",
|
|
"ViT-B-16__openai",
|
|
"ViT-B-32-SigLIP2-256__webli",
|
|
"ViT-B-32__laion2b-s34b-b79k",
|
|
"ViT-B-32__laion2b_e16",
|
|
"ViT-B-32__laion400m_e31",
|
|
"ViT-B-32__laion400m_e32",
|
|
"ViT-B-32__openai",
|
|
"ViT-H-14-378-quickgelu__dfn5b",
|
|
"ViT-H-14-quickgelu__dfn5b",
|
|
"ViT-H-14__laion2b-s32b-b79k",
|
|
"ViT-L-14-336__openai",
|
|
"ViT-L-14-quickgelu__dfn2b",
|
|
"ViT-L-14__laion2b-s32b-b82k",
|
|
"ViT-L-14__laion400m_e31",
|
|
"ViT-L-14__laion400m_e32",
|
|
"ViT-L-14__openai",
|
|
"ViT-L-16-SigLIP-256__webli",
|
|
"ViT-L-16-SigLIP-384__webli",
|
|
"ViT-L-16-SigLIP2-256__webli",
|
|
"ViT-L-16-SigLIP2-384__webli",
|
|
"ViT-L-16-SigLIP2-512__webli",
|
|
"ViT-SO400M-14-SigLIP-384__webli",
|
|
"ViT-SO400M-14-SigLIP2-378__webli",
|
|
"ViT-SO400M-14-SigLIP2__webli",
|
|
"ViT-SO400M-16-SigLIP2-256__webli",
|
|
"ViT-SO400M-16-SigLIP2-384__webli",
|
|
"ViT-SO400M-16-SigLIP2-512__webli",
|
|
"ViT-gopt-16-SigLIP2-256__webli",
|
|
"ViT-gopt-16-SigLIP2-384__webli",
|
|
"nllb-clip-base-siglip__mrl",
|
|
"nllb-clip-base-siglip__v1",
|
|
"nllb-clip-large-siglip__mrl",
|
|
"nllb-clip-large-siglip__v1",
|
|
"xlm-roberta-base-ViT-B-32__laion5b_s13b_b90k",
|
|
"xlm-roberta-large-ViT-H-14__frozen_laion5b_s13b_b90k",
|
|
]
|
|
|
|
insightface = [
|
|
"antelopev2",
|
|
"buffalo_l",
|
|
"buffalo_m",
|
|
"buffalo_s",
|
|
]
|
|
|
|
|
|
def export_models(models: list[str], source: ModelSource) -> None:
|
|
profiling_dir = Path("profiling")
|
|
profiling_dir.mkdir(exist_ok=True)
|
|
for model in models:
|
|
try:
|
|
model_dir = f"models/{clean_name(model)}"
|
|
task = SOURCE_TO_TASK[source]
|
|
|
|
print(f"Processing model {model}")
|
|
subprocess.check_call(["python", "-m", "immich_model_exporter", "export", model, source])
|
|
subprocess.check_call(
|
|
[
|
|
"python",
|
|
"-m",
|
|
"immich_model_exporter",
|
|
"profile",
|
|
model_dir,
|
|
task,
|
|
"--output_path",
|
|
profiling_dir / f"{model}.json",
|
|
]
|
|
)
|
|
subprocess.check_call(["python", "-m", "immich_model_exporter", "upload", model_dir])
|
|
except Exception as e:
|
|
print(f"Failed to export model {model}: {e}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
export_models(mclip, ModelSource.MCLIP)
|
|
export_models(openclip, ModelSource.OPENCLIP)
|
|
export_models(insightface, ModelSource.INSIGHTFACE)
|
|
|
|
Path("results").mkdir(exist_ok=True)
|
|
subprocess.check_call(
|
|
[
|
|
"python",
|
|
"clip_benchmark",
|
|
"eval",
|
|
"--pretrained_model",
|
|
*[name.replace("__", ",") for name in openclip],
|
|
"--task",
|
|
"zeroshot_retrieval",
|
|
"--dataset",
|
|
"crossmodal3600",
|
|
"--batch_size",
|
|
"64",
|
|
"--language",
|
|
"ar",
|
|
"bn",
|
|
"cs",
|
|
"da",
|
|
"de",
|
|
"el",
|
|
"en",
|
|
"es",
|
|
"fa",
|
|
"fi",
|
|
"fil",
|
|
"fr",
|
|
"he",
|
|
"hi",
|
|
"hr",
|
|
"hu",
|
|
"id",
|
|
"it",
|
|
"ja",
|
|
"ko",
|
|
"mi",
|
|
"nl",
|
|
"no",
|
|
"pl",
|
|
"pt",
|
|
"quz",
|
|
"ro",
|
|
"ru",
|
|
"sv",
|
|
"sw",
|
|
"te",
|
|
"th",
|
|
"tr",
|
|
"uk",
|
|
"vi",
|
|
"zh",
|
|
"--recall_k",
|
|
"1",
|
|
"5",
|
|
"10",
|
|
"--no_amp",
|
|
"--output",
|
|
"results/{dataset}_{language}_{model}_{pretrained}.json",
|
|
]
|
|
)
|