From e1d0d2c186a3f8931c7c93c9f989242e4e18845f Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Thu, 9 Nov 2023 21:59:53 +0100 Subject: [PATCH] Cache identify request of the transcoder --- transcoder/Cargo.lock | 168 +++++++++++++++++++++++++++++++++++++ transcoder/Cargo.toml | 1 + transcoder/src/identify.rs | 12 +-- 3 files changed, 176 insertions(+), 5 deletions(-) diff --git a/transcoder/Cargo.lock b/transcoder/Cargo.lock index 105391fc..1afbb0e9 100644 --- a/transcoder/Cargo.lock +++ b/transcoder/Cargo.lock @@ -255,12 +255,29 @@ dependencies = [ "alloc-no-stdlib", ] +[[package]] +name = "allocator-api2" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5" + [[package]] name = "askama_escape" version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "619743e34b5ba4e9703bba34deac3427c72507c7159f5fd030aea8cac0cfe341" +[[package]] +name = "async-trait" +version = "0.1.74" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.32", +] + [[package]] name = "autocfg" version = "1.1.0" @@ -351,6 +368,42 @@ dependencies = [ "bytes", ] +[[package]] +name = "cached" +version = "0.46.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7c8c50262271cdf5abc979a5f76515c234e764fa025d1ba4862c0f0bcda0e95" +dependencies = [ + "ahash", + "async-trait", + "cached_proc_macro", + "cached_proc_macro_types", + "futures", + "hashbrown 0.14.0", + "instant", + "once_cell", + "thiserror", + "tokio", +] + +[[package]] +name = "cached_proc_macro" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c878c71c2821aa2058722038a59a67583a4240524687c6028571c9b395ded61f" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "cached_proc_macro_types" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a4f925191b4367301851c6d99b09890311d74b0d43f274c0b34c86d308a3663" + [[package]] name = "cc" version = "1.0.83" @@ -412,6 +465,41 @@ dependencies = [ "typenum", ] +[[package]] +name = "darling" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 1.0.109", +] + +[[package]] +name = "darling_macro" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e" +dependencies = [ + "darling_core", + "quote", + "syn 1.0.109", +] + [[package]] name = "deranged" version = "0.3.8" @@ -481,6 +569,20 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "futures" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + [[package]] name = "futures-channel" version = "0.3.28" @@ -488,6 +590,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" dependencies = [ "futures-core", + "futures-sink", ] [[package]] @@ -496,6 +599,12 @@ version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" +[[package]] +name = "futures-io" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bf34a163b5c4c52d0478a4d757da8fb65cabef42ba90515efee0f6f9fa45aaa" + [[package]] name = "futures-macro" version = "0.3.28" @@ -527,6 +636,7 @@ checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" dependencies = [ "futures-core", "futures-macro", + "futures-sink", "futures-task", "pin-project-lite", "pin-utils", @@ -590,6 +700,10 @@ name = "hashbrown" version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" +dependencies = [ + "ahash", + "allocator-api2", +] [[package]] name = "http" @@ -669,6 +783,12 @@ dependencies = [ "tokio-rustls", ] +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + [[package]] name = "idna" version = "0.4.0" @@ -700,6 +820,15 @@ dependencies = [ "serde", ] +[[package]] +name = "instant" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +dependencies = [ + "cfg-if", +] + [[package]] name = "ipnet" version = "2.8.0" @@ -1241,6 +1370,12 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + [[package]] name = "syn" version = "1.0.109" @@ -1263,6 +1398,26 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "thiserror" +version = "1.0.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9a7210f5c9a7156bb50aa36aed4c95afb51df0df00713949448cf9e97d382d2" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.32", +] + [[package]] name = "time" version = "0.3.28" @@ -1320,9 +1475,21 @@ dependencies = [ "pin-project-lite", "signal-hook-registry", "socket2 0.5.3", + "tokio-macros", "windows-sys", ] +[[package]] +name = "tokio-macros" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.32", +] + [[package]] name = "tokio-rustls" version = "0.24.1" @@ -1380,6 +1547,7 @@ version = "0.1.0" dependencies = [ "actix-files", "actix-web", + "cached", "derive_more", "json", "rand", diff --git a/transcoder/Cargo.toml b/transcoder/Cargo.toml index 23d66708..a7574893 100644 --- a/transcoder/Cargo.toml +++ b/transcoder/Cargo.toml @@ -13,3 +13,4 @@ derive_more = "0.99.17" reqwest = { version = "0.11.16", default_features = false, features = ["json", "rustls-tls"] } utoipa = { version = "3", features = ["actix_extras"] } json = "0.12.4" +cached = { version = "0.46.1", features = ["async"] } diff --git a/transcoder/src/identify.rs b/transcoder/src/identify.rs index 17d74293..5fd809a2 100644 --- a/transcoder/src/identify.rs +++ b/transcoder/src/identify.rs @@ -1,3 +1,4 @@ +use cached::proc_macro::cached; use json::JsonValue; use serde::Serialize; use std::{ @@ -12,7 +13,7 @@ use utoipa::ToSchema; use crate::transcode::Quality; -#[derive(Serialize, ToSchema)] +#[derive(Serialize, ToSchema, Clone)] #[serde(rename_all = "camelCase")] pub struct MediaInfo { /// The sha1 of the video file. @@ -35,7 +36,7 @@ pub struct MediaInfo { pub chapters: Vec, } -#[derive(Serialize, ToSchema)] +#[derive(Serialize, ToSchema, Clone)] #[serde(rename_all = "camelCase")] pub struct Video { /// The codec of this stream (defined as the RFC 6381). @@ -52,7 +53,7 @@ pub struct Video { pub bitrate: u32, } -#[derive(Serialize, ToSchema)] +#[derive(Serialize, ToSchema, Clone)] #[serde(rename_all = "camelCase")] pub struct Audio { /// The index of this track on the media. @@ -69,7 +70,7 @@ pub struct Audio { pub is_forced: bool, } -#[derive(Serialize, ToSchema)] +#[derive(Serialize, ToSchema, Clone)] #[serde(rename_all = "camelCase")] pub struct Subtitle { /// The index of this track on the media. @@ -90,7 +91,7 @@ pub struct Subtitle { pub link: Option, } -#[derive(Serialize, ToSchema)] +#[derive(Serialize, ToSchema, Clone)] #[serde(rename_all = "camelCase")] pub struct Chapter { /// The start time of the chapter (in second from the start of the episode). @@ -127,6 +128,7 @@ async fn extract(path: String, sha: &String, subs: &Vec) { .expect("Error running ffmpeg extract"); } +#[cached(sync_writes = true, time = 30)] pub async fn identify(path: String) -> Option { let extension_table: HashMap<&str, &str> = HashMap::from([("subrip", "srt"), ("ass", "ass"), ("vtt", "vtt")]);