From 9ee35534bd15b0748b0825095e2ed601730a8151 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Tue, 23 Apr 2024 22:06:20 +0200 Subject: [PATCH 1/2] Add a private route to get items to auto refresh --- .../Kyoo.Core/Controllers/MiscRepository.cs | 35 +++++++++++++++++++ back/src/Kyoo.Core/Views/Admin/Misc.cs | 13 +++++++ 2 files changed, 48 insertions(+) diff --git a/back/src/Kyoo.Core/Controllers/MiscRepository.cs b/back/src/Kyoo.Core/Controllers/MiscRepository.cs index f3013a5e..06afb8ad 100644 --- a/back/src/Kyoo.Core/Controllers/MiscRepository.cs +++ b/back/src/Kyoo.Core/Controllers/MiscRepository.cs @@ -28,6 +28,7 @@ using Kyoo.Abstractions.Models; using Kyoo.Postgresql; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.DependencyInjection; +using static System.Text.Json.JsonNamingPolicy; namespace Kyoo.Core.Controllers; @@ -82,4 +83,38 @@ public class MiscRepository( .Concat(context.Movies.Select(x => x.Path)) .ToListAsync(); } + + public async Task> GetRefreshableItems(DateTime end) + { + IQueryable GetItems() + where T : class, IResource, IRefreshable + { + return context + .Set() + .Select(x => new RefreshableItem + { + Kind = CamelCase.ConvertName(typeof(T).Name), + Id = x.Id, + RefreshDate = x.NextMetadataRefresh!.Value + }); + } + + return await GetItems() + .Concat(GetItems()) + .Concat(GetItems()) + .Concat(GetItems()) + .Concat(GetItems()) + .Where(x => x.RefreshDate <= end) + .OrderBy(x => x.RefreshDate) + .ToListAsync(); + } +} + +public class RefreshableItem +{ + public string Kind { get; set; } + + public Guid Id { get; set; } + + public DateTime RefreshDate { get; set; } } diff --git a/back/src/Kyoo.Core/Views/Admin/Misc.cs b/back/src/Kyoo.Core/Views/Admin/Misc.cs index dd54fea7..b2a35991 100644 --- a/back/src/Kyoo.Core/Views/Admin/Misc.cs +++ b/back/src/Kyoo.Core/Views/Admin/Misc.cs @@ -16,6 +16,7 @@ // You should have received a copy of the GNU General Public License // along with Kyoo. If not, see . +using System; using System.Collections.Generic; using System.Threading.Tasks; using Kyoo.Abstractions.Models.Permissions; @@ -42,4 +43,16 @@ public class Misc(MiscRepository repo) : BaseApi { return repo.GetRegisteredPaths(); } + + /// + /// List items to refresh. + /// + /// The upper limit for the refresh date. + /// The items that should be refreshed before the given date + [HttpGet("/refreshables")] + [ProducesResponseType(StatusCodes.Status200OK)] + public Task> GetAllPaths([FromQuery] DateTime? date) + { + return repo.GetRefreshableItems(date ?? DateTime.UtcNow); + } } From 733817216fdf2a0dae7d605b93b9e0b8b5074242 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Tue, 23 Apr 2024 22:20:31 +0200 Subject: [PATCH 2/2] Add refresh listener on the scanner --- scanner/matcher/matcher.py | 2 +- scanner/providers/kyoo_client.py | 6 ++---- scanner/scanner/__init__.py | 2 ++ scanner/scanner/publisher.py | 9 +++++++++ scanner/scanner/refresher.py | 16 ++++++++++++++++ 5 files changed, 30 insertions(+), 5 deletions(-) create mode 100644 scanner/scanner/refresher.py diff --git a/scanner/matcher/matcher.py b/scanner/matcher/matcher.py index 6d4db3f0..d45c8381 100644 --- a/scanner/matcher/matcher.py +++ b/scanner/matcher/matcher.py @@ -198,7 +198,7 @@ class Matcher: "episode": id_episode, } - current = await self._client.get(kind, kyoo_id) + current = await self._client.get(f"{kind}/{kyoo_id}") if self._provider.name not in current["externalId"]: logger.error( f"Could not refresh metadata of {kind}/{kyoo_id}. Missing provider id." diff --git a/scanner/providers/kyoo_client.py b/scanner/providers/kyoo_client.py index 5c305ad6..d759d90e 100644 --- a/scanner/providers/kyoo_client.py +++ b/scanner/providers/kyoo_client.py @@ -128,11 +128,9 @@ class KyooClient: await self.delete_issue(path) - async def get( - self, kind: Literal["movie", "show", "season", "episode", "collection"], id: str - ): + async def get(self, path: str): async with self.client.get( - f"{self._url}/{kind}/{id}", + f"{self._url}/{path}", headers={"X-API-Key": self._api_key}, ) as r: if not r.ok: diff --git a/scanner/scanner/__init__.py b/scanner/scanner/__init__.py index 76951d53..e034b653 100644 --- a/scanner/scanner/__init__.py +++ b/scanner/scanner/__init__.py @@ -4,6 +4,7 @@ async def main(): import logging from .monitor import monitor from .scanner import scan + from .refresher import refresh from .publisher import Publisher from providers.kyoo_client import KyooClient @@ -15,4 +16,5 @@ async def main(): await asyncio.gather( monitor(path, publisher), scan(path, publisher, client), + refresh(publisher, client), ) diff --git a/scanner/scanner/publisher.py b/scanner/scanner/publisher.py index 2c4e1838..315855e1 100644 --- a/scanner/scanner/publisher.py +++ b/scanner/scanner/publisher.py @@ -1,6 +1,7 @@ import os from guessit.jsonutils import json from aio_pika import Message, connect_robust +from typing import Literal class Publisher: @@ -31,3 +32,11 @@ class Publisher: async def delete(self, path: str): await self._publish({"action": "delete", "path": path}) + + async def refresh( + self, + kind: Literal["collection", "show", "movie", "season", "episode"], + id: str, + **_kwargs, + ): + await self._publish({"action": "refresh", "kind": kind, "id": id}) diff --git a/scanner/scanner/refresher.py b/scanner/scanner/refresher.py new file mode 100644 index 00000000..44c086ae --- /dev/null +++ b/scanner/scanner/refresher.py @@ -0,0 +1,16 @@ +import asyncio +from logging import getLogger + +from providers.kyoo_client import KyooClient +from scanner.publisher import Publisher + + +logger = getLogger(__name__) + + +async def refresh(publisher: Publisher, client: KyooClient): + while True: + # Check for updates every 4 hours + await asyncio.sleep(60 * 60 * 4) + todo = await client.get("refreshables") + await asyncio.gather(*(publisher.refresh(**x) for x in todo))