From 14a8f5c1e53d6d963d01725f748a873b9075efdc Mon Sep 17 00:00:00 2001 From: Joe Milazzo Date: Fri, 20 Jun 2025 14:09:29 -0500 Subject: [PATCH] Scrobbling Stability (#3863) Co-authored-by: Amelia <77553571+Fesaa@users.noreply.github.com> --- .../Extensions/QueryableExtensionsTests.cs | 4 +- .../Services/ExternalMetadataServiceTests.cs | 2 + API.Tests/Services/ScrobblingServiceTests.cs | 415 +++- API/Controllers/ScrobblingController.cs | 16 +- API/DTOs/Scrobbling/ScrobbleEventDto.cs | 1 + API/DTOs/Scrobbling/ScrobbleResponseDto.cs | 1 + .../Repositories/ScrobbleEventRepository.cs | 35 +- API/Entities/Scrobble/ScrobbleEvent.cs | 10 + .../ApplicationServiceExtensions.cs | 1 + .../RestrictByAgeExtensions.cs | 6 + API/Services/Plus/KavitaPlusApiService.cs | 75 + API/Services/Plus/ScrobblingService.cs | 1724 +++++++++-------- .../app/_models/scrobbling/scrobble-event.ts | 1 + .../src/app/_services/scrobbling.service.ts | 6 +- .../user-scrobble-history.component.html | 21 +- .../user-scrobble-history.component.ts | 84 +- .../manage-scrobble-errors.component.html | 2 +- .../app/typeahead/_models/selection-model.ts | 22 + UI/Web/src/assets/langs/en.json | 2 + 19 files changed, 1622 insertions(+), 806 deletions(-) create mode 100644 API/Services/Plus/KavitaPlusApiService.cs diff --git a/API.Tests/Extensions/QueryableExtensionsTests.cs b/API.Tests/Extensions/QueryableExtensionsTests.cs index 866e0202c..96d74b46d 100644 --- a/API.Tests/Extensions/QueryableExtensionsTests.cs +++ b/API.Tests/Extensions/QueryableExtensionsTests.cs @@ -67,7 +67,7 @@ public class QueryableExtensionsTests [Theory] [InlineData(true, 2)] - [InlineData(false, 1)] + [InlineData(false, 2)] public void RestrictAgainstAgeRestriction_Genre_ShouldRestrictEverythingAboveTeen(bool includeUnknowns, int expectedCount) { var items = new List() @@ -94,7 +94,7 @@ public class QueryableExtensionsTests [Theory] [InlineData(true, 2)] - [InlineData(false, 1)] + [InlineData(false, 2)] public void RestrictAgainstAgeRestriction_Tag_ShouldRestrictEverythingAboveTeen(bool includeUnknowns, int expectedCount) { var items = new List() diff --git a/API.Tests/Services/ExternalMetadataServiceTests.cs b/API.Tests/Services/ExternalMetadataServiceTests.cs index 8310ed269..833e8fe5f 100644 --- a/API.Tests/Services/ExternalMetadataServiceTests.cs +++ b/API.Tests/Services/ExternalMetadataServiceTests.cs @@ -2935,6 +2935,8 @@ public class ExternalMetadataServiceTests : AbstractDbTest metadataSettings.EnableTags = false; metadataSettings.EnablePublicationStatus = false; metadataSettings.EnableStartDate = false; + metadataSettings.FieldMappings = []; + metadataSettings.AgeRatingMappings = new Dictionary(); Context.MetadataSettings.Update(metadataSettings); await Context.SaveChangesAsync(); diff --git a/API.Tests/Services/ScrobblingServiceTests.cs b/API.Tests/Services/ScrobblingServiceTests.cs index 50398a146..9245c8ecd 100644 --- a/API.Tests/Services/ScrobblingServiceTests.cs +++ b/API.Tests/Services/ScrobblingServiceTests.cs @@ -1,11 +1,17 @@ -using System.Linq; +using System.Collections.Generic; +using System.Linq; +using System.Threading; using System.Threading.Tasks; +using API.Data.Repositories; using API.DTOs.Scrobbling; +using API.Entities; using API.Entities.Enums; +using API.Entities.Scrobble; using API.Helpers.Builders; using API.Services; using API.Services.Plus; using API.SignalR; +using Kavita.Common; using Microsoft.Extensions.Logging; using NSubstitute; using Xunit; @@ -15,11 +21,33 @@ namespace API.Tests.Services; public class ScrobblingServiceTests : AbstractDbTest { + private const int ChapterPages = 100; + + /// + /// { + /// "Issuer": "Issuer", + /// "Issued At": "2025-06-15T21:01:57.615Z", + /// "Expiration": "2200-06-15T21:01:57.615Z" + /// } + /// + /// Our UnitTests will fail in 2200 :( + private const string ValidJwtToken = + "eyJhbGciOiJIUzI1NiJ9.eyJJc3N1ZXIiOiJJc3N1ZXIiLCJleHAiOjcyNzI0NTAxMTcsImlhdCI6MTc1MDAyMTMxN30.zADmcGq_BfxbcV8vy4xw5Cbzn4COkmVINxgqpuL17Ng"; + private readonly ScrobblingService _service; private readonly ILicenseService _licenseService; private readonly ILocalizationService _localizationService; private readonly ILogger _logger; private readonly IEmailService _emailService; + private readonly IKavitaPlusApiService _kavitaPlusApiService; + /// + /// IReaderService, without the ScrobblingService injected + /// + private readonly IReaderService _readerService; + /// + /// IReaderService, with the _service injected + /// + private readonly IReaderService _hookedUpReaderService; public ScrobblingServiceTests() { @@ -27,8 +55,24 @@ public class ScrobblingServiceTests : AbstractDbTest _localizationService = Substitute.For(); _logger = Substitute.For>(); _emailService = Substitute.For(); + _kavitaPlusApiService = Substitute.For(); - _service = new ScrobblingService(UnitOfWork, Substitute.For(), _logger, _licenseService, _localizationService, _emailService); + _service = new ScrobblingService(UnitOfWork, Substitute.For(), _logger, _licenseService, + _localizationService, _emailService, _kavitaPlusApiService); + + _readerService = new ReaderService(UnitOfWork, + Substitute.For>(), + Substitute.For(), + Substitute.For(), + Substitute.For(), + Substitute.For()); // Do not use the actual one + + _hookedUpReaderService = new ReaderService(UnitOfWork, + Substitute.For>(), + Substitute.For(), + Substitute.For(), + Substitute.For(), + _service); } protected override async Task ResetDb() @@ -46,6 +90,30 @@ public class ScrobblingServiceTests : AbstractDbTest var series = new SeriesBuilder("Test Series") .WithFormat(MangaFormat.Archive) .WithMetadata(new SeriesMetadataBuilder().Build()) + .WithVolume(new VolumeBuilder("Volume 1") + .WithChapters([ + new ChapterBuilder("1") + .WithPages(ChapterPages) + .Build(), + new ChapterBuilder("2") + .WithPages(ChapterPages) + .Build(), + new ChapterBuilder("3") + .WithPages(ChapterPages) + .Build()]) + .Build()) + .WithVolume(new VolumeBuilder("Volume 2") + .WithChapters([ + new ChapterBuilder("4") + .WithPages(ChapterPages) + .Build(), + new ChapterBuilder("5") + .WithPages(ChapterPages) + .Build(), + new ChapterBuilder("6") + .WithPages(ChapterPages) + .Build()]) + .Build()) .Build(); var library = new LibraryBuilder("Test Library", LibraryType.Manga) @@ -67,6 +135,296 @@ public class ScrobblingServiceTests : AbstractDbTest await UnitOfWork.CommitAsync(); } + private async Task CreateScrobbleEvent(int? seriesId = null) + { + var evt = new ScrobbleEvent + { + ScrobbleEventType = ScrobbleEventType.ChapterRead, + Format = PlusMediaFormat.Manga, + SeriesId = seriesId ?? 0, + LibraryId = 0, + AppUserId = 0, + }; + + if (seriesId != null) + { + var series = await UnitOfWork.SeriesRepository.GetSeriesByIdAsync(seriesId.Value); + if (series != null) evt.Series = series; + } + + return evt; + } + + + #region K+ API Request Tests + + [Fact] + public async Task PostScrobbleUpdate_AuthErrors() + { + _kavitaPlusApiService.PostScrobbleUpdate(null!, "") + .ReturnsForAnyArgs(new ScrobbleResponseDto() + { + ErrorMessage = "Unauthorized" + }); + + var evt = await CreateScrobbleEvent(); + await Assert.ThrowsAsync(async () => + { + await _service.PostScrobbleUpdate(new ScrobbleDto(), "", evt); + }); + Assert.True(evt.IsErrored); + Assert.Equal("Kavita+ subscription no longer active", evt.ErrorDetails); + } + + [Fact] + public async Task PostScrobbleUpdate_UnknownSeriesLoggedAsError() + { + _kavitaPlusApiService.PostScrobbleUpdate(null!, "") + .ReturnsForAnyArgs(new ScrobbleResponseDto() + { + ErrorMessage = "Unknown Series" + }); + + await SeedData(); + var evt = await CreateScrobbleEvent(1); + + await _service.PostScrobbleUpdate(new ScrobbleDto(), "", evt); + await UnitOfWork.CommitAsync(); + Assert.True(evt.IsErrored); + + var series = await UnitOfWork.SeriesRepository.GetSeriesByIdAsync(1); + Assert.NotNull(series); + Assert.True(series.IsBlacklisted); + + var errors = await UnitOfWork.ScrobbleRepository.GetAllScrobbleErrorsForSeries(1); + Assert.Single(errors); + Assert.Equal("Series cannot be matched for Scrobbling", errors.First().Comment); + Assert.Equal(series.Id, errors.First().SeriesId); + } + + [Fact] + public async Task PostScrobbleUpdate_InvalidAccessToken() + { + _kavitaPlusApiService.PostScrobbleUpdate(null!, "") + .ReturnsForAnyArgs(new ScrobbleResponseDto() + { + ErrorMessage = "Access token is invalid" + }); + + var evt = await CreateScrobbleEvent(); + + await Assert.ThrowsAsync(async () => + { + await _service.PostScrobbleUpdate(new ScrobbleDto(), "", evt); + }); + + Assert.True(evt.IsErrored); + Assert.Equal("Access Token needs to be rotated to continue scrobbling", evt.ErrorDetails); + } + + #endregion + + #region K+ API Request data tests + + [Fact] + public async Task ProcessReadEvents_CreatesNoEventsWhenNoProgress() + { + await ResetDb(); + await SeedData(); + + // Set Returns + _licenseService.HasActiveLicense().Returns(Task.FromResult(true)); + _kavitaPlusApiService.GetRateLimit(Arg.Any(), Arg.Any()) + .Returns(100); + + var user = await UnitOfWork.UserRepository.GetUserByIdAsync(1); + Assert.NotNull(user); + + // Ensure CanProcessScrobbleEvent returns true + user.AniListAccessToken = ValidJwtToken; + UnitOfWork.UserRepository.Update(user); + await UnitOfWork.CommitAsync(); + + var chapter = await UnitOfWork.ChapterRepository.GetChapterAsync(4); + Assert.NotNull(chapter); + + var volume = await UnitOfWork.VolumeRepository.GetVolumeAsync(1, VolumeIncludes.Chapters); + Assert.NotNull(volume); + + // Call Scrobble without having any progress + await _service.ScrobbleReadingUpdate(1, 1); + var events = await UnitOfWork.ScrobbleRepository.GetAllEventsForSeries(1); + Assert.Empty(events); + } + + [Fact] + public async Task ProcessReadEvents_UpdateVolumeAndChapterData() + { + await ResetDb(); + await SeedData(); + + // Set Returns + _licenseService.HasActiveLicense().Returns(Task.FromResult(true)); + _kavitaPlusApiService.GetRateLimit(Arg.Any(), Arg.Any()) + .Returns(100); + + var user = await UnitOfWork.UserRepository.GetUserByIdAsync(1); + Assert.NotNull(user); + + // Ensure CanProcessScrobbleEvent returns true + user.AniListAccessToken = ValidJwtToken; + UnitOfWork.UserRepository.Update(user); + await UnitOfWork.CommitAsync(); + + var chapter = await UnitOfWork.ChapterRepository.GetChapterAsync(4); + Assert.NotNull(chapter); + + var volume = await UnitOfWork.VolumeRepository.GetVolumeAsync(1, VolumeIncludes.Chapters); + Assert.NotNull(volume); + + // Mark something as read to trigger event creation + await _readerService.MarkChaptersAsRead(user, 1, new List() {volume.Chapters[0]}); + await UnitOfWork.CommitAsync(); + + // Call Scrobble while having some progress + await _service.ScrobbleReadingUpdate(user.Id, 1); + var events = await UnitOfWork.ScrobbleRepository.GetAllEventsForSeries(1); + Assert.Single(events); + + // Give it some (more) read progress + await _readerService.MarkChaptersAsRead(user, 1, volume.Chapters); + await _readerService.MarkChaptersAsRead(user, 1, [chapter]); + await UnitOfWork.CommitAsync(); + + await _service.ProcessUpdatesSinceLastSync(); + + await _kavitaPlusApiService.Received(1).PostScrobbleUpdate( + Arg.Is(data => + data.ChapterNumber == (int)chapter.MaxNumber && + data.VolumeNumber == (int)volume.MaxNumber + ), + Arg.Any()); + } + + #endregion + + #region Scrobble Reading Update Tests + + [Fact] + public async Task ScrobbleReadingUpdate_IgnoreNoLicense() + { + await ResetDb(); + await SeedData(); + + _licenseService.HasActiveLicense().Returns(false); + + await _service.ScrobbleReadingUpdate(1, 1); + var events = await UnitOfWork.ScrobbleRepository.GetAllEventsForSeries(1); + Assert.Empty(events); + } + + [Fact] + public async Task ScrobbleReadingUpdate_RemoveWhenNoProgress() + { + await ResetDb(); + await SeedData(); + + _licenseService.HasActiveLicense().Returns(true); + + var user = await UnitOfWork.UserRepository.GetUserByIdAsync(1); + Assert.NotNull(user); + + var volume = await UnitOfWork.VolumeRepository.GetVolumeAsync(1, VolumeIncludes.Chapters); + Assert.NotNull(volume); + + await _readerService.MarkChaptersAsRead(user, 1, new List() {volume.Chapters[0]}); + await UnitOfWork.CommitAsync(); + + await _service.ScrobbleReadingUpdate(1, 1); + var events = await UnitOfWork.ScrobbleRepository.GetAllEventsForSeries(1); + Assert.Single(events); + + var readEvent = events.First(); + Assert.False(readEvent.IsProcessed); + + await _hookedUpReaderService.MarkSeriesAsUnread(user, 1); + await UnitOfWork.CommitAsync(); + + // Existing event is deleted + await _service.ScrobbleReadingUpdate(1, 1); + events = await UnitOfWork.ScrobbleRepository.GetAllEventsForSeries(1); + Assert.Empty(events); + + await _hookedUpReaderService.MarkSeriesAsUnread(user, 1); + await UnitOfWork.CommitAsync(); + + // No new events are added + events = await UnitOfWork.ScrobbleRepository.GetAllEventsForSeries(1); + Assert.Empty(events); + } + + [Fact] + public async Task ScrobbleReadingUpdate_UpdateExistingNotIsProcessed() + { + await ResetDb(); + await SeedData(); + + var user = await UnitOfWork.UserRepository.GetUserByIdAsync(1); + Assert.NotNull(user); + + var chapter1 = await UnitOfWork.ChapterRepository.GetChapterAsync(1); + var chapter2 = await UnitOfWork.ChapterRepository.GetChapterAsync(2); + var chapter3 = await UnitOfWork.ChapterRepository.GetChapterAsync(3); + Assert.NotNull(chapter1); + Assert.NotNull(chapter2); + Assert.NotNull(chapter3); + + _licenseService.HasActiveLicense().Returns(true); + + var events = await UnitOfWork.ScrobbleRepository.GetAllEventsForSeries(1); + Assert.Empty(events); + + + await _readerService.MarkChaptersAsRead(user, 1, [chapter1]); + await UnitOfWork.CommitAsync(); + + // Scrobble update + await _service.ScrobbleReadingUpdate(1, 1); + events = await UnitOfWork.ScrobbleRepository.GetAllEventsForSeries(1); + Assert.Single(events); + + var readEvent = events[0]; + Assert.False(readEvent.IsProcessed); + Assert.Equal(1, readEvent.ChapterNumber); + + // Mark as processed + readEvent.IsProcessed = true; + await UnitOfWork.CommitAsync(); + + await _readerService.MarkChaptersAsRead(user, 1, [chapter2]); + await UnitOfWork.CommitAsync(); + + // Scrobble update + await _service.ScrobbleReadingUpdate(1, 1); + events = await UnitOfWork.ScrobbleRepository.GetAllEventsForSeries(1); + Assert.Equal(2, events.Count); + Assert.Single(events.Where(e => e.IsProcessed).ToList()); + Assert.Single(events.Where(e => !e.IsProcessed).ToList()); + + // Should update the existing non processed event + await _readerService.MarkChaptersAsRead(user, 1, [chapter3]); + await UnitOfWork.CommitAsync(); + + // Scrobble update + await _service.ScrobbleReadingUpdate(1, 1); + events = await UnitOfWork.ScrobbleRepository.GetAllEventsForSeries(1); + Assert.Equal(2, events.Count); + Assert.Single(events.Where(e => e.IsProcessed).ToList()); + Assert.Single(events.Where(e => !e.IsProcessed).ToList()); + } + + #endregion + #region ScrobbleWantToReadUpdate Tests [Fact] @@ -203,6 +561,59 @@ public class ScrobblingServiceTests : AbstractDbTest #endregion + #region Scrobble Rating Update Test + + [Fact] + public async Task ScrobbleRatingUpdate_IgnoreNoLicense() + { + await ResetDb(); + await SeedData(); + + _licenseService.HasActiveLicense().Returns(false); + + await _service.ScrobbleRatingUpdate(1, 1, 1); + var events = await UnitOfWork.ScrobbleRepository.GetAllEventsForSeries(1); + Assert.Empty(events); + } + + [Fact] + public async Task ScrobbleRatingUpdate_UpdateExistingNotIsProcessed() + { + await ResetDb(); + await SeedData(); + + _licenseService.HasActiveLicense().Returns(true); + + var user = await UnitOfWork.UserRepository.GetUserByIdAsync(1); + Assert.NotNull(user); + + var series = await UnitOfWork.SeriesRepository.GetSeriesByIdAsync(1); + Assert.NotNull(series); + + await _service.ScrobbleRatingUpdate(user.Id, series.Id, 1); + var events = await UnitOfWork.ScrobbleRepository.GetAllEventsForSeries(1); + Assert.Single(events); + Assert.Equal(1, events.First().Rating); + + // Mark as processed + events.First().IsProcessed = true; + await UnitOfWork.CommitAsync(); + + await _service.ScrobbleRatingUpdate(user.Id, series.Id, 5); + events = await UnitOfWork.ScrobbleRepository.GetAllEventsForSeries(1); + Assert.Equal(2, events.Count); + Assert.Single(events, evt => evt.IsProcessed); + Assert.Single(events, evt => !evt.IsProcessed); + + await _service.ScrobbleRatingUpdate(user.Id, series.Id, 5); + events = await UnitOfWork.ScrobbleRepository.GetAllEventsForSeries(1); + Assert.Single(events, evt => !evt.IsProcessed); + Assert.Equal(5, events.First(evt => !evt.IsProcessed).Rating); + + } + + #endregion + [Theory] [InlineData("https://anilist.co/manga/35851/Byeontaega-Doeja/", 35851)] [InlineData("https://anilist.co/manga/30105", 30105)] diff --git a/API/Controllers/ScrobblingController.cs b/API/Controllers/ScrobblingController.cs index 3904cb8e0..986f4f8e7 100644 --- a/API/Controllers/ScrobblingController.cs +++ b/API/Controllers/ScrobblingController.cs @@ -254,7 +254,7 @@ public class ScrobblingController : BaseApiController } /// - /// Adds a hold against the Series for user's scrobbling + /// Remove a hold against the Series for user's scrobbling /// /// /// @@ -281,4 +281,18 @@ public class ScrobblingController : BaseApiController var user = await _unitOfWork.UserRepository.GetUserByIdAsync(User.GetUserId()); return Ok(user is {HasRunScrobbleEventGeneration: true}); } + + /// + /// Delete the given scrobble events if they belong to that user + /// + /// + /// + [HttpPost("bulk-remove-events")] + public async Task BulkRemoveScrobbleEvents(IList eventIds) + { + var events = await _unitOfWork.ScrobbleRepository.GetUserEvents(User.GetUserId(), eventIds); + _unitOfWork.ScrobbleRepository.Remove(events); + await _unitOfWork.CommitAsync(); + return Ok(); + } } diff --git a/API/DTOs/Scrobbling/ScrobbleEventDto.cs b/API/DTOs/Scrobbling/ScrobbleEventDto.cs index 7b1ccd75a..562d923ff 100644 --- a/API/DTOs/Scrobbling/ScrobbleEventDto.cs +++ b/API/DTOs/Scrobbling/ScrobbleEventDto.cs @@ -5,6 +5,7 @@ namespace API.DTOs.Scrobbling; public sealed record ScrobbleEventDto { + public long Id { get; init; } public string SeriesName { get; set; } public int SeriesId { get; set; } public int LibraryId { get; set; } diff --git a/API/DTOs/Scrobbling/ScrobbleResponseDto.cs b/API/DTOs/Scrobbling/ScrobbleResponseDto.cs index 53d3a0cc9..ad66729d0 100644 --- a/API/DTOs/Scrobbling/ScrobbleResponseDto.cs +++ b/API/DTOs/Scrobbling/ScrobbleResponseDto.cs @@ -8,5 +8,6 @@ public sealed record ScrobbleResponseDto { public bool Successful { get; set; } public string? ErrorMessage { get; set; } + public string? ExtraInformation {get; set;} public int RateLeft { get; set; } } diff --git a/API/Data/Repositories/ScrobbleEventRepository.cs b/API/Data/Repositories/ScrobbleEventRepository.cs index c5f30c2ec..144a3b88e 100644 --- a/API/Data/Repositories/ScrobbleEventRepository.cs +++ b/API/Data/Repositories/ScrobbleEventRepository.cs @@ -29,8 +29,23 @@ public interface IScrobbleRepository Task> GetAllScrobbleErrorsForSeries(int seriesId); Task ClearScrobbleErrors(); Task HasErrorForSeries(int seriesId); - Task GetEvent(int userId, int seriesId, ScrobbleEventType eventType); + /// + /// Get all events for a specific user and type + /// + /// + /// + /// + /// If true, only returned not processed events + /// + Task GetEvent(int userId, int seriesId, ScrobbleEventType eventType, bool isNotProcessed = false); Task> GetUserEventsForSeries(int userId, int seriesId); + /// + /// Return the events with given ids, when belonging to the passed user + /// + /// + /// + /// + Task> GetUserEvents(int userId, IList scrobbleEventIds); Task> GetUserEvents(int userId, ScrobbleEventFilter filter, UserParams pagination); Task> GetAllEventsForSeries(int seriesId); Task> GetAllEventsWithSeriesIds(IEnumerable seriesIds); @@ -146,22 +161,32 @@ public class ScrobbleRepository : IScrobbleRepository return await _context.ScrobbleError.AnyAsync(n => n.SeriesId == seriesId); } - public async Task GetEvent(int userId, int seriesId, ScrobbleEventType eventType) + public async Task GetEvent(int userId, int seriesId, ScrobbleEventType eventType, bool isNotProcessed = false) { - return await _context.ScrobbleEvent.FirstOrDefaultAsync(e => - e.AppUserId == userId && e.SeriesId == seriesId && e.ScrobbleEventType == eventType); + return await _context.ScrobbleEvent + .Where(e => e.AppUserId == userId && e.SeriesId == seriesId && e.ScrobbleEventType == eventType) + .WhereIf(isNotProcessed, e => !e.IsProcessed) + .OrderBy(e => e.LastModifiedUtc) + .FirstOrDefaultAsync(); } public async Task> GetUserEventsForSeries(int userId, int seriesId) { return await _context.ScrobbleEvent - .Where(e => e.AppUserId == userId && !e.IsProcessed) + .Where(e => e.AppUserId == userId && !e.IsProcessed && e.SeriesId == seriesId) .Include(e => e.Series) .OrderBy(e => e.LastModifiedUtc) .AsSplitQuery() .ToListAsync(); } + public async Task> GetUserEvents(int userId, IList scrobbleEventIds) + { + return await _context.ScrobbleEvent + .Where(e => e.AppUserId == userId && scrobbleEventIds.Contains(e.Id)) + .ToListAsync(); + } + public async Task> GetUserEvents(int userId, ScrobbleEventFilter filter, UserParams pagination) { var query = _context.ScrobbleEvent diff --git a/API/Entities/Scrobble/ScrobbleEvent.cs b/API/Entities/Scrobble/ScrobbleEvent.cs index b8708c115..8adfdcc2e 100644 --- a/API/Entities/Scrobble/ScrobbleEvent.cs +++ b/API/Entities/Scrobble/ScrobbleEvent.cs @@ -68,4 +68,14 @@ public class ScrobbleEvent : IEntityDate public DateTime LastModified { get; set; } public DateTime CreatedUtc { get; set; } public DateTime LastModifiedUtc { get; set; } + + /// + /// Sets the ErrorDetail and marks the event as + /// + /// + public void SetErrorMessage(string errorMessage) + { + ErrorDetails = errorMessage; + IsErrored = true; + } } diff --git a/API/Extensions/ApplicationServiceExtensions.cs b/API/Extensions/ApplicationServiceExtensions.cs index 0d1d7d561..bd4783f25 100644 --- a/API/Extensions/ApplicationServiceExtensions.cs +++ b/API/Extensions/ApplicationServiceExtensions.cs @@ -76,6 +76,7 @@ public static class ApplicationServiceExtensions services.AddScoped(); + services.AddScoped(); services.AddScoped(); services.AddScoped(); services.AddScoped(); diff --git a/API/Extensions/QueryExtensions/RestrictByAgeExtensions.cs b/API/Extensions/QueryExtensions/RestrictByAgeExtensions.cs index aef595596..350372e5b 100644 --- a/API/Extensions/QueryExtensions/RestrictByAgeExtensions.cs +++ b/API/Extensions/QueryExtensions/RestrictByAgeExtensions.cs @@ -56,6 +56,12 @@ public static class RestrictByAgeExtensions sm.Metadata.AgeRating <= restriction.AgeRating && sm.Metadata.AgeRating > AgeRating.Unknown)); } + /// + /// Returns all Genres where any of the linked Series/Chapters are less than or equal to restriction age rating + /// + /// + /// + /// public static IQueryable RestrictAgainstAgeRestriction(this IQueryable queryable, AgeRestriction restriction) { if (restriction.AgeRating == AgeRating.NotApplicable) return queryable; diff --git a/API/Services/Plus/KavitaPlusApiService.cs b/API/Services/Plus/KavitaPlusApiService.cs new file mode 100644 index 000000000..cdf9471f8 --- /dev/null +++ b/API/Services/Plus/KavitaPlusApiService.cs @@ -0,0 +1,75 @@ +#nullable enable +using System.Threading.Tasks; +using API.DTOs.Scrobbling; +using API.Extensions; +using Flurl.Http; +using Kavita.Common; +using Microsoft.Extensions.Logging; + +namespace API.Services.Plus; + +/// +/// All Http requests to K+ should be contained in this service, the service will not handle any errors. +/// This is expected from the caller. +/// +public interface IKavitaPlusApiService +{ + Task HasTokenExpired(string license, string token, ScrobbleProvider provider); + Task GetRateLimit(string license, string token); + Task PostScrobbleUpdate(ScrobbleDto data, string license); +} + +public class KavitaPlusApiService(ILogger logger): IKavitaPlusApiService +{ + private const string ScrobblingPath = "/api/scrobbling/"; + + public async Task HasTokenExpired(string license, string token, ScrobbleProvider provider) + { + var res = await Get(ScrobblingPath + "valid-key?provider=" + provider + "&key=" + token, license, token); + var str = await res.GetStringAsync(); + return bool.Parse(str); + } + + public async Task GetRateLimit(string license, string token) + { + var res = await Get(ScrobblingPath + "rate-limit?accessToken=" + token, license, token); + var str = await res.GetStringAsync(); + return int.Parse(str); + } + + public async Task PostScrobbleUpdate(ScrobbleDto data, string license) + { + return await PostAndReceive(ScrobblingPath + "update", data, license); + } + + /// + /// Send a GET request to K+ + /// + /// only path of the uri, the host is added + /// + /// + /// + private static async Task Get(string url, string license, string? aniListToken = null) + { + return await (Configuration.KavitaPlusApiUrl + url) + .WithKavitaPlusHeaders(license, aniListToken) + .GetAsync(); + } + + /// + /// Send a POST request to K+ + /// + /// only path of the uri, the host is added + /// + /// + /// + /// Return type + /// + private static async Task PostAndReceive(string url, object body, string license, string? aniListToken = null) + { + return await (Configuration.KavitaPlusApiUrl + url) + .WithKavitaPlusHeaders(license, aniListToken) + .PostJsonAsync(body) + .ReceiveJson(); + } +} diff --git a/API/Services/Plus/ScrobblingService.cs b/API/Services/Plus/ScrobblingService.cs index 85814dcd9..f9c3fdb09 100644 --- a/API/Services/Plus/ScrobblingService.cs +++ b/API/Services/Plus/ScrobblingService.cs @@ -38,31 +38,124 @@ public enum ScrobbleProvider Kavita = 0, AniList = 1, Mal = 2, - [Obsolete] + [Obsolete("No longer supported")] GoogleBooks = 3, Cbr = 4 } public interface IScrobblingService { + /// + /// An automated job that will run against all user's tokens and validate if they are still active + /// + /// This service can validate without license check as the task which calls will be guarded + /// Task CheckExternalAccessTokens(); + + /// + /// Checks if the token has expired with , if it has double checks with K+, + /// otherwise return false. + /// + /// + /// + /// + /// Returns true if there is no license present Task HasTokenExpired(int userId, ScrobbleProvider provider); + /// + /// Create, or update a non-processed, event, for the given series + /// + /// + /// + /// + /// Task ScrobbleRatingUpdate(int userId, int seriesId, float rating); + /// + /// NOP, until hardcover support has been worked out + /// + /// + /// + /// + /// + /// Task ScrobbleReviewUpdate(int userId, int seriesId, string? reviewTitle, string reviewBody); + /// + /// Create, or update a non-processed, event, for the given series + /// + /// + /// + /// Task ScrobbleReadingUpdate(int userId, int seriesId); + /// + /// Creates an or for + /// the given series + /// + /// + /// + /// + /// + /// Only the result of both WantToRead types is send to K+ Task ScrobbleWantToReadUpdate(int userId, int seriesId, bool onWantToRead); + /// + /// Removed all processed events that are at least 7 days old + /// + /// [DisableConcurrentExecution(60 * 60 * 60)] [AutomaticRetry(Attempts = 3, OnAttemptsExceeded = AttemptsExceededAction.Delete)] public Task ClearProcessedEvents(); + + /// + /// Makes K+ requests for all non-processed events until rate limits are reached + /// + /// [DisableConcurrentExecution(60 * 60 * 60)] [AutomaticRetry(Attempts = 3, OnAttemptsExceeded = AttemptsExceededAction.Delete)] Task ProcessUpdatesSinceLastSync(); + Task CreateEventsFromExistingHistory(int userId = 0); Task CreateEventsFromExistingHistoryForSeries(int seriesId); Task ClearEventsForSeries(int userId, int seriesId); } +/// +/// Context used when syncing scrobble events. Do NOT reuse between syncs +/// +public class ScrobbleSyncContext +{ + public required List ReadEvents {get; init;} + public required List RatingEvents {get; init;} + /// Do not use this as events to send to K+, use + public required List AddToWantToRead {get; init;} + /// Do not use this as events to send to K+, use + public required List RemoveWantToRead {get; init;} + /// + /// Final events list if all AddTo- and RemoveWantToRead would be processed sequentially + /// + public required List Decisions {get; init;} + /// + /// K+ license + /// + public required string License { get; init; } + /// + /// Maps userId to left over request amount + /// + public required Dictionary RateLimits { get; init; } + + /// + /// All users being scrobbled for + /// + public List Users { get; set; } = []; + /// + /// Amount of already processed events + /// + public int ProgressCounter { get; set; } + + /// + /// Sum of all events to process + /// + public int TotalCount => ReadEvents.Count + RatingEvents.Count + AddToWantToRead.Count + RemoveWantToRead.Count; +} + public class ScrobblingService : IScrobblingService { private readonly IUnitOfWork _unitOfWork; @@ -71,6 +164,7 @@ public class ScrobblingService : IScrobblingService private readonly ILicenseService _licenseService; private readonly ILocalizationService _localizationService; private readonly IEmailService _emailService; + private readonly IKavitaPlusApiService _kavitaPlusApiService; public const string AniListWeblinkWebsite = "https://anilist.co/manga/"; public const string MalWeblinkWebsite = "https://myanimelist.net/manga/"; @@ -80,7 +174,7 @@ public class ScrobblingService : IScrobblingService public const string AniListCharacterWebsite = "https://anilist.co/character/"; - private static readonly Dictionary WeblinkExtractionMap = new Dictionary() + private static readonly Dictionary WeblinkExtractionMap = new() { {AniListWeblinkWebsite, 0}, {MalWeblinkWebsite, 0}, @@ -104,10 +198,14 @@ public class ScrobblingService : IScrobblingService private const string UnknownSeriesErrorMessage = "Series cannot be matched for Scrobbling"; private const string AccessTokenErrorMessage = "Access Token needs to be rotated to continue scrobbling"; + private const string InvalidKPlusLicenseErrorMessage = "Kavita+ subscription no longer active"; + private const string ReviewFailedErrorMessage = "Review was unable to be saved due to upstream requirements"; + private const string BadPayLoadErrorMessage = "Bad payload from Scrobble Provider"; public ScrobblingService(IUnitOfWork unitOfWork, IEventHub eventHub, ILogger logger, - ILicenseService licenseService, ILocalizationService localizationService, IEmailService emailService) + ILicenseService licenseService, ILocalizationService localizationService, IEmailService emailService, + IKavitaPlusApiService kavitaPlusApiService) { _unitOfWork = unitOfWork; _eventHub = eventHub; @@ -115,10 +213,12 @@ public class ScrobblingService : IScrobblingService _licenseService = licenseService; _localizationService = localizationService; _emailService = emailService; + _kavitaPlusApiService = kavitaPlusApiService; FlurlConfiguration.ConfigureClientForUrl(Configuration.KavitaPlusApiUrl); } + #region Access token checks /// /// An automated job that will run against all user's tokens and validate if they are still active @@ -196,7 +296,6 @@ public class ScrobblingService : IScrobblingService } - public async Task HasTokenExpired(int userId, ScrobbleProvider provider) { var token = await GetTokenForProvider(userId, provider); @@ -214,19 +313,14 @@ public class ScrobblingService : IScrobblingService private async Task HasTokenExpired(string token, ScrobbleProvider provider) { - if (string.IsNullOrEmpty(token) || - !TokenService.HasTokenExpired(token)) return false; + if (string.IsNullOrEmpty(token) || !TokenService.HasTokenExpired(token)) return false; var license = await _unitOfWork.SettingsRepository.GetSettingAsync(ServerSettingKey.LicenseKey); if (string.IsNullOrEmpty(license.Value)) return true; try { - var response = await (Configuration.KavitaPlusApiUrl + "/api/scrobbling/valid-key?provider=" + provider + "&key=" + token) - .WithKavitaPlusHeaders(license.Value, token) - .GetStringAsync(); - - return bool.Parse(response); + return await _kavitaPlusApiService.HasTokenExpired(license.Value, token, provider); } catch (HttpRequestException e) { @@ -252,59 +346,14 @@ public class ScrobblingService : IScrobblingService } ?? string.Empty; } - public async Task ScrobbleReviewUpdate(int userId, int seriesId, string? reviewTitle, string reviewBody) + #endregion + + #region Scrobble ingest + + public Task ScrobbleReviewUpdate(int userId, int seriesId, string? reviewTitle, string reviewBody) { // Currently disabled until at least hardcover is implemented - return; - if (!await _licenseService.HasActiveLicense()) return; - - var series = await _unitOfWork.SeriesRepository.GetSeriesByIdAsync(seriesId, SeriesIncludes.Metadata | SeriesIncludes.Library); - if (series == null) throw new KavitaException(await _localizationService.Translate(userId, "series-doesnt-exist")); - - _logger.LogInformation("Processing Scrobbling review event for {AppUserId} on {SeriesName}", userId, series.Name); - if (await CheckIfCannotScrobble(userId, seriesId, series)) return; - - if (IsAniListReviewValid(reviewTitle, reviewBody)) - { - _logger.LogDebug( - "Rejecting Scrobble event for {Series}. Review is not long enough to meet requirements", series.Name); - return; - } - - var existingEvt = await _unitOfWork.ScrobbleRepository.GetEvent(userId, series.Id, - ScrobbleEventType.Review); - if (existingEvt is {IsProcessed: false}) - { - _logger.LogDebug("Overriding Review scrobble event for {Series}", existingEvt.Series.Name); - existingEvt.ReviewBody = reviewBody; - existingEvt.ReviewTitle = reviewTitle; - _unitOfWork.ScrobbleRepository.Update(existingEvt); - await _unitOfWork.CommitAsync(); - return; - } - - var evt = new ScrobbleEvent() - { - SeriesId = series.Id, - LibraryId = series.LibraryId, - ScrobbleEventType = ScrobbleEventType.Review, - AniListId = ExtractId(series.Metadata.WebLinks, AniListWeblinkWebsite), - MalId = GetMalId(series), - AppUserId = userId, - Format = series.Library.Type.ConvertToPlusMediaFormat(series.Format), - ReviewBody = reviewBody, - ReviewTitle = reviewTitle - }; - _unitOfWork.ScrobbleRepository.Attach(evt); - await _unitOfWork.CommitAsync(); - _logger.LogDebug("Added Scrobbling Review update on {SeriesName} with Userid {AppUserId} ", series.Name, userId); - } - - private static bool IsAniListReviewValid(string reviewTitle, string reviewBody) - { - return string.IsNullOrEmpty(reviewTitle) || string.IsNullOrEmpty(reviewBody) || (reviewTitle.Length < 2200 || - reviewTitle.Length > 120 || - reviewTitle.Length < 20); + return Task.CompletedTask; } public async Task ScrobbleRatingUpdate(int userId, int seriesId, float rating) @@ -321,7 +370,7 @@ public class ScrobblingService : IScrobblingService if (await CheckIfCannotScrobble(userId, seriesId, series)) return; var existingEvt = await _unitOfWork.ScrobbleRepository.GetEvent(userId, series.Id, - ScrobbleEventType.ScoreUpdated); + ScrobbleEventType.ScoreUpdated, true); if (existingEvt is {IsProcessed: false}) { // We need to just update Volume/Chapter number @@ -349,18 +398,6 @@ public class ScrobblingService : IScrobblingService _logger.LogDebug("Added Scrobbling Rating update on {SeriesName} with Userid {AppUserId}", series.Name, userId); } - public static long? GetMalId(Series series) - { - var malId = ExtractId(series.Metadata.WebLinks, MalWeblinkWebsite); - return malId ?? series.ExternalSeriesMetadata?.MalId; - } - - public static int? GetAniListId(Series seriesWithExternalMetadata) - { - var aniListId = ExtractId(seriesWithExternalMetadata.Metadata.WebLinks, AniListWeblinkWebsite); - return aniListId ?? seriesWithExternalMetadata.ExternalSeriesMetadata?.AniListId; - } - public async Task ScrobbleReadingUpdate(int userId, int seriesId) { if (!await _licenseService.HasActiveLicense()) return; @@ -374,28 +411,49 @@ public class ScrobblingService : IScrobblingService _logger.LogInformation("Processing Scrobbling reading event for {AppUserId} on {SeriesName}", userId, series.Name); if (await CheckIfCannotScrobble(userId, seriesId, series)) return; + var isAnyProgressOnSeries = await _unitOfWork.AppUserProgressRepository.HasAnyProgressOnSeriesAsync(seriesId, userId); + + var volumeNumber = (int) await _unitOfWork.AppUserProgressRepository.GetHighestFullyReadVolumeForSeries(seriesId, userId); + var chapterNumber = await _unitOfWork.AppUserProgressRepository.GetHighestFullyReadChapterForSeries(seriesId, userId); + + // Check if there is an existing not yet processed event, if so update it var existingEvt = await _unitOfWork.ScrobbleRepository.GetEvent(userId, series.Id, - ScrobbleEventType.ChapterRead); + ScrobbleEventType.ChapterRead, true); + if (existingEvt is {IsProcessed: false}) { + if (!isAnyProgressOnSeries) + { + _unitOfWork.ScrobbleRepository.Remove(existingEvt); + await _unitOfWork.CommitAsync(); + _logger.LogDebug("Removed scrobble event for {Series} as there is no reading progress", series.Name); + return; + } + // We need to just update Volume/Chapter number var prevChapter = $"{existingEvt.ChapterNumber}"; var prevVol = $"{existingEvt.VolumeNumber}"; - existingEvt.VolumeNumber = - (int) await _unitOfWork.AppUserProgressRepository.GetHighestFullyReadVolumeForSeries(seriesId, userId); - existingEvt.ChapterNumber = - await _unitOfWork.AppUserProgressRepository.GetHighestFullyReadChapterForSeries(seriesId, userId); + existingEvt.VolumeNumber = volumeNumber; + existingEvt.ChapterNumber = chapterNumber; + _unitOfWork.ScrobbleRepository.Update(existingEvt); await _unitOfWork.CommitAsync(); + _logger.LogDebug("Overriding scrobble event for {Series} from vol {PrevVol} ch {PrevChap} -> vol {UpdatedVol} ch {UpdatedChap}", existingEvt.Series.Name, prevVol, prevChapter, existingEvt.VolumeNumber, existingEvt.ChapterNumber); return; } + if (!isAnyProgressOnSeries) + { + // Do not create a new scrobble event if there is no progress + return; + } + try { - var evt = new ScrobbleEvent() + var evt = new ScrobbleEvent { SeriesId = series.Id, LibraryId = series.LibraryId, @@ -403,10 +461,8 @@ public class ScrobblingService : IScrobblingService AniListId = GetAniListId(series), MalId = GetMalId(series), AppUserId = userId, - VolumeNumber = - (int) await _unitOfWork.AppUserProgressRepository.GetHighestFullyReadVolumeForSeries(seriesId, userId), - ChapterNumber = - await _unitOfWork.AppUserProgressRepository.GetHighestFullyReadChapterForSeries(seriesId, userId), + VolumeNumber = volumeNumber, + ChapterNumber = chapterNumber, Format = series.Library.Type.ConvertToPlusMediaFormat(series.Format), }; @@ -431,7 +487,9 @@ public class ScrobblingService : IScrobblingService if (!await _licenseService.HasActiveLicense()) return; var series = await _unitOfWork.SeriesRepository.GetSeriesByIdAsync(seriesId, SeriesIncludes.Metadata | SeriesIncludes.Library | SeriesIncludes.ExternalMetadata); - if (series == null || !series.Library.AllowScrobbling) throw new KavitaException(await _localizationService.Translate(userId, "series-doesnt-exist")); + if (series == null) throw new KavitaException(await _localizationService.Translate(userId, "series-doesnt-exist")); + + if (!series.Library.AllowScrobbling) return; var user = await _unitOfWork.UserRepository.GetUserByIdAsync(userId, AppUserIncludes.UserPreferences); if (user == null || !user.UserPreferences.AniListScrobblingEnabled) return; @@ -444,10 +502,7 @@ public class ScrobblingService : IScrobblingService .Where(e => new[] { ScrobbleEventType.AddWantToRead, ScrobbleEventType.RemoveWantToRead }.Contains(e.ScrobbleEventType)); // Remove all existing want-to-read events for this series/user - foreach (var existingEvent in existingEvents) - { - _unitOfWork.ScrobbleRepository.Remove(existingEvent); - } + _unitOfWork.ScrobbleRepository.Remove(existingEvents); // Create the new event var evt = new ScrobbleEvent() @@ -466,704 +521,27 @@ public class ScrobblingService : IScrobblingService _logger.LogDebug("Added Scrobbling WantToRead update on {SeriesName} with Userid {AppUserId} ", series.Name, userId); } - private async Task CheckIfCannotScrobble(int userId, int seriesId, Series series) + #endregion + + #region Scrobble provider methods + + private static bool IsAniListReviewValid(string reviewTitle, string reviewBody) { - if (series.DontMatch) return true; - if (await _unitOfWork.UserRepository.HasHoldOnSeries(userId, seriesId)) - { - _logger.LogInformation("Series {SeriesName} is on AppUserId {AppUserId}'s hold list. Not scrobbling", series.Name, - userId); - return true; - } - - var library = await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(series.LibraryId); - if (library is not {AllowScrobbling: true}) return true; - if (!ExternalMetadataService.IsPlusEligible(library.Type)) return true; - - return false; + return string.IsNullOrEmpty(reviewTitle) || string.IsNullOrEmpty(reviewBody) || (reviewTitle.Length < 2200 || + reviewTitle.Length > 120 || + reviewTitle.Length < 20); } - private async Task GetRateLimit(string license, string aniListToken) + public static long? GetMalId(Series series) { - if (string.IsNullOrWhiteSpace(aniListToken)) return 0; - try - { - var response = await (Configuration.KavitaPlusApiUrl + "/api/scrobbling/rate-limit?accessToken=" + aniListToken) - .WithKavitaPlusHeaders(license, aniListToken) - .GetStringAsync(); - - return int.Parse(response); - } - catch (Exception e) - { - _logger.LogError(e, "An error happened trying to get rate limit from Kavita+ API"); - } - - return 0; + var malId = ExtractId(series.Metadata.WebLinks, MalWeblinkWebsite); + return malId ?? series.ExternalSeriesMetadata?.MalId; } - private async Task PostScrobbleUpdate(ScrobbleDto data, string license, ScrobbleEvent evt) + public static int? GetAniListId(Series seriesWithExternalMetadata) { - try - { - var response = await (Configuration.KavitaPlusApiUrl + "/api/scrobbling/update") - .WithKavitaPlusHeaders(license) - .PostJsonAsync(data) - .ReceiveJson(); - - if (!response.Successful) - { - // Might want to log this under ScrobbleError - if (response.ErrorMessage != null && response.ErrorMessage.Contains("Too Many Requests")) - { - _logger.LogInformation("Hit Too many requests, sleeping to regain requests and retrying"); - await Task.Delay(TimeSpan.FromMinutes(10)); - return await PostScrobbleUpdate(data, license, evt); - } - if (response.ErrorMessage != null && response.ErrorMessage.Contains("Unauthorized")) - { - _logger.LogCritical("Kavita+ responded with Unauthorized. Please check your subscription"); - await _licenseService.HasActiveLicense(true); - evt.IsErrored = true; - evt.ErrorDetails = "Kavita+ subscription no longer active"; - throw new KavitaException("Kavita+ responded with Unauthorized. Please check your subscription"); - } - if (response.ErrorMessage != null && response.ErrorMessage.Contains("Access token is invalid")) - { - evt.IsErrored = true; - evt.ErrorDetails = AccessTokenErrorMessage; - throw new KavitaException("Access token is invalid"); - } - if (response.ErrorMessage != null && response.ErrorMessage.Contains("Unknown Series")) - { - // Log the Series name and Id in ScrobbleErrors - _logger.LogInformation("Kavita+ was unable to match the series: {SeriesName}", evt.Series.Name); - if (!await _unitOfWork.ScrobbleRepository.HasErrorForSeries(evt.SeriesId)) - { - // Create a new ExternalMetadata entry to indicate that this is not matchable - var series = await _unitOfWork.SeriesRepository.GetSeriesByIdAsync(evt.SeriesId, SeriesIncludes.ExternalMetadata); - if (series == null) return 0; - - series.ExternalSeriesMetadata ??= new ExternalSeriesMetadata() {SeriesId = evt.SeriesId}; - series.IsBlacklisted = true; - _unitOfWork.SeriesRepository.Update(series); - - _unitOfWork.ScrobbleRepository.Attach(new ScrobbleError() - { - Comment = UnknownSeriesErrorMessage, - Details = data.SeriesName, - LibraryId = evt.LibraryId, - SeriesId = evt.SeriesId - }); - - } - - evt.IsErrored = true; - evt.ErrorDetails = UnknownSeriesErrorMessage; - } else if (response.ErrorMessage != null && response.ErrorMessage.StartsWith("Review")) - { - // Log the Series name and Id in ScrobbleErrors - _logger.LogInformation("Kavita+ was unable to save the review"); - if (!await _unitOfWork.ScrobbleRepository.HasErrorForSeries(evt.SeriesId)) - { - _unitOfWork.ScrobbleRepository.Attach(new ScrobbleError() - { - Comment = response.ErrorMessage, - Details = data.SeriesName, - LibraryId = evt.LibraryId, - SeriesId = evt.SeriesId - }); - } - evt.IsErrored = true; - evt.ErrorDetails = "Review was unable to be saved due to upstream requirements"; - } - } - - return response.RateLeft; - } - catch (FlurlHttpException ex) - { - var errorMessage = await ex.GetResponseStringAsync(); - // Trim quotes if the response is a JSON string - errorMessage = errorMessage.Trim('"'); - - if (errorMessage.Contains("Too Many Requests")) - { - _logger.LogInformation("Hit Too many requests, sleeping to regain requests and retrying"); - await Task.Delay(TimeSpan.FromMinutes(10)); - return await PostScrobbleUpdate(data, license, evt); - } - - _logger.LogError(ex, "Scrobbling to Kavita+ API failed due to error: {ErrorMessage}", ex.Message); - if (ex.Message.Contains("Call failed with status code 500 (Internal Server Error)")) - { - if (!await _unitOfWork.ScrobbleRepository.HasErrorForSeries(evt.SeriesId)) - { - _unitOfWork.ScrobbleRepository.Attach(new ScrobbleError() - { - Comment = UnknownSeriesErrorMessage, - Details = data.SeriesName, - LibraryId = evt.LibraryId, - SeriesId = evt.SeriesId - }); - } - evt.IsErrored = true; - evt.ErrorDetails = "Bad payload from Scrobble Provider"; - throw new KavitaException("Bad payload from Scrobble Provider"); - } - throw; - } - } - - /// - /// This will backfill events from existing progress history, ratings, and want to read for users that have a valid license - /// - /// Defaults to 0 meaning all users. Allows a userId to be set if a scrobble key is added to a user - public async Task CreateEventsFromExistingHistory(int userId = 0) - { - if (!await _licenseService.HasActiveLicense()) return; - - if (userId != 0) - { - var user = await _unitOfWork.UserRepository.GetUserByIdAsync(userId); - if (user == null || string.IsNullOrEmpty(user.AniListAccessToken)) return; - if (user.HasRunScrobbleEventGeneration) - { - _logger.LogWarning("User {UserName} has already run scrobble event generation, Kavita will not generate more events", user.UserName); - return; - } - } - - - - var libAllowsScrobbling = (await _unitOfWork.LibraryRepository.GetLibrariesAsync()) - .ToDictionary(lib => lib.Id, lib => lib.AllowScrobbling); - - var userIds = (await _unitOfWork.UserRepository.GetAllUsersAsync()) - .Where(l => userId == 0 || userId == l.Id) - .Select(u => u.Id); - - foreach (var uId in userIds) - { - var wantToRead = await _unitOfWork.SeriesRepository.GetWantToReadForUserAsync(uId); - foreach (var wtr in wantToRead) - { - if (!libAllowsScrobbling[wtr.LibraryId]) continue; - await ScrobbleWantToReadUpdate(uId, wtr.Id, true); - } - - var ratings = await _unitOfWork.UserRepository.GetSeriesWithRatings(uId); - foreach (var rating in ratings) - { - if (!libAllowsScrobbling[rating.Series.LibraryId]) continue; - await ScrobbleRatingUpdate(uId, rating.SeriesId, rating.Rating); - } - - var seriesWithProgress = await _unitOfWork.SeriesRepository.GetSeriesDtoForLibraryIdAsync(0, uId, - new UserParams(), new FilterDto() - { - ReadStatus = new ReadStatus() - { - Read = true, - InProgress = true, - NotRead = false - }, - Libraries = libAllowsScrobbling.Keys.Where(k => libAllowsScrobbling[k]).ToList() - }); - - foreach (var series in seriesWithProgress) - { - if (!libAllowsScrobbling[series.LibraryId]) continue; - if (series.PagesRead <= 0) continue; // Since we only scrobble when things are higher, we can - await ScrobbleReadingUpdate(uId, series.Id); - } - - var user = await _unitOfWork.UserRepository.GetUserByIdAsync(uId); - if (user != null) - { - user.HasRunScrobbleEventGeneration = true; - user.ScrobbleEventGenerationRan = DateTime.UtcNow; - await _unitOfWork.CommitAsync(); - } - } - } - - public async Task CreateEventsFromExistingHistoryForSeries(int seriesId) - { - if (!await _licenseService.HasActiveLicense()) return; - - var series = await _unitOfWork.SeriesRepository.GetSeriesByIdAsync(seriesId, SeriesIncludes.Library); - if (series == null || !series.Library.AllowScrobbling) return; - - _logger.LogInformation("Creating Scrobbling events for Series {SeriesName}", series.Name); - - var userIds = (await _unitOfWork.UserRepository.GetAllUsersAsync()) - .Select(u => u.Id); - - foreach (var uId in userIds) - { - // Handle "Want to Read" updates specific to the series - var wantToRead = await _unitOfWork.SeriesRepository.GetWantToReadForUserAsync(uId); - foreach (var wtr in wantToRead.Where(wtr => wtr.Id == seriesId)) - { - await ScrobbleWantToReadUpdate(uId, wtr.Id, true); - } - - // Handle ratings specific to the series - var ratings = await _unitOfWork.UserRepository.GetSeriesWithRatings(uId); - foreach (var rating in ratings.Where(rating => rating.SeriesId == seriesId)) - { - await ScrobbleRatingUpdate(uId, rating.SeriesId, rating.Rating); - } - - // Handle progress updates for the specific series - var seriesProgress = await _unitOfWork.SeriesRepository.GetSeriesDtoForLibraryIdAsync( - series.LibraryId, - uId, - new UserParams(), - new FilterDto - { - ReadStatus = new ReadStatus - { - Read = true, - InProgress = true, - NotRead = false - }, - Libraries = new List { series.LibraryId }, - SeriesNameQuery = series.Name - }); - - foreach (var progress in seriesProgress.Where(progress => progress.Id == seriesId)) - { - if (progress.PagesRead > 0) - { - await ScrobbleReadingUpdate(uId, progress.Id); - } - } - } - } - - /// - /// Removes all events (active) that are tied to a now-on hold series - /// - /// - /// - public async Task ClearEventsForSeries(int userId, int seriesId) - { - _logger.LogInformation("Clearing Pre-existing Scrobble events for Series {SeriesId} by User {AppUserId} as Series is now on hold list", seriesId, userId); - var events = await _unitOfWork.ScrobbleRepository.GetUserEventsForSeries(userId, seriesId); - foreach (var scrobble in events) - { - _unitOfWork.ScrobbleRepository.Remove(scrobble); - } - - await _unitOfWork.CommitAsync(); - } - - /// - /// Removes all events that have been processed that are 7 days old - /// - [DisableConcurrentExecution(60 * 60 * 60)] - [AutomaticRetry(Attempts = 3, OnAttemptsExceeded = AttemptsExceededAction.Delete)] - public async Task ClearProcessedEvents() - { - const int daysAgo = 7; - var events = await _unitOfWork.ScrobbleRepository.GetProcessedEvents(daysAgo); - _unitOfWork.ScrobbleRepository.Remove(events); - _logger.LogInformation("Removing {Count} scrobble events that have been processed {DaysAgo}+ days ago", events.Count, daysAgo); - await _unitOfWork.CommitAsync(); - } - - /// - /// This is a task that is ran on a fixed schedule (every few hours or every day) that clears out the scrobble event table - /// and offloads the data to the API server which performs the syncing to the providers. - /// - [DisableConcurrentExecution(60 * 60 * 60)] - [AutomaticRetry(Attempts = 3, OnAttemptsExceeded = AttemptsExceededAction.Delete)] - public async Task ProcessUpdatesSinceLastSync() - { - // Check how many scrobble events we have available then only do those. - var userRateLimits = new Dictionary(); - - var progressCounter = 0; - - var librariesWithScrobbling = (await _unitOfWork.LibraryRepository.GetLibrariesAsync()) - .AsEnumerable() - .Where(l => l.AllowScrobbling) - .Select(l => l.Id) - .ToImmutableHashSet(); - - var errors = (await _unitOfWork.ScrobbleRepository.GetScrobbleErrors()) - .Where(e => e.Comment == "Unknown Series" || e.Comment == UnknownSeriesErrorMessage || e.Comment == AccessTokenErrorMessage) - .Select(e => e.SeriesId) - .ToList(); - - var readEvents = (await _unitOfWork.ScrobbleRepository.GetByEvent(ScrobbleEventType.ChapterRead)) - .Where(e => librariesWithScrobbling.Contains(e.LibraryId)) - .Where(e => !errors.Contains(e.SeriesId)) - .ToList(); - var addToWantToRead = (await _unitOfWork.ScrobbleRepository.GetByEvent(ScrobbleEventType.AddWantToRead)) - .Where(e => librariesWithScrobbling.Contains(e.LibraryId)) - .Where(e => !errors.Contains(e.SeriesId)) - .ToList(); - var removeWantToRead = (await _unitOfWork.ScrobbleRepository.GetByEvent(ScrobbleEventType.RemoveWantToRead)) - .Where(e => librariesWithScrobbling.Contains(e.LibraryId)) - .Where(e => !errors.Contains(e.SeriesId)) - .ToList(); - var ratingEvents = (await _unitOfWork.ScrobbleRepository.GetByEvent(ScrobbleEventType.ScoreUpdated)) - .Where(e => librariesWithScrobbling.Contains(e.LibraryId)) - .Where(e => !errors.Contains(e.SeriesId)) - .ToList(); - - var decisions = CalculateNetWantToReadDecisions(addToWantToRead, removeWantToRead); - - // Clear any events that are already on error table - var erroredEvents = await _unitOfWork.ScrobbleRepository.GetAllEventsWithSeriesIds(errors); - if (erroredEvents.Count > 0) - { - _unitOfWork.ScrobbleRepository.Remove(erroredEvents); - await _unitOfWork.CommitAsync(); - } - - var totalEvents = readEvents.Count + decisions.Count + ratingEvents.Count; - if (totalEvents == 0) return; - - // Get all the applicable users to scrobble and set their rate limits - var license = await _unitOfWork.SettingsRepository.GetSettingAsync(ServerSettingKey.LicenseKey); - var usersToScrobble = await PrepareUsersToScrobble(readEvents, addToWantToRead, removeWantToRead, ratingEvents, userRateLimits, license); - - - _logger.LogInformation("Scrobble Processing Details:" + - "\n Read Events: {ReadEventsCount}" + - "\n Want to Read Events: {WantToReadEventsCount}" + - "\n Rating Events: {RatingEventsCount}" + - "\n Users to Scrobble: {UsersToScrobbleCount}" + - "\n Total Events to Process: {TotalEvents}", - readEvents.Count, - decisions.Count, - ratingEvents.Count, - usersToScrobble.Count, - totalEvents); - - try - { - progressCounter = await ProcessReadEvents(readEvents, userRateLimits, usersToScrobble, totalEvents, progressCounter); - - progressCounter = await ProcessRatingEvents(ratingEvents, userRateLimits, usersToScrobble, totalEvents, progressCounter); - - progressCounter = await ProcessWantToReadRatingEvents(decisions, userRateLimits, usersToScrobble, totalEvents, progressCounter); - } - catch (FlurlHttpException ex) - { - _logger.LogError(ex, "Kavita+ API or a Scrobble service may be experiencing an outage. Stopping sending data"); - return; - } - - - await SaveToDb(progressCounter, true); - _logger.LogInformation("Scrobbling Events is complete"); - - // Cleanup any events that are due to bugs or legacy - try - { - var eventsWithoutAnilistToken = (await _unitOfWork.ScrobbleRepository.GetEvents()) - .Where(e => !e.IsProcessed && !e.IsErrored) - .Where(e => string.IsNullOrEmpty(e.AppUser.AniListAccessToken)); - - _unitOfWork.ScrobbleRepository.Remove(eventsWithoutAnilistToken); - await _unitOfWork.CommitAsync(); - } - catch (Exception ex) - { - _logger.LogError(ex, "There was an exception when trying to delete old scrobble events when the user has no active token"); - } - } - - /// - /// Calculates the net want-to-read decisions by considering all events. - /// Returns events that represent the final state for each user/series pair. - /// - /// List of events for adding to want-to-read - /// List of events for removing from want-to-read - /// List of events that represent the final state (add or remove) - private static List CalculateNetWantToReadDecisions(List addEvents, List removeEvents) - { - // Create a dictionary to track the latest event for each user/series combination - var latestEvents = new Dictionary<(int SeriesId, int AppUserId), ScrobbleEvent>(); - - // Process all add events - foreach (var addEvent in addEvents) - { - var key = (addEvent.SeriesId, addEvent.AppUserId); - - if (latestEvents.TryGetValue(key, out var value) && addEvent.CreatedUtc <= value.CreatedUtc) continue; - - value = addEvent; - latestEvents[key] = value; - } - - // Process all remove events - foreach (var removeEvent in removeEvents) - { - var key = (removeEvent.SeriesId, removeEvent.AppUserId); - - if (latestEvents.TryGetValue(key, out var value) && removeEvent.CreatedUtc <= value.CreatedUtc) continue; - - value = removeEvent; - latestEvents[key] = value; - } - - // Return all events that represent the final state - return latestEvents.Values.ToList(); - } - - private async Task ProcessWantToReadRatingEvents(List decisions, Dictionary userRateLimits, List usersToScrobble, int totalEvents, int progressCounter) - { - progressCounter = await ProcessEvents(decisions, userRateLimits, usersToScrobble.Count, progressCounter, - totalEvents, evt => Task.FromResult(new ScrobbleDto() - { - Format = evt.Format, - AniListId = evt.AniListId, - MALId = (int?) evt.MalId, - ScrobbleEventType = evt.ScrobbleEventType, - ChapterNumber = evt.ChapterNumber, - VolumeNumber = (int?) evt.VolumeNumber, - AniListToken = evt.AppUser.AniListAccessToken, - SeriesName = evt.Series.Name, - LocalizedSeriesName = evt.Series.LocalizedName, - Year = evt.Series.Metadata.ReleaseYear - })); - - // After decisions, we need to mark all the want to read and remove from want to read as completed - if (decisions.Any(d => d.IsProcessed)) - { - foreach (var scrobbleEvent in decisions.Where(d => d.IsProcessed)) - { - scrobbleEvent.IsProcessed = true; - scrobbleEvent.ProcessDateUtc = DateTime.UtcNow; - _unitOfWork.ScrobbleRepository.Update(scrobbleEvent); - } - await _unitOfWork.CommitAsync(); - } - - return progressCounter; - } - - private async Task ProcessRatingEvents(List ratingEvents, Dictionary userRateLimits, List usersToScrobble, - int totalEvents, int progressCounter) - { - return await ProcessEvents(ratingEvents, userRateLimits, usersToScrobble.Count, progressCounter, - totalEvents, evt => Task.FromResult(new ScrobbleDto() - { - Format = evt.Format, - AniListId = evt.AniListId, - MALId = (int?) evt.MalId, - ScrobbleEventType = evt.ScrobbleEventType, - AniListToken = evt.AppUser.AniListAccessToken, - SeriesName = evt.Series.Name, - LocalizedSeriesName = evt.Series.LocalizedName, - Rating = evt.Rating, - Year = evt.Series.Metadata.ReleaseYear - })); - } - - - private async Task ProcessReadEvents(List readEvents, Dictionary userRateLimits, List usersToScrobble, int totalEvents, - int progressCounter) - { - // Recalculate the highest volume/chapter - foreach (var readEvt in readEvents) - { - // Note: this causes skewing in the scrobble history because it makes it look like there are duplicate events - readEvt.VolumeNumber = - (int) await _unitOfWork.AppUserProgressRepository.GetHighestFullyReadVolumeForSeries(readEvt.SeriesId, - readEvt.AppUser.Id); - readEvt.ChapterNumber = - await _unitOfWork.AppUserProgressRepository.GetHighestFullyReadChapterForSeries(readEvt.SeriesId, - readEvt.AppUser.Id); - _unitOfWork.ScrobbleRepository.Update(readEvt); - } - - return await ProcessEvents(readEvents, userRateLimits, usersToScrobble.Count, progressCounter, totalEvents, - async evt => new ScrobbleDto() - { - Format = evt.Format, - AniListId = evt.AniListId, - MALId = (int?) evt.MalId, - ScrobbleEventType = evt.ScrobbleEventType, - ChapterNumber = evt.ChapterNumber, - VolumeNumber = (int?) evt.VolumeNumber, - AniListToken = evt.AppUser.AniListAccessToken!, - SeriesName = evt.Series.Name, - LocalizedSeriesName = evt.Series.LocalizedName, - ScrobbleDateUtc = evt.LastModifiedUtc, - Year = evt.Series.Metadata.ReleaseYear, - StartedReadingDateUtc = await _unitOfWork.AppUserProgressRepository.GetFirstProgressForSeries(evt.SeriesId, evt.AppUser.Id), - LatestReadingDateUtc = await _unitOfWork.AppUserProgressRepository.GetLatestProgressForSeries(evt.SeriesId, evt.AppUser.Id), - }); - } - - - private async Task> PrepareUsersToScrobble(List readEvents, List addToWantToRead, List removeWantToRead, List ratingEvents, - Dictionary userRateLimits, ServerSetting license) - { - // For all userIds, ensure that we can connect and have access - var usersToScrobble = readEvents.Select(r => r.AppUser) - .Concat(addToWantToRead.Select(r => r.AppUser)) - .Concat(removeWantToRead.Select(r => r.AppUser)) - .Concat(ratingEvents.Select(r => r.AppUser)) - .Where(user => !string.IsNullOrEmpty(user.AniListAccessToken)) - .Where(user => user.UserPreferences.AniListScrobblingEnabled) - .DistinctBy(u => u.Id) - .ToList(); - - foreach (var user in usersToScrobble) - { - await SetAndCheckRateLimit(userRateLimits, user, license.Value); - } - - return usersToScrobble; - } - - - private async Task ProcessEvents(IEnumerable events, Dictionary userRateLimits, - int usersToScrobble, int progressCounter, int totalProgress, Func> createEvent) - { - var license = await _unitOfWork.SettingsRepository.GetSettingAsync(ServerSettingKey.LicenseKey); - foreach (var evt in events) - { - _logger.LogDebug("Processing Reading Events: {Count} / {Total}", progressCounter, totalProgress); - progressCounter++; - - // Check if this media item can even be processed for this user - if (!CanProcessScrobbleEvent(evt)) - { - continue; - } - - if (TokenService.HasTokenExpired(evt.AppUser.AniListAccessToken)) - { - _unitOfWork.ScrobbleRepository.Attach(new ScrobbleError() - { - Comment = "AniList token has expired and needs rotating. Scrobbling wont work until then", - Details = $"User: {evt.AppUser.UserName}, Expired: {TokenService.GetTokenExpiry(evt.AppUser.AniListAccessToken)}", - LibraryId = evt.LibraryId, - SeriesId = evt.SeriesId - }); - await _unitOfWork.CommitAsync(); - continue; - } - - if (evt.Series.IsBlacklisted || evt.Series.DontMatch) - { - _logger.LogInformation("Series {SeriesName} ({SeriesId}) can't be matched and thus cannot scrobble this event", evt.Series.Name, evt.SeriesId); - _unitOfWork.ScrobbleRepository.Attach(new ScrobbleError() - { - Comment = UnknownSeriesErrorMessage, - Details = $"User: {evt.AppUser.UserName} Series: {evt.Series.Name}", - LibraryId = evt.LibraryId, - SeriesId = evt.SeriesId - }); - evt.IsErrored = true; - evt.ErrorDetails = UnknownSeriesErrorMessage; - evt.ProcessDateUtc = DateTime.UtcNow; - _unitOfWork.ScrobbleRepository.Update(evt); - await _unitOfWork.CommitAsync(); - - continue; - } - - var count = await SetAndCheckRateLimit(userRateLimits, evt.AppUser, license.Value); - userRateLimits[evt.AppUserId] = count; - if (count == 0) - { - if (usersToScrobble == 1) break; - continue; - } - - try - { - var data = await createEvent(evt); - // We need to handle the encoding and changing it to the old one until we can update the API layer to handle these - // which could happen in v0.8.3 - if (data.VolumeNumber is Parser.SpecialVolumeNumber or Parser.DefaultChapterNumber) - { - data.VolumeNumber = 0; - } - - if (data.ChapterNumber is Parser.DefaultChapterNumber) - { - data.ChapterNumber = 0; - } - userRateLimits[evt.AppUserId] = await PostScrobbleUpdate(data, license.Value, evt); - evt.IsProcessed = true; - evt.ProcessDateUtc = DateTime.UtcNow; - _unitOfWork.ScrobbleRepository.Update(evt); - } - catch (FlurlHttpException) - { - // If a flurl exception occured, the API is likely down. Kill processing - throw; - } - catch (KavitaException ex) - { - if (ex.Message.Contains("Access token is invalid")) - { - _logger.LogCritical(ex, "Access Token for AppUserId: {AppUserId} needs to be regenerated/renewed to continue scrobbling", evt.AppUser.Id); - evt.IsErrored = true; - evt.ErrorDetails = AccessTokenErrorMessage; - _unitOfWork.ScrobbleRepository.Update(evt); - } - } - catch (Exception ex) - { - /* Swallow as it's already been handled in PostScrobbleUpdate */ - _logger.LogError(ex, "Error processing event {EventId}", evt.Id); - } - await SaveToDb(progressCounter); - // We can use count to determine how long to sleep based on rate gain. It might be specific to AniList, but we can model others - var delay = count > 10 ? TimeSpan.FromMilliseconds(ScrobbleSleepTime) : TimeSpan.FromSeconds(60); - await Task.Delay(delay); - } - - await SaveToDb(progressCounter, true); - return progressCounter; - } - - private async Task SaveToDb(int progressCounter, bool force = false) - { - if ((force || progressCounter % 5 == 0) && _unitOfWork.HasChanges()) - { - _logger.LogDebug("Saving Scrobbling Event Processing Progress"); - await _unitOfWork.CommitAsync(); - } - } - - - private static bool CanProcessScrobbleEvent(ScrobbleEvent readEvent) - { - var userProviders = GetUserProviders(readEvent.AppUser); - switch (readEvent.Series.Library.Type) - { - case LibraryType.Manga when MangaProviders.Intersect(userProviders).Any(): - case LibraryType.Comic when - ComicProviders.Intersect(userProviders).Any(): - case LibraryType.Book when - BookProviders.Intersect(userProviders).Any(): - case LibraryType.LightNovel when - LightNovelProviders.Intersect(userProviders).Any(): - return true; - default: - return false; - } - } - - private static List GetUserProviders(AppUser appUser) - { - var providers = new List(); - if (!string.IsNullOrEmpty(appUser.AniListAccessToken)) providers.Add(ScrobbleProvider.AniList); - - return providers; + var aniListId = ExtractId(seriesWithExternalMetadata.Metadata.WebLinks, AniListWeblinkWebsite); + return aniListId ?? seriesWithExternalMetadata.ExternalSeriesMetadata?.AniListId; } /// @@ -1178,23 +556,23 @@ public class ScrobblingService : IScrobblingService foreach (var webLink in webLinks.Split(',')) { if (!webLink.StartsWith(website)) continue; + var tokens = webLink.Split(website)[1].Split('/'); var value = tokens[index]; + if (typeof(T) == typeof(int?)) { - if (int.TryParse(value, CultureInfo.InvariantCulture, out var intValue)) - return (T)(object)intValue; + if (int.TryParse(value, CultureInfo.InvariantCulture, out var intValue)) return (T)(object)intValue; } else if (typeof(T) == typeof(int)) { - if (int.TryParse(value, CultureInfo.InvariantCulture, out var intValue)) - return (T)(object)intValue; + if (int.TryParse(value, CultureInfo.InvariantCulture, out var intValue)) return (T)(object)intValue; + return default; } else if (typeof(T) == typeof(long?)) { - if (long.TryParse(value, CultureInfo.InvariantCulture, out var longValue)) - return (T)(object)longValue; + if (long.TryParse(value, CultureInfo.InvariantCulture, out var longValue)) return (T)(object)longValue; } else if (typeof(T) == typeof(string)) { @@ -1238,6 +616,778 @@ public class ScrobblingService : IScrobblingService return id is null or 0 ? string.Empty : $"{url}{id}/"; } + #endregion + + /// + /// Returns false if, the series is on hold or Don't Match, or when the library has scrobbling disable or not eligible + /// + /// + /// + /// + /// + private async Task CheckIfCannotScrobble(int userId, int seriesId, Series series) + { + if (series.DontMatch) return true; + + if (await _unitOfWork.UserRepository.HasHoldOnSeries(userId, seriesId)) + { + _logger.LogInformation("Series {SeriesName} is on AppUserId {AppUserId}'s hold list. Not scrobbling", series.Name, userId); + return true; + } + + var library = await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(series.LibraryId); + if (library is not {AllowScrobbling: true} || !ExternalMetadataService.IsPlusEligible(library.Type)) return true; + + return false; + } + + /// + /// Returns the rate limit from the K+ api + /// + /// + /// + /// + private async Task GetRateLimit(string license, string aniListToken) + { + if (string.IsNullOrWhiteSpace(aniListToken)) return 0; + + try + { + return await _kavitaPlusApiService.GetRateLimit(license, aniListToken); + } + catch (Exception e) + { + _logger.LogError(e, "An error happened trying to get rate limit from Kavita+ API"); + } + + return 0; + } + + #region Scrobble process (Requests to K+) + + /// + /// Retrieve all events for which the series has not errored, then delete all current errors + /// + private async Task PrepareScrobbleContext() + { + var librariesWithScrobbling = (await _unitOfWork.LibraryRepository.GetLibrariesAsync()) + .AsEnumerable() + .Where(l => l.AllowScrobbling) + .Select(l => l.Id) + .ToImmutableHashSet(); + + var erroredSeries = (await _unitOfWork.ScrobbleRepository.GetScrobbleErrors()) + .Where(e => e.Comment is "Unknown Series" or UnknownSeriesErrorMessage or AccessTokenErrorMessage) + .Select(e => e.SeriesId) + .ToList(); + + var readEvents = (await _unitOfWork.ScrobbleRepository.GetByEvent(ScrobbleEventType.ChapterRead)) + .Where(e => librariesWithScrobbling.Contains(e.LibraryId)) + .Where(e => !erroredSeries.Contains(e.SeriesId)) + .ToList(); + var addToWantToRead = (await _unitOfWork.ScrobbleRepository.GetByEvent(ScrobbleEventType.AddWantToRead)) + .Where(e => librariesWithScrobbling.Contains(e.LibraryId)) + .Where(e => !erroredSeries.Contains(e.SeriesId)) + .ToList(); + var removeWantToRead = (await _unitOfWork.ScrobbleRepository.GetByEvent(ScrobbleEventType.RemoveWantToRead)) + .Where(e => librariesWithScrobbling.Contains(e.LibraryId)) + .Where(e => !erroredSeries.Contains(e.SeriesId)) + .ToList(); + var ratingEvents = (await _unitOfWork.ScrobbleRepository.GetByEvent(ScrobbleEventType.ScoreUpdated)) + .Where(e => librariesWithScrobbling.Contains(e.LibraryId)) + .Where(e => !erroredSeries.Contains(e.SeriesId)) + .ToList(); + + return new ScrobbleSyncContext + { + ReadEvents = readEvents, + RatingEvents = ratingEvents, + AddToWantToRead = addToWantToRead, + RemoveWantToRead = removeWantToRead, + Decisions = CalculateNetWantToReadDecisions(addToWantToRead, removeWantToRead), + RateLimits = [], + License = (await _unitOfWork.SettingsRepository.GetSettingAsync(ServerSettingKey.LicenseKey)).Value, + }; + } + + /// + /// Filters users who can scrobble, sets their rate limit and updates the + /// + /// + /// + private async Task PrepareUsersToScrobble(ScrobbleSyncContext ctx) + { + // For all userIds, ensure that we can connect and have access + var usersToScrobble = ctx.ReadEvents.Select(r => r.AppUser) + .Concat(ctx.AddToWantToRead.Select(r => r.AppUser)) + .Concat(ctx.RemoveWantToRead.Select(r => r.AppUser)) + .Concat(ctx.RatingEvents.Select(r => r.AppUser)) + .Where(user => !string.IsNullOrEmpty(user.AniListAccessToken)) + .Where(user => user.UserPreferences.AniListScrobblingEnabled) + .DistinctBy(u => u.Id) + .ToList(); + + foreach (var user in usersToScrobble) + { + await SetAndCheckRateLimit(ctx.RateLimits, user, ctx.License); + } + + ctx.Users = usersToScrobble; + } + + /// + /// Cleans up any events that are due to bugs or legacy + /// + private async Task CleanupOldOrBuggedEvents() + { + try + { + var eventsWithoutAnilistToken = (await _unitOfWork.ScrobbleRepository.GetEvents()) + .Where(e => e is { IsProcessed: false, IsErrored: false }) + .Where(e => string.IsNullOrEmpty(e.AppUser.AniListAccessToken)); + + _unitOfWork.ScrobbleRepository.Remove(eventsWithoutAnilistToken); + await _unitOfWork.CommitAsync(); + } + catch (Exception ex) + { + _logger.LogError(ex, "There was an exception when trying to delete old scrobble events when the user has no active token"); + } + } + + /// + /// This is a task that is run on a fixed schedule (every few hours or every day) that clears out the scrobble event table + /// and offloads the data to the API server which performs the syncing to the providers. + /// + [DisableConcurrentExecution(60 * 60 * 60)] + [AutomaticRetry(Attempts = 3, OnAttemptsExceeded = AttemptsExceededAction.Delete)] + public async Task ProcessUpdatesSinceLastSync() + { + var ctx = await PrepareScrobbleContext(); + if (ctx.TotalCount == 0) return; + + // Get all the applicable users to scrobble and set their rate limits + await PrepareUsersToScrobble(ctx); + + _logger.LogInformation("Scrobble Processing Details:" + + "\n Read Events: {ReadEventsCount}" + + "\n Want to Read Events: {WantToReadEventsCount}" + + "\n Rating Events: {RatingEventsCount}" + + "\n Users to Scrobble: {UsersToScrobbleCount}" + + "\n Total Events to Process: {TotalEvents}", + ctx.ReadEvents.Count, + ctx.Decisions.Count, + ctx.RatingEvents.Count, + ctx.Users.Count, + ctx.TotalCount); + + try + { + await ProcessReadEvents(ctx); + await ProcessRatingEvents(ctx); + await ProcessWantToReadRatingEvents(ctx); + } + catch (FlurlHttpException ex) + { + _logger.LogError(ex, "Kavita+ API or a Scrobble service may be experiencing an outage. Stopping sending data"); + return; + } + + + await SaveToDb(ctx.ProgressCounter, true); + _logger.LogInformation("Scrobbling Events is complete"); + + await CleanupOldOrBuggedEvents(); + } + + /// + /// Calculates the net want-to-read decisions by considering all events. + /// Returns events that represent the final state for each user/series pair. + /// + /// List of events for adding to want-to-read + /// List of events for removing from want-to-read + /// List of events that represent the final state (add or remove) + private static List CalculateNetWantToReadDecisions(List addEvents, List removeEvents) + { + // Create a dictionary to track the latest event for each user/series combination + var latestEvents = new Dictionary<(int SeriesId, int AppUserId), ScrobbleEvent>(); + + // Process all add events + foreach (var addEvent in addEvents) + { + var key = (addEvent.SeriesId, addEvent.AppUserId); + + if (latestEvents.TryGetValue(key, out var value) && addEvent.CreatedUtc <= value.CreatedUtc) continue; + + value = addEvent; + latestEvents[key] = value; + } + + // Process all remove events + foreach (var removeEvent in removeEvents) + { + var key = (removeEvent.SeriesId, removeEvent.AppUserId); + + if (latestEvents.TryGetValue(key, out var value) && removeEvent.CreatedUtc <= value.CreatedUtc) continue; + + value = removeEvent; + latestEvents[key] = value; + } + + // Return all events that represent the final state + return latestEvents.Values.ToList(); + } + + private async Task ProcessWantToReadRatingEvents(ScrobbleSyncContext ctx) + { + await ProcessEvents(ctx.Decisions, ctx, evt => Task.FromResult(new ScrobbleDto + { + Format = evt.Format, + AniListId = evt.AniListId, + MALId = (int?) evt.MalId, + ScrobbleEventType = evt.ScrobbleEventType, + ChapterNumber = evt.ChapterNumber, + VolumeNumber = (int?) evt.VolumeNumber, + AniListToken = evt.AppUser.AniListAccessToken ?? string.Empty, + SeriesName = evt.Series.Name, + LocalizedSeriesName = evt.Series.LocalizedName, + Year = evt.Series.Metadata.ReleaseYear + })); + + // After decisions, we need to mark all the want to read and remove from want to read as completed + var processedDecisions = ctx.Decisions.Where(d => d.IsProcessed).ToList(); + if (processedDecisions.Count > 0) + { + foreach (var scrobbleEvent in processedDecisions) + { + scrobbleEvent.IsProcessed = true; + scrobbleEvent.ProcessDateUtc = DateTime.UtcNow; + _unitOfWork.ScrobbleRepository.Update(scrobbleEvent); + } + await _unitOfWork.CommitAsync(); + } + } + + private async Task ProcessRatingEvents(ScrobbleSyncContext ctx) + { + await ProcessEvents(ctx.RatingEvents, ctx, evt => Task.FromResult(new ScrobbleDto + { + Format = evt.Format, + AniListId = evt.AniListId, + MALId = (int?) evt.MalId, + ScrobbleEventType = evt.ScrobbleEventType, + AniListToken = evt.AppUser.AniListAccessToken ?? string.Empty, + SeriesName = evt.Series.Name, + LocalizedSeriesName = evt.Series.LocalizedName, + Rating = evt.Rating, + Year = evt.Series.Metadata.ReleaseYear + })); + } + + private async Task ProcessReadEvents(ScrobbleSyncContext ctx) + { + // Recalculate the highest volume/chapter + foreach (var readEvt in ctx.ReadEvents) + { + // Note: this causes skewing in the scrobble history because it makes it look like there are duplicate events + readEvt.VolumeNumber = + (int) await _unitOfWork.AppUserProgressRepository.GetHighestFullyReadVolumeForSeries(readEvt.SeriesId, + readEvt.AppUser.Id); + readEvt.ChapterNumber = + await _unitOfWork.AppUserProgressRepository.GetHighestFullyReadChapterForSeries(readEvt.SeriesId, + readEvt.AppUser.Id); + _unitOfWork.ScrobbleRepository.Update(readEvt); + } + + await ProcessEvents(ctx.ReadEvents, ctx, async evt => new ScrobbleDto + { + Format = evt.Format, + AniListId = evt.AniListId, + MALId = (int?) evt.MalId, + ScrobbleEventType = evt.ScrobbleEventType, + ChapterNumber = evt.ChapterNumber, + VolumeNumber = (int?) evt.VolumeNumber, + AniListToken = evt.AppUser.AniListAccessToken ?? string.Empty, + SeriesName = evt.Series.Name, + LocalizedSeriesName = evt.Series.LocalizedName, + ScrobbleDateUtc = evt.LastModifiedUtc, + Year = evt.Series.Metadata.ReleaseYear, + StartedReadingDateUtc = await _unitOfWork.AppUserProgressRepository.GetFirstProgressForSeries(evt.SeriesId, evt.AppUser.Id), + LatestReadingDateUtc = await _unitOfWork.AppUserProgressRepository.GetLatestProgressForSeries(evt.SeriesId, evt.AppUser.Id), + }); + } + + /// + /// Returns true if the user token is valid + /// + /// + /// + /// If the token is not, adds a scrobble error + private async Task ValidateUserToken(ScrobbleEvent evt) + { + if (!TokenService.HasTokenExpired(evt.AppUser.AniListAccessToken)) + return true; + + _unitOfWork.ScrobbleRepository.Attach(new ScrobbleError + { + Comment = "AniList token has expired and needs rotating. Scrobbling wont work until then", + Details = $"User: {evt.AppUser.UserName}, Expired: {TokenService.GetTokenExpiry(evt.AppUser.AniListAccessToken)}", + LibraryId = evt.LibraryId, + SeriesId = evt.SeriesId + }); + await _unitOfWork.CommitAsync(); + return false; + } + + /// + /// Returns true if the series can be scrobbled + /// + /// + /// + /// If the series cannot be scrobbled, adds a scrobble error + private async Task ValidateSeriesCanBeScrobbled(ScrobbleEvent evt) + { + if (evt.Series is { IsBlacklisted: false, DontMatch: false }) + return true; + + _logger.LogInformation("Series {SeriesName} ({SeriesId}) can't be matched and thus cannot scrobble this event", + evt.Series.Name, evt.SeriesId); + + _unitOfWork.ScrobbleRepository.Attach(new ScrobbleError + { + Comment = UnknownSeriesErrorMessage, + Details = $"User: {evt.AppUser.UserName} Series: {evt.Series.Name}", + LibraryId = evt.LibraryId, + SeriesId = evt.SeriesId + }); + + evt.SetErrorMessage(UnknownSeriesErrorMessage); + evt.ProcessDateUtc = DateTime.UtcNow; + _unitOfWork.ScrobbleRepository.Update(evt); + await _unitOfWork.CommitAsync(); + return false; + } + + /// + /// Removed Special parses numbers from chatter and volume numbers + /// + /// + /// + private static ScrobbleDto NormalizeScrobbleData(ScrobbleDto data) + { + // We need to handle the encoding and changing it to the old one until we can update the API layer to handle these + // which could happen in v0.8.3 + if (data.VolumeNumber is Parser.SpecialVolumeNumber or Parser.DefaultChapterNumber) + { + data.VolumeNumber = 0; + } + + + if (data.ChapterNumber is Parser.DefaultChapterNumber) + { + data.ChapterNumber = 0; + } + + + return data; + } + + /// + /// Loops through all events, and post them to K+ + /// + /// + /// + /// + private async Task ProcessEvents(IEnumerable events, ScrobbleSyncContext ctx, Func> createEvent) + { + foreach (var evt in events.Where(CanProcessScrobbleEvent)) + { + _logger.LogDebug("Processing Scrobble Events: {Count} / {Total}", ctx.ProgressCounter, ctx.TotalCount); + ctx.ProgressCounter++; + + if (!await ValidateUserToken(evt)) continue; + if (!await ValidateSeriesCanBeScrobbled(evt)) continue; + + var count = await SetAndCheckRateLimit(ctx.RateLimits, evt.AppUser, ctx.License); + if (count == 0) + { + if (ctx.Users.Count == 1) break; + continue; + } + + try + { + var data = NormalizeScrobbleData(await createEvent(evt)); + + ctx.RateLimits[evt.AppUserId] = await PostScrobbleUpdate(data, ctx.License, evt); + + evt.IsProcessed = true; + evt.ProcessDateUtc = DateTime.UtcNow; + _unitOfWork.ScrobbleRepository.Update(evt); + } + catch (FlurlHttpException) + { + // If a flurl exception occured, the API is likely down. Kill processing + throw; + } + catch (KavitaException ex) + { + if (ex.Message.Contains("Access token is invalid")) + { + _logger.LogCritical(ex, "Access Token for AppUserId: {AppUserId} needs to be regenerated/renewed to continue scrobbling", evt.AppUser.Id); + evt.SetErrorMessage(AccessTokenErrorMessage); + _unitOfWork.ScrobbleRepository.Update(evt); + + // Ensure series with this error do not get re-processed next sync + _unitOfWork.ScrobbleRepository.Attach(new ScrobbleError + { + Comment = AccessTokenErrorMessage, + Details = $"{evt.AppUser.UserName} has an invalid access token (K+ Error)", + LibraryId = evt.LibraryId, + SeriesId = evt.SeriesId, + }); + } + } + catch (Exception ex) + { + /* Swallow as it's already been handled in PostScrobbleUpdate */ + _logger.LogError(ex, "Error processing event {EventId}", evt.Id); + } + + await SaveToDb(ctx.ProgressCounter); + + // We can use count to determine how long to sleep based on rate gain. It might be specific to AniList, but we can model others + var delay = count > 10 ? TimeSpan.FromMilliseconds(ScrobbleSleepTime) : TimeSpan.FromSeconds(60); + await Task.Delay(delay); + } + + await SaveToDb(ctx.ProgressCounter, true); + } + + /// + /// Save changes every five updates + /// + /// + /// Ignore update count check + private async Task SaveToDb(int progressCounter, bool force = false) + { + if ((force || progressCounter % 5 == 0) && _unitOfWork.HasChanges()) + { + _logger.LogDebug("Saving Scrobbling Event Processing Progress"); + await _unitOfWork.CommitAsync(); + } + } + + /// + /// If no errors have been logged for the given series, creates a new Unknown series error, and blacklists the series + /// + /// + /// + private async Task MarkSeriesAsUnknown(ScrobbleDto data, ScrobbleEvent evt) + { + if (await _unitOfWork.ScrobbleRepository.HasErrorForSeries(evt.SeriesId)) return; + + // Create a new ExternalMetadata entry to indicate that this is not matchable + var series = await _unitOfWork.SeriesRepository.GetSeriesByIdAsync(evt.SeriesId, SeriesIncludes.ExternalMetadata); + if (series == null) return; + + series.ExternalSeriesMetadata ??= new ExternalSeriesMetadata {SeriesId = evt.SeriesId}; + series.IsBlacklisted = true; + _unitOfWork.SeriesRepository.Update(series); + + _unitOfWork.ScrobbleRepository.Attach(new ScrobbleError + { + Comment = UnknownSeriesErrorMessage, + Details = data.SeriesName, + LibraryId = evt.LibraryId, + SeriesId = evt.SeriesId + }); + } + + /// + /// Makes the K+ request, and handles any exceptions that occur + /// + /// Data to send to K+ + /// K+ license key + /// Related scrobble event + /// + /// Exceptions may be rethrown as a KavitaException + /// Some FlurlHttpException are also rethrown + public async Task PostScrobbleUpdate(ScrobbleDto data, string license, ScrobbleEvent evt) + { + try + { + var response = await _kavitaPlusApiService.PostScrobbleUpdate(data, license); + + _logger.LogDebug("K+ API Scrobble response for series {SeriesName}: Successful {Successful}, ErrorMessage {ErrorMessage}, ExtraInformation: {ExtraInformation}, RateLeft: {RateLeft}", + data.SeriesName, response.Successful, response.ErrorMessage, response.ExtraInformation, response.RateLeft); + + if (response.Successful || response.ErrorMessage == null) return response.RateLeft; + + // Might want to log this under ScrobbleError + if (response.ErrorMessage.Contains("Too Many Requests")) + { + _logger.LogInformation("Hit Too many requests while posting scrobble updates, sleeping to regain requests and retrying"); + await Task.Delay(TimeSpan.FromMinutes(10)); + return await PostScrobbleUpdate(data, license, evt); + } + + if (response.ErrorMessage.Contains("Unauthorized")) + { + _logger.LogCritical("Kavita+ responded with Unauthorized. Please check your subscription"); + await _licenseService.HasActiveLicense(true); + evt.SetErrorMessage(InvalidKPlusLicenseErrorMessage); + throw new KavitaException("Kavita+ responded with Unauthorized. Please check your subscription"); + } + + if (response.ErrorMessage.Contains("Access token is invalid")) + { + evt.SetErrorMessage(AccessTokenErrorMessage); + throw new KavitaException("Access token is invalid"); + } + + if (response.ErrorMessage.Contains("Unknown Series")) + { + // Log the Series name and Id in ScrobbleErrors + _logger.LogInformation("Kavita+ was unable to match the series: {SeriesName}", evt.Series.Name); + await MarkSeriesAsUnknown(data, evt); + evt.SetErrorMessage(UnknownSeriesErrorMessage); + } else if (response.ErrorMessage.StartsWith("Review")) + { + // Log the Series name and Id in ScrobbleErrors + _logger.LogInformation("Kavita+ was unable to save the review"); + if (!await _unitOfWork.ScrobbleRepository.HasErrorForSeries(evt.SeriesId)) + { + _unitOfWork.ScrobbleRepository.Attach(new ScrobbleError() + { + Comment = response.ErrorMessage, + Details = data.SeriesName, + LibraryId = evt.LibraryId, + SeriesId = evt.SeriesId + }); + } + evt.SetErrorMessage(ReviewFailedErrorMessage); + } + + return response.RateLeft; + } + catch (FlurlHttpException ex) + { + var errorMessage = await ex.GetResponseStringAsync(); + // Trim quotes if the response is a JSON string + errorMessage = errorMessage.Trim('"'); + + if (errorMessage.Contains("Too Many Requests")) + { + _logger.LogInformation("Hit Too many requests while posting scrobble updates, sleeping to regain requests and retrying"); + await Task.Delay(TimeSpan.FromMinutes(10)); + return await PostScrobbleUpdate(data, license, evt); + } + + _logger.LogError(ex, "Scrobbling to Kavita+ API failed due to error: {ErrorMessage}", ex.Message); + if (ex.StatusCode == 500 || ex.Message.Contains("Call failed with status code 500 (Internal Server Error)")) + { + if (!await _unitOfWork.ScrobbleRepository.HasErrorForSeries(evt.SeriesId)) + { + _unitOfWork.ScrobbleRepository.Attach(new ScrobbleError() + { + Comment = UnknownSeriesErrorMessage, + Details = data.SeriesName, + LibraryId = evt.LibraryId, + SeriesId = evt.SeriesId + }); + } + evt.SetErrorMessage(BadPayLoadErrorMessage); + throw new KavitaException(BadPayLoadErrorMessage); + } + throw; + } + } + + #endregion + + #region BackFill + + + /// + /// This will backfill events from existing progress history, ratings, and want to read for users that have a valid license + /// + /// Defaults to 0 meaning all users. Allows a userId to be set if a scrobble key is added to a user + public async Task CreateEventsFromExistingHistory(int userId = 0) + { + if (!await _licenseService.HasActiveLicense()) return; + + if (userId != 0) + { + var user = await _unitOfWork.UserRepository.GetUserByIdAsync(userId); + if (user == null || string.IsNullOrEmpty(user.AniListAccessToken)) return; + if (user.HasRunScrobbleEventGeneration) + { + _logger.LogWarning("User {UserName} has already run scrobble event generation, Kavita will not generate more events", user.UserName); + return; + } + } + + var libAllowsScrobbling = (await _unitOfWork.LibraryRepository.GetLibrariesAsync()) + .ToDictionary(lib => lib.Id, lib => lib.AllowScrobbling); + + var userIds = (await _unitOfWork.UserRepository.GetAllUsersAsync()) + .Where(l => userId == 0 || userId == l.Id) + .Where(u => !u.HasRunScrobbleEventGeneration) + .Select(u => u.Id); + + foreach (var uId in userIds) + { + await CreateEventsFromExistingHistoryForUser(uId, libAllowsScrobbling); + } + } + + /// + /// Creates wantToRead, rating, reviews, and series progress events for the suer + /// + /// + /// + private async Task CreateEventsFromExistingHistoryForUser(int userId, Dictionary libAllowsScrobbling) + { + var wantToRead = await _unitOfWork.SeriesRepository.GetWantToReadForUserAsync(userId); + foreach (var wtr in wantToRead) + { + if (!libAllowsScrobbling[wtr.LibraryId]) continue; + await ScrobbleWantToReadUpdate(userId, wtr.Id, true); + } + + var ratings = await _unitOfWork.UserRepository.GetSeriesWithRatings(userId); + foreach (var rating in ratings) + { + if (!libAllowsScrobbling[rating.Series.LibraryId]) continue; + await ScrobbleRatingUpdate(userId, rating.SeriesId, rating.Rating); + } + + var reviews = await _unitOfWork.UserRepository.GetSeriesWithReviews(userId); + foreach (var review in reviews.Where(r => !string.IsNullOrEmpty(r.Review))) + { + if (!libAllowsScrobbling[review.Series.LibraryId]) continue; + await ScrobbleReviewUpdate(userId, review.SeriesId, string.Empty, review.Review!); + } + + var seriesWithProgress = await _unitOfWork.SeriesRepository.GetSeriesDtoForLibraryIdAsync(0, userId, + new UserParams(), new FilterDto + { + ReadStatus = new ReadStatus + { + Read = true, + InProgress = true, + NotRead = false + }, + Libraries = libAllowsScrobbling.Keys.Where(k => libAllowsScrobbling[k]).ToList() + }); + + foreach (var series in seriesWithProgress.Where(series => series.PagesRead > 0)) + { + if (!libAllowsScrobbling[series.LibraryId]) continue; + await ScrobbleReadingUpdate(userId, series.Id); + } + + var user = await _unitOfWork.UserRepository.GetUserByIdAsync(userId); + if (user != null) + { + user.HasRunScrobbleEventGeneration = true; + user.ScrobbleEventGenerationRan = DateTime.UtcNow; + await _unitOfWork.CommitAsync(); + } + } + + public async Task CreateEventsFromExistingHistoryForSeries(int seriesId) + { + if (!await _licenseService.HasActiveLicense()) return; + + var series = await _unitOfWork.SeriesRepository.GetSeriesByIdAsync(seriesId, SeriesIncludes.Library); + if (series == null || !series.Library.AllowScrobbling) return; + + _logger.LogInformation("Creating Scrobbling events for Series {SeriesName}", series.Name); + + var userIds = (await _unitOfWork.UserRepository.GetAllUsersAsync()).Select(u => u.Id); + + foreach (var uId in userIds) + { + // Handle "Want to Read" updates specific to the series + var wantToRead = await _unitOfWork.SeriesRepository.GetWantToReadForUserAsync(uId); + foreach (var wtr in wantToRead.Where(wtr => wtr.Id == seriesId)) + { + await ScrobbleWantToReadUpdate(uId, wtr.Id, true); + } + + // Handle ratings specific to the series + var ratings = await _unitOfWork.UserRepository.GetSeriesWithRatings(uId); + foreach (var rating in ratings.Where(rating => rating.SeriesId == seriesId)) + { + await ScrobbleRatingUpdate(uId, rating.SeriesId, rating.Rating); + } + + // Handle review specific to the series + var reviews = await _unitOfWork.UserRepository.GetSeriesWithReviews(uId); + foreach (var review in reviews.Where(r => r.SeriesId == seriesId && !string.IsNullOrEmpty(r.Review))) + { + await ScrobbleReviewUpdate(uId, review.SeriesId, string.Empty, review.Review!); + } + + // Handle progress updates for the specific series + await ScrobbleReadingUpdate(uId, seriesId); + } + } + + #endregion + + /// + /// Removes all events (active) that are tied to a now-on hold series + /// + /// + /// + public async Task ClearEventsForSeries(int userId, int seriesId) + { + _logger.LogInformation("Clearing Pre-existing Scrobble events for Series {SeriesId} by User {AppUserId} as Series is now on hold list", seriesId, userId); + + var events = await _unitOfWork.ScrobbleRepository.GetUserEventsForSeries(userId, seriesId); + _unitOfWork.ScrobbleRepository.Remove(events); + await _unitOfWork.CommitAsync(); + } + + /// + /// Removes all events that have been processed that are 7 days old + /// + [DisableConcurrentExecution(60 * 60 * 60)] + [AutomaticRetry(Attempts = 3, OnAttemptsExceeded = AttemptsExceededAction.Delete)] + public async Task ClearProcessedEvents() + { + const int daysAgo = 7; + var events = await _unitOfWork.ScrobbleRepository.GetProcessedEvents(daysAgo); + _unitOfWork.ScrobbleRepository.Remove(events); + _logger.LogInformation("Removing {Count} scrobble events that have been processed {DaysAgo}+ days ago", events.Count, daysAgo); + await _unitOfWork.CommitAsync(); + } + + private static bool CanProcessScrobbleEvent(ScrobbleEvent readEvent) + { + var userProviders = GetUserProviders(readEvent.AppUser); + switch (readEvent.Series.Library.Type) + { + case LibraryType.Manga when MangaProviders.Intersect(userProviders).Any(): + case LibraryType.Comic when ComicProviders.Intersect(userProviders).Any(): + case LibraryType.Book when BookProviders.Intersect(userProviders).Any(): + case LibraryType.LightNovel when LightNovelProviders.Intersect(userProviders).Any(): + return true; + default: + return false; + } + } + + private static List GetUserProviders(AppUser appUser) + { + var providers = new List(); + if (!string.IsNullOrEmpty(appUser.AniListAccessToken)) providers.Add(ScrobbleProvider.AniList); + + return providers; + } private async Task SetAndCheckRateLimit(IDictionary userRateLimits, AppUser user, string license) { diff --git a/UI/Web/src/app/_models/scrobbling/scrobble-event.ts b/UI/Web/src/app/_models/scrobbling/scrobble-event.ts index 48a75afda..7db1ceeaa 100644 --- a/UI/Web/src/app/_models/scrobbling/scrobble-event.ts +++ b/UI/Web/src/app/_models/scrobbling/scrobble-event.ts @@ -7,6 +7,7 @@ export enum ScrobbleEventType { } export interface ScrobbleEvent { + id: number; seriesName: string; seriesId: number; libraryId: number; diff --git a/UI/Web/src/app/_services/scrobbling.service.ts b/UI/Web/src/app/_services/scrobbling.service.ts index 76b9212f4..cfc7b34ac 100644 --- a/UI/Web/src/app/_services/scrobbling.service.ts +++ b/UI/Web/src/app/_services/scrobbling.service.ts @@ -104,6 +104,10 @@ export class ScrobblingService { triggerScrobbleEventGeneration() { return this.httpClient.post(this.baseUrl + 'scrobbling/generate-scrobble-events', TextResonse); - } + + bulkRemoveEvents(eventIds: number[]) { + return this.httpClient.post(this.baseUrl + "scrobbling/bulk-remove-events", eventIds) + } + } diff --git a/UI/Web/src/app/_single-module/user-scrobble-history/user-scrobble-history.component.html b/UI/Web/src/app/_single-module/user-scrobble-history/user-scrobble-history.component.html index 51caae2f3..96fd71b95 100644 --- a/UI/Web/src/app/_single-module/user-scrobble-history/user-scrobble-history.component.html +++ b/UI/Web/src/app/_single-module/user-scrobble-history/user-scrobble-history.component.html @@ -20,7 +20,10 @@
- +
+ + +
@@ -40,6 +43,20 @@ [sorts]="[{prop: 'createdUtc', dir: 'desc'}]" > + + +
+ + +
+ + + + + + {{t('created-header')}} @@ -101,7 +118,7 @@ - + {{t('is-processed-header')}} diff --git a/UI/Web/src/app/_single-module/user-scrobble-history/user-scrobble-history.component.ts b/UI/Web/src/app/_single-module/user-scrobble-history/user-scrobble-history.component.ts index c0306c4cf..ac48b6add 100644 --- a/UI/Web/src/app/_single-module/user-scrobble-history/user-scrobble-history.component.ts +++ b/UI/Web/src/app/_single-module/user-scrobble-history/user-scrobble-history.component.ts @@ -1,4 +1,12 @@ -import {ChangeDetectionStrategy, ChangeDetectorRef, Component, DestroyRef, inject, OnInit} from '@angular/core'; +import { + ChangeDetectionStrategy, + ChangeDetectorRef, + Component, + DestroyRef, + HostListener, + inject, + OnInit +} from '@angular/core'; import {ScrobbleProvider, ScrobblingService} from "../../_services/scrobbling.service"; import {takeUntilDestroyed} from "@angular/core/rxjs-interop"; @@ -9,7 +17,7 @@ import {ScrobbleEventSortField} from "../../_models/scrobbling/scrobble-event-fi import {debounceTime, take} from "rxjs/operators"; import {PaginatedResult} from "../../_models/pagination"; import {SortEvent} from "../table/_directives/sortable-header.directive"; -import {FormControl, FormGroup, ReactiveFormsModule} from "@angular/forms"; +import {FormControl, FormGroup, FormsModule, ReactiveFormsModule} from "@angular/forms"; import {translate, TranslocoModule} from "@jsverse/transloco"; import {DefaultValuePipe} from "../../_pipes/default-value.pipe"; import {TranslocoLocaleModule} from "@jsverse/transloco-locale"; @@ -19,6 +27,7 @@ import {ColumnMode, NgxDatatableModule} from "@siemens/ngx-datatable"; import {AsyncPipe} from "@angular/common"; import {AccountService} from "../../_services/account.service"; import {ToastrService} from "ngx-toastr"; +import {SelectionModel} from "../../typeahead/_models/selection-model"; export interface DataTablePage { pageNumber: number, @@ -30,7 +39,7 @@ export interface DataTablePage { @Component({ selector: 'app-user-scrobble-history', imports: [ScrobbleEventTypePipe, ReactiveFormsModule, TranslocoModule, - DefaultValuePipe, TranslocoLocaleModule, UtcToLocalTimePipe, NgbTooltip, NgxDatatableModule, AsyncPipe], + DefaultValuePipe, TranslocoLocaleModule, UtcToLocalTimePipe, NgbTooltip, NgxDatatableModule, AsyncPipe, FormsModule], templateUrl: './user-scrobble-history.component.html', styleUrls: ['./user-scrobble-history.component.scss'], changeDetection: ChangeDetectionStrategy.OnPush @@ -48,8 +57,6 @@ export class UserScrobbleHistoryComponent implements OnInit { private readonly toastr = inject(ToastrService); protected readonly accountService = inject(AccountService); - - tokenExpired = false; formGroup: FormGroup = new FormGroup({ 'filter': new FormControl('', []) @@ -68,6 +75,21 @@ export class UserScrobbleHistoryComponent implements OnInit { }; hasRunScrobbleGen: boolean = false; + selections: SelectionModel = new SelectionModel(); + selectAll: boolean = false; + isShiftDown: boolean = false; + lastSelectedIndex: number | null = null; + + @HostListener('document:keydown.shift', ['$event']) + handleKeypress(_: KeyboardEvent) { + this.isShiftDown = true; + } + + @HostListener('document:keyup.shift', ['$event']) + handleKeyUp(_: KeyboardEvent) { + this.isShiftDown = false; + } + ngOnInit() { this.pageInfo.pageNumber = 0; @@ -118,6 +140,7 @@ export class UserScrobbleHistoryComponent implements OnInit { .pipe(take(1)) .subscribe((result: PaginatedResult) => { this.events = result.result; + this.selections = new SelectionModel(false, this.events); this.pageInfo.totalPages = result.pagination.totalPages - 1; // ngx-datatable is 0 based, Kavita is 1 based this.pageInfo.size = result.pagination.itemsPerPage; @@ -143,4 +166,55 @@ export class UserScrobbleHistoryComponent implements OnInit { this.toastr.info(translate('toasts.scrobble-gen-init')) }); } + + bulkDelete() { + if (!this.selections.hasAnySelected()) { + return; + } + + const eventIds = this.selections.selected().map(e => e.id); + + this.scrobblingService.bulkRemoveEvents(eventIds).subscribe({ + next: () => { + this.events = this.events.filter(e => !eventIds.includes(e.id)); + this.selectAll = false; + this.selections.clearSelected(); + this.pageInfo.totalElements -= eventIds.length; + this.cdRef.markForCheck(); + }, + error: err => { + console.error(err); + } + }); + } + + toggleAll() { + this.selectAll = !this.selectAll; + this.events.forEach(e => this.selections.toggle(e, this.selectAll)); + this.cdRef.markForCheck(); + } + + handleSelection(item: ScrobbleEvent, index: number) { + if (this.isShiftDown && this.lastSelectedIndex !== null) { + // Bulk select items between the last selected item and the current one + const start = Math.min(this.lastSelectedIndex, index); + const end = Math.max(this.lastSelectedIndex, index); + + for (let i = start; i <= end; i++) { + const event = this.events[i]; + if (!this.selections.isSelected(event, (e1, e2) => e1.id == e2.id)) { + this.selections.toggle(event, true); + } + } + } else { + this.selections.toggle(item); + } + + this.lastSelectedIndex = index; + + + const numberOfSelected = this.selections.selected().length; + this.selectAll = numberOfSelected === this.events.length; + this.cdRef.markForCheck(); + } } diff --git a/UI/Web/src/app/admin/manage-scrobble-errors/manage-scrobble-errors.component.html b/UI/Web/src/app/admin/manage-scrobble-errors/manage-scrobble-errors.component.html index 78724272c..59a45873e 100644 --- a/UI/Web/src/app/admin/manage-scrobble-errors/manage-scrobble-errors.component.html +++ b/UI/Web/src/app/admin/manage-scrobble-errors/manage-scrobble-errors.component.html @@ -8,7 +8,7 @@
- +
diff --git a/UI/Web/src/app/typeahead/_models/selection-model.ts b/UI/Web/src/app/typeahead/_models/selection-model.ts index c4b2ab18a..8493a4eed 100644 --- a/UI/Web/src/app/typeahead/_models/selection-model.ts +++ b/UI/Web/src/app/typeahead/_models/selection-model.ts @@ -70,6 +70,28 @@ export class SelectionModel { return (selectedCount !== this._data.length && selectedCount !== 0) } + /** + * @return If at least one item is selected + */ + hasAnySelected(): boolean { + for (const d of this._data) { + if (d.selected) { + return true; + } + } + return false; + } + + /** + * Marks every data entry has not selected + */ + clearSelected() { + this._data = this._data.map(d => { + d.selected = false; + return d; + }); + } + /** * * @returns All Selected items diff --git a/UI/Web/src/assets/langs/en.json b/UI/Web/src/assets/langs/en.json index 2a2d40c4f..91a3dac9e 100644 --- a/UI/Web/src/assets/langs/en.json +++ b/UI/Web/src/assets/langs/en.json @@ -42,6 +42,8 @@ "series-header": "Series", "data-header": "Data", "is-processed-header": "Is Processed", + "select-all-label": "Select all", + "delete-selected-label": "Delete selected", "no-data": "{{common.no-data}}", "volume-and-chapter-num": "Volume {{v}} Chapter {{n}}", "volume-num": "Volume {{num}}",