From 9035b6cc4ef3194518ca44407126fc85850ef163 Mon Sep 17 00:00:00 2001 From: Joseph Milazzo Date: Mon, 15 Mar 2021 08:43:43 -0500 Subject: [PATCH 1/9] Hangfire now dies gracefully when using CTRL+C rather than Stop button in Rider. Implemented one stream method for testing. Regenerated a few migrations due to oversight in index not taking account of library. --- API/Controllers/SeriesController.cs | 6 +++ API/Data/LibraryRepository.cs | 3 ++ .../Migrations/20210313001830_SearchIndex.cs | 23 -------- ...8_SearchIndexAndProgressDates.Designer.cs} | 12 +++-- ...10315134028_SearchIndexAndProgressDates.cs | 46 ++++++++++++++++ .../Migrations/DataContextModelSnapshot.cs | 8 ++- API/Data/SeriesRepository.cs | 53 ++++++++++++++++++- API/Entities/AppUserProgress.cs | 9 +++- API/Entities/Series.cs | 3 +- API/Interfaces/ISeriesRepository.cs | 1 + API/Program.cs | 1 - API/Services/CacheService.cs | 1 + API/Services/TaskScheduler.cs | 8 ++- API/Services/Tasks/BackupService.cs | 3 ++ API/Services/Tasks/CleanupService.cs | 2 + API/Services/Tasks/ScannerService.cs | 3 +- API/Startup.cs | 15 +++++- 17 files changed, 156 insertions(+), 41 deletions(-) delete mode 100644 API/Data/Migrations/20210313001830_SearchIndex.cs rename API/Data/Migrations/{20210313001830_SearchIndex.Designer.cs => 20210315134028_SearchIndexAndProgressDates.Designer.cs} (98%) create mode 100644 API/Data/Migrations/20210315134028_SearchIndexAndProgressDates.cs diff --git a/API/Controllers/SeriesController.cs b/API/Controllers/SeriesController.cs index 8be412c07..d4bd4c5a9 100644 --- a/API/Controllers/SeriesController.cs +++ b/API/Controllers/SeriesController.cs @@ -126,5 +126,11 @@ namespace API.Controllers return BadRequest("There was an error with updating the series"); } + + [HttpGet("recently-added")] + public async Task>> GetRecentlyAdded(int libraryId = 0) + { + return Ok(await _unitOfWork.SeriesRepository.GetRecentlyAdded(libraryId)); + } } } \ No newline at end of file diff --git a/API/Data/LibraryRepository.cs b/API/Data/LibraryRepository.cs index 80dbbb553..c33c42281 100644 --- a/API/Data/LibraryRepository.cs +++ b/API/Data/LibraryRepository.cs @@ -1,4 +1,5 @@ using System; +using System.Collections; using System.Collections.Generic; using System.Diagnostics; using System.Linq; @@ -112,5 +113,7 @@ namespace API.Data .Include(l => l.Folders) .ProjectTo(_mapper.ConfigurationProvider).ToListAsync(); } + + } } \ No newline at end of file diff --git a/API/Data/Migrations/20210313001830_SearchIndex.cs b/API/Data/Migrations/20210313001830_SearchIndex.cs deleted file mode 100644 index 2272f73bf..000000000 --- a/API/Data/Migrations/20210313001830_SearchIndex.cs +++ /dev/null @@ -1,23 +0,0 @@ -using Microsoft.EntityFrameworkCore.Migrations; - -namespace API.Data.Migrations -{ - public partial class SearchIndex : Migration - { - protected override void Up(MigrationBuilder migrationBuilder) - { - migrationBuilder.CreateIndex( - name: "IX_Series_Name_NormalizedName_LocalizedName", - table: "Series", - columns: new[] { "Name", "NormalizedName", "LocalizedName" }, - unique: true); - } - - protected override void Down(MigrationBuilder migrationBuilder) - { - migrationBuilder.DropIndex( - name: "IX_Series_Name_NormalizedName_LocalizedName", - table: "Series"); - } - } -} diff --git a/API/Data/Migrations/20210313001830_SearchIndex.Designer.cs b/API/Data/Migrations/20210315134028_SearchIndexAndProgressDates.Designer.cs similarity index 98% rename from API/Data/Migrations/20210313001830_SearchIndex.Designer.cs rename to API/Data/Migrations/20210315134028_SearchIndexAndProgressDates.Designer.cs index 5aff37c5d..a407ccc28 100644 --- a/API/Data/Migrations/20210313001830_SearchIndex.Designer.cs +++ b/API/Data/Migrations/20210315134028_SearchIndexAndProgressDates.Designer.cs @@ -9,8 +9,8 @@ using Microsoft.EntityFrameworkCore.Storage.ValueConversion; namespace API.Data.Migrations { [DbContext(typeof(DataContext))] - [Migration("20210313001830_SearchIndex")] - partial class SearchIndex + [Migration("20210315134028_SearchIndexAndProgressDates")] + partial class SearchIndexAndProgressDates { protected override void BuildTargetModel(ModelBuilder modelBuilder) { @@ -161,6 +161,12 @@ namespace API.Data.Migrations b.Property("ChapterId") .HasColumnType("INTEGER"); + b.Property("Created") + .HasColumnType("TEXT"); + + b.Property("LastModified") + .HasColumnType("TEXT"); + b.Property("PagesRead") .HasColumnType("INTEGER"); @@ -367,7 +373,7 @@ namespace API.Data.Migrations b.HasIndex("LibraryId"); - b.HasIndex("Name", "NormalizedName", "LocalizedName") + b.HasIndex("Name", "NormalizedName", "LocalizedName", "LibraryId") .IsUnique(); b.ToTable("Series"); diff --git a/API/Data/Migrations/20210315134028_SearchIndexAndProgressDates.cs b/API/Data/Migrations/20210315134028_SearchIndexAndProgressDates.cs new file mode 100644 index 000000000..02dc1db2c --- /dev/null +++ b/API/Data/Migrations/20210315134028_SearchIndexAndProgressDates.cs @@ -0,0 +1,46 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +namespace API.Data.Migrations +{ + public partial class SearchIndexAndProgressDates : Migration + { + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.AddColumn( + name: "Created", + table: "AppUserProgresses", + type: "TEXT", + nullable: false, + defaultValue: new DateTime(1, 1, 1, 0, 0, 0, 0, DateTimeKind.Unspecified)); + + migrationBuilder.AddColumn( + name: "LastModified", + table: "AppUserProgresses", + type: "TEXT", + nullable: false, + defaultValue: new DateTime(1, 1, 1, 0, 0, 0, 0, DateTimeKind.Unspecified)); + + migrationBuilder.CreateIndex( + name: "IX_Series_Name_NormalizedName_LocalizedName_LibraryId", + table: "Series", + columns: new[] { "Name", "NormalizedName", "LocalizedName", "LibraryId" }, + unique: true); + } + + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropIndex( + name: "IX_Series_Name_NormalizedName_LocalizedName_LibraryId", + table: "Series"); + + migrationBuilder.DropColumn( + name: "Created", + table: "AppUserProgresses"); + + migrationBuilder.DropColumn( + name: "LastModified", + table: "AppUserProgresses"); + } + } +} diff --git a/API/Data/Migrations/DataContextModelSnapshot.cs b/API/Data/Migrations/DataContextModelSnapshot.cs index 922966162..c0fcd6f87 100644 --- a/API/Data/Migrations/DataContextModelSnapshot.cs +++ b/API/Data/Migrations/DataContextModelSnapshot.cs @@ -159,6 +159,12 @@ namespace API.Data.Migrations b.Property("ChapterId") .HasColumnType("INTEGER"); + b.Property("Created") + .HasColumnType("TEXT"); + + b.Property("LastModified") + .HasColumnType("TEXT"); + b.Property("PagesRead") .HasColumnType("INTEGER"); @@ -365,7 +371,7 @@ namespace API.Data.Migrations b.HasIndex("LibraryId"); - b.HasIndex("Name", "NormalizedName", "LocalizedName") + b.HasIndex("Name", "NormalizedName", "LocalizedName", "LibraryId") .IsUnique(); b.ToTable("Series"); diff --git a/API/Data/SeriesRepository.cs b/API/Data/SeriesRepository.cs index d1b5e3eb5..e4bda9997 100644 --- a/API/Data/SeriesRepository.cs +++ b/API/Data/SeriesRepository.cs @@ -1,4 +1,6 @@ -using System.Collections.Generic; +using System; +using System.Collections; +using System.Collections.Generic; using System.Diagnostics; using System.Linq; using System.Threading.Tasks; @@ -275,5 +277,54 @@ namespace API.Data v.PagesRead = userProgress.Where(p => p.VolumeId == v.Id).Sum(p => p.PagesRead); } } + + /// + /// Returns a list of Series that were added within 2 weeks. + /// + /// Library to restrict to, if 0, will apply to all libraries + /// + public async Task> GetRecentlyAdded(int libraryId) + { + // && (libraryId <= 0 || s.LibraryId == libraryId) + var twoWeeksAgo = DateTime.Today.Subtract(TimeSpan.FromDays(14)); + _logger.LogDebug("2 weeks from today is: {Date}", twoWeeksAgo); + return await _context.Series + .Where(s => s.Created > twoWeeksAgo) + .Take(20) + .AsNoTracking() + .ProjectTo(_mapper.ConfigurationProvider) + .ToListAsync(); + + } + + + public async Task> GetSeriesStream(int userId) + { + // Testing out In Progress to figure out how to design generalized solution + var userProgress = await _context.AppUserProgresses + .Where(p => p.AppUserId == userId && p.PagesRead > 0) + .AsNoTracking() + .ToListAsync(); + if (!userProgress.Any()) return new SeriesDto[] {}; + + var seriesIds = userProgress.Select(p => p.SeriesId).ToList(); + /* + *select P.*, S.Name, S.Pages from AppUserProgresses AS P + LEFT join Series as "S" on s.Id = P.SeriesId + where AppUserId = 1 AND P.PagesRead > 0 AND P.PagesRead < S.Pages + * + */ + + + + + // var series = await _context.Series + // .Where(s => seriesIds.Contains(s.Id) && s.Pages) // I need a join + + + + + return new SeriesDto[] {}; + } } } \ No newline at end of file diff --git a/API/Entities/AppUserProgress.cs b/API/Entities/AppUserProgress.cs index be3953246..170a249ac 100644 --- a/API/Entities/AppUserProgress.cs +++ b/API/Entities/AppUserProgress.cs @@ -1,20 +1,25 @@  +using System; +using API.Entities.Interfaces; + namespace API.Entities { /// /// Represents the progress a single user has on a given Volume. Progress is realistically tracked against the Volume's chapters. /// - public class AppUserProgress + public class AppUserProgress : IEntityDate { public int Id { get; set; } public int PagesRead { get; set; } public int VolumeId { get; set; } public int SeriesId { get; set; } - public int ChapterId { get; set; } // Relationships public AppUser AppUser { get; set; } public int AppUserId { get; set; } + + public DateTime Created { get; set; } + public DateTime LastModified { get; set; } } } \ No newline at end of file diff --git a/API/Entities/Series.cs b/API/Entities/Series.cs index c3d5ba68e..6406e118f 100644 --- a/API/Entities/Series.cs +++ b/API/Entities/Series.cs @@ -5,7 +5,7 @@ using Microsoft.EntityFrameworkCore; namespace API.Entities { - [Index(nameof(Name), nameof(NormalizedName), nameof(LocalizedName), IsUnique = true)] + [Index(nameof(Name), nameof(NormalizedName), nameof(LocalizedName), nameof(LibraryId), IsUnique = true)] public class Series : IEntityDate { public int Id { get; set; } @@ -36,7 +36,6 @@ namespace API.Entities public DateTime Created { get; set; } public DateTime LastModified { get; set; } public byte[] CoverImage { get; set; } - // NOTE: Do I want to store a thumbImage for search results? /// /// Sum of all Volume page counts /// diff --git a/API/Interfaces/ISeriesRepository.cs b/API/Interfaces/ISeriesRepository.cs index e9d950937..03cc758b1 100644 --- a/API/Interfaces/ISeriesRepository.cs +++ b/API/Interfaces/ISeriesRepository.cs @@ -55,5 +55,6 @@ namespace API.Interfaces Task GetVolumeCoverImageAsync(int volumeId); Task GetSeriesCoverImageAsync(int seriesId); + Task> GetRecentlyAdded(int libraryId); } } \ No newline at end of file diff --git a/API/Program.cs b/API/Program.cs index 7d7628252..7bd6ef289 100644 --- a/API/Program.cs +++ b/API/Program.cs @@ -41,7 +41,6 @@ namespace API logger.LogError(ex, "An error occurred during migration"); } - await host.RunAsync(); } diff --git a/API/Services/CacheService.cs b/API/Services/CacheService.cs index a5bdb4220..8e6ede340 100644 --- a/API/Services/CacheService.cs +++ b/API/Services/CacheService.cs @@ -52,6 +52,7 @@ namespace API.Services return chapter; } + public void Cleanup() { _logger.LogInformation("Performing cleanup of Cache directory"); diff --git a/API/Services/TaskScheduler.cs b/API/Services/TaskScheduler.cs index 492ff6357..758d653a2 100644 --- a/API/Services/TaskScheduler.cs +++ b/API/Services/TaskScheduler.cs @@ -21,7 +21,7 @@ namespace API.Services private readonly ICleanupService _cleanupService; private readonly IDirectoryService _directoryService; - public BackgroundJobServer Client => new BackgroundJobServer(); + public static BackgroundJobServer Client => new BackgroundJobServer(); // new BackgroundJobServerOptions() // { // WorkerCount = 1 @@ -39,16 +39,14 @@ namespace API.Services _backupService = backupService; _cleanupService = cleanupService; _directoryService = directoryService; - - //Hangfire.RecurringJob.RemoveIfExists(); + ScheduleTasks(); - //JobStorage.Current.GetMonitoringApi().EnqueuedJobs() - } public void ScheduleTasks() { _logger.LogInformation("Scheduling reoccurring tasks"); + string setting = null; setting = Task.Run(() => _unitOfWork.SettingsRepository.GetSettingAsync(ServerSettingKey.TaskScan)).Result.Value; if (setting != null) diff --git a/API/Services/Tasks/BackupService.cs b/API/Services/Tasks/BackupService.cs index 3a5685035..9642a0faa 100644 --- a/API/Services/Tasks/BackupService.cs +++ b/API/Services/Tasks/BackupService.cs @@ -3,11 +3,13 @@ using System.Collections.Generic; using System.IO; using System.IO.Compression; using System.Linq; +using System.Threading; using System.Threading.Tasks; using API.Entities.Enums; using API.Extensions; using API.Interfaces; using API.Interfaces.Services; +using Hangfire; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; @@ -57,6 +59,7 @@ namespace API.Services.Tasks return files; } + [AutomaticRetry(Attempts = 3, LogEvents = false, OnAttemptsExceeded = AttemptsExceededAction.Fail)] public void BackupDatabase() { _logger.LogInformation("Beginning backup of Database at {BackupTime}", DateTime.Now); diff --git a/API/Services/Tasks/CleanupService.cs b/API/Services/Tasks/CleanupService.cs index d7079b4a7..a33cf746f 100644 --- a/API/Services/Tasks/CleanupService.cs +++ b/API/Services/Tasks/CleanupService.cs @@ -1,5 +1,6 @@ using System.IO; using API.Interfaces.Services; +using Hangfire; using Microsoft.Extensions.Logging; namespace API.Services.Tasks @@ -20,6 +21,7 @@ namespace API.Services.Tasks _logger = logger; } + [AutomaticRetry(Attempts = 3, LogEvents = false, OnAttemptsExceeded = AttemptsExceededAction.Fail)] public void Cleanup() { _logger.LogInformation("Cleaning temp directory"); diff --git a/API/Services/Tasks/ScannerService.cs b/API/Services/Tasks/ScannerService.cs index 5587ad0c8..af7b6eca8 100644 --- a/API/Services/Tasks/ScannerService.cs +++ b/API/Services/Tasks/ScannerService.cs @@ -33,7 +33,8 @@ namespace API.Services.Tasks _metadataService = metadataService; } - [DisableConcurrentExecution(timeoutInSeconds: 120)] + [DisableConcurrentExecution(timeoutInSeconds: 5)] + [AutomaticRetry(Attempts = 0, LogEvents = false, OnAttemptsExceeded = AttemptsExceededAction.Delete)] public void ScanLibraries() { var libraries = Task.Run(() => _unitOfWork.LibraryRepository.GetLibrariesAsync()).Result.ToList(); diff --git a/API/Startup.cs b/API/Startup.cs index 4e9319b37..d53e346d6 100644 --- a/API/Startup.cs +++ b/API/Startup.cs @@ -18,6 +18,7 @@ using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Microsoft.OpenApi.Models; +using IApplicationLifetime = Microsoft.AspNetCore.Hosting.IApplicationLifetime; namespace API { @@ -66,11 +67,11 @@ namespace API services .AddStartupTask() .TryAddSingleton(services); - + } // This method gets called by the runtime. Use this method to configure the HTTP request pipeline. - public void Configure(IApplicationBuilder app, IBackgroundJobClient backgroundJobs, IWebHostEnvironment env) + public void Configure(IApplicationBuilder app, IBackgroundJobClient backgroundJobs, IWebHostEnvironment env, IHostApplicationLifetime applicationLifetime) { app.UseMiddleware(); @@ -125,6 +126,16 @@ namespace API endpoints.MapHangfireDashboard(); endpoints.MapFallbackToController("Index", "Fallback"); }); + + applicationLifetime.ApplicationStopping.Register(OnShutdown); + } + + private void OnShutdown() + { + Console.WriteLine("Server is shutting down. Going to dispose Hangfire"); + //this code is called when the application stops + //TaskScheduler.Client.Dispose(); + System.Threading.Thread.Sleep(1000); } } } From bb0a59448f3502ca6125df4f657280d0acc8cac1 Mon Sep 17 00:00:00 2001 From: Joseph Milazzo Date: Mon, 15 Mar 2021 13:49:06 -0500 Subject: [PATCH 2/9] Lots of Hangfire attempts to solve the lock issue. Not sure how to fix it. Added some APIs for streams. --- API/Extensions/ServiceCollectionExtensions.cs | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 API/Extensions/ServiceCollectionExtensions.cs diff --git a/API/Extensions/ServiceCollectionExtensions.cs b/API/Extensions/ServiceCollectionExtensions.cs new file mode 100644 index 000000000..aa4c6f0f7 --- /dev/null +++ b/API/Extensions/ServiceCollectionExtensions.cs @@ -0,0 +1,7 @@ +namespace API.Extensions +{ + public class ServiceCollectionExtensions + { + + } +} \ No newline at end of file From fad4ca4414006c4d97744d7b633298de730794e3 Mon Sep 17 00:00:00 2001 From: Joseph Milazzo Date: Mon, 15 Mar 2021 13:49:13 -0500 Subject: [PATCH 3/9] Lots of Hangfire attempts to solve the lock issue. Not sure how to fix it. Added some APIs for streams. --- API/Controllers/SeriesController.cs | 13 +++++- API/Data/SeriesRepository.cs | 44 +++++++++++++++++-- API/Extensions/ServiceCollectionExtensions.cs | 11 +++-- API/Interfaces/ISeriesRepository.cs | 3 +- API/Middleware/ExceptionMiddleware.cs | 1 + API/Program.cs | 11 +++++ API/Services/TaskScheduler.cs | 10 ++--- API/Services/Tasks/ScannerService.cs | 4 +- API/Services/WarmupServiceStartupTask.cs | 11 +++-- API/Startup.cs | 9 ++-- 10 files changed, 89 insertions(+), 28 deletions(-) diff --git a/API/Controllers/SeriesController.cs b/API/Controllers/SeriesController.cs index d4bd4c5a9..611fff59b 100644 --- a/API/Controllers/SeriesController.cs +++ b/API/Controllers/SeriesController.cs @@ -128,9 +128,18 @@ namespace API.Controllers } [HttpGet("recently-added")] - public async Task>> GetRecentlyAdded(int libraryId = 0) + public async Task>> GetRecentlyAdded(int libraryId = 0, int limit = 20) { - return Ok(await _unitOfWork.SeriesRepository.GetRecentlyAdded(libraryId)); + return Ok(await _unitOfWork.SeriesRepository.GetRecentlyAdded(libraryId, limit)); } + + [HttpGet("in-progress")] + public async Task>> GetInProgress(int libraryId = 0, int limit = 20) + { + var user = await _unitOfWork.UserRepository.GetUserByUsernameAsync(User.GetUsername()); + return Ok(await _unitOfWork.SeriesRepository.GetInProgress(user.Id, libraryId, limit)); + } + + } } \ No newline at end of file diff --git a/API/Data/SeriesRepository.cs b/API/Data/SeriesRepository.cs index e4bda9997..29e3aaadb 100644 --- a/API/Data/SeriesRepository.cs +++ b/API/Data/SeriesRepository.cs @@ -11,6 +11,7 @@ using API.Interfaces; using AutoMapper; using AutoMapper.QueryableExtensions; using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Internal; using Microsoft.Extensions.Logging; namespace API.Data @@ -282,20 +283,55 @@ namespace API.Data /// Returns a list of Series that were added within 2 weeks. /// /// Library to restrict to, if 0, will apply to all libraries + /// /// - public async Task> GetRecentlyAdded(int libraryId) + public async Task> GetRecentlyAdded(int libraryId, int limit) { - // && (libraryId <= 0 || s.LibraryId == libraryId) + // TODO: Remove 2 week condition var twoWeeksAgo = DateTime.Today.Subtract(TimeSpan.FromDays(14)); _logger.LogDebug("2 weeks from today is: {Date}", twoWeeksAgo); return await _context.Series - .Where(s => s.Created > twoWeeksAgo) - .Take(20) + .Where(s => s.Created > twoWeeksAgo && (libraryId <= 0 || s.LibraryId == libraryId)) + .Take(limit) + .OrderBy(s => s.Created) .AsNoTracking() .ProjectTo(_mapper.ConfigurationProvider) .ToListAsync(); } + + public async Task> GetInProgress(int userId, int libraryId, int limit) + { + //&& (libraryId <= 0 || s.Series.LibraryId == libraryId) + var twoWeeksAgo = DateTime.Today.Subtract(TimeSpan.FromDays(14)); + _logger.LogInformation("GetInProgress"); + _logger.LogDebug("2 weeks from today is: {Date}", twoWeeksAgo); + // var series = await _context.Series + // .Join(_context.AppUserProgresses, s => s.Id, progress => progress.SeriesId, (s, progress) => new + // { + // Series = s, + // Progress = progress + // }) + // .DefaultIfEmpty() + // .Where(s => s.Series.Created > twoWeeksAgo + // && s.Progress.AppUserId == userId + // && s.Progress.PagesRead > s.Series.Pages) + // .Take(limit) + // .OrderBy(s => s.Series.Created) + // .AsNoTracking() + // .Select(s => s.Series) + // .ProjectTo(_mapper.ConfigurationProvider) + // .ToListAsync(); + var series = await _context.Series + .Where(s => s.Created > twoWeeksAgo) // && (libraryId <= 0 || s.LibraryId == libraryId) + .AsNoTracking() + .ProjectTo(_mapper.ConfigurationProvider) + .ToListAsync(); + + await AddSeriesModifiers(userId, series); + + return series.Where(s => s.PagesRead > 0).Take(limit).ToList(); + } public async Task> GetSeriesStream(int userId) diff --git a/API/Extensions/ServiceCollectionExtensions.cs b/API/Extensions/ServiceCollectionExtensions.cs index aa4c6f0f7..d3cae4191 100644 --- a/API/Extensions/ServiceCollectionExtensions.cs +++ b/API/Extensions/ServiceCollectionExtensions.cs @@ -1,7 +1,12 @@ -namespace API.Extensions +using API.Interfaces.Services; +using Microsoft.Extensions.DependencyInjection; + +namespace API.Extensions { - public class ServiceCollectionExtensions + public static class ServiceCollectionExtensions { - + public static IServiceCollection AddStartupTask(this IServiceCollection services) + where T : class, IStartupTask + => services.AddTransient(); } } \ No newline at end of file diff --git a/API/Interfaces/ISeriesRepository.cs b/API/Interfaces/ISeriesRepository.cs index 03cc758b1..647469e69 100644 --- a/API/Interfaces/ISeriesRepository.cs +++ b/API/Interfaces/ISeriesRepository.cs @@ -55,6 +55,7 @@ namespace API.Interfaces Task GetVolumeCoverImageAsync(int volumeId); Task GetSeriesCoverImageAsync(int seriesId); - Task> GetRecentlyAdded(int libraryId); + Task> GetInProgress(int userId, int libraryId, int limit); + Task> GetRecentlyAdded(int libraryId, int limit); } } \ No newline at end of file diff --git a/API/Middleware/ExceptionMiddleware.cs b/API/Middleware/ExceptionMiddleware.cs index 5c168cf3d..8c76b8644 100644 --- a/API/Middleware/ExceptionMiddleware.cs +++ b/API/Middleware/ExceptionMiddleware.cs @@ -25,6 +25,7 @@ namespace API.Middleware public async Task InvokeAsync(HttpContext context) { + // BUG: I think Hangfire timeouts are triggering the middleware to hijack an API call try { await _next(context); // downstream middlewares or http call diff --git a/API/Program.cs b/API/Program.cs index 7bd6ef289..c38736407 100644 --- a/API/Program.cs +++ b/API/Program.cs @@ -3,6 +3,8 @@ using System.Threading.Tasks; using API.Data; using API.Entities; using API.Interfaces; +using API.Interfaces.Services; +using API.Services; using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Identity; using Microsoft.EntityFrameworkCore; @@ -40,6 +42,15 @@ namespace API var logger = services.GetRequiredService < ILogger>(); logger.LogError(ex, "An error occurred during migration"); } + + // Load all tasks from DI (TODO: This is not working) + var startupTasks = host.Services.GetServices(); + + // Execute all the tasks + foreach (var startupTask in startupTasks) + { + await startupTask.ExecuteAsync(); + } await host.RunAsync(); } diff --git a/API/Services/TaskScheduler.cs b/API/Services/TaskScheduler.cs index 758d653a2..70bca9121 100644 --- a/API/Services/TaskScheduler.cs +++ b/API/Services/TaskScheduler.cs @@ -21,11 +21,11 @@ namespace API.Services private readonly ICleanupService _cleanupService; private readonly IDirectoryService _directoryService; - public static BackgroundJobServer Client => new BackgroundJobServer(); - // new BackgroundJobServerOptions() - // { - // WorkerCount = 1 - // } + public static BackgroundJobServer Client => new BackgroundJobServer(new BackgroundJobServerOptions() + { + WorkerCount = 1 + }); + public TaskScheduler(ICacheService cacheService, ILogger logger, IScannerService scannerService, IUnitOfWork unitOfWork, IMetadataService metadataService, IBackupService backupService, ICleanupService cleanupService, diff --git a/API/Services/Tasks/ScannerService.cs b/API/Services/Tasks/ScannerService.cs index af7b6eca8..61842eac8 100644 --- a/API/Services/Tasks/ScannerService.cs +++ b/API/Services/Tasks/ScannerService.cs @@ -33,7 +33,7 @@ namespace API.Services.Tasks _metadataService = metadataService; } - [DisableConcurrentExecution(timeoutInSeconds: 5)] + //[DisableConcurrentExecution(timeoutInSeconds: 5)] [AutomaticRetry(Attempts = 0, LogEvents = false, OnAttemptsExceeded = AttemptsExceededAction.Delete)] public void ScanLibraries() { @@ -64,7 +64,7 @@ namespace API.Services.Tasks _scannedSeries = null; } - [DisableConcurrentExecution(5)] + //[DisableConcurrentExecution(5)] [AutomaticRetry(Attempts = 0, LogEvents = false, OnAttemptsExceeded = AttemptsExceededAction.Delete)] public void ScanLibrary(int libraryId, bool forceUpdate) { diff --git a/API/Services/WarmupServiceStartupTask.cs b/API/Services/WarmupServiceStartupTask.cs index fd9b1745b..36463451a 100644 --- a/API/Services/WarmupServiceStartupTask.cs +++ b/API/Services/WarmupServiceStartupTask.cs @@ -18,14 +18,13 @@ namespace API.Services _provider = provider; } - public Task ExecuteAsync(CancellationToken cancellationToken) + public Task ExecuteAsync(CancellationToken cancellationToken = default) { - using (var scope = _provider.CreateScope()) + using var scope = _provider.CreateScope(); + foreach (var singleton in GetServices(_services)) { - foreach (var singleton in GetServices(_services)) - { - scope.ServiceProvider.GetServices(singleton); - } + Console.WriteLine("DI preloading of " + singleton.FullName); + scope.ServiceProvider.GetServices(singleton); } return Task.CompletedTask; diff --git a/API/Startup.cs b/API/Startup.cs index d53e346d6..55b7f6f9e 100644 --- a/API/Startup.cs +++ b/API/Startup.cs @@ -64,10 +64,9 @@ namespace API services.AddResponseCaching(); - services - .AddStartupTask() - .TryAddSingleton(services); - + // services + // .AddStartupTask() + // //.TryAddSingleton(services); } // This method gets called by the runtime. Use this method to configure the HTTP request pipeline. @@ -134,7 +133,7 @@ namespace API { Console.WriteLine("Server is shutting down. Going to dispose Hangfire"); //this code is called when the application stops - //TaskScheduler.Client.Dispose(); + TaskScheduler.Client.Dispose(); System.Threading.Thread.Sleep(1000); } } From 55cd0c5fe54e959d3fc9e081624e7ab9808f2745 Mon Sep 17 00:00:00 2001 From: Joseph Milazzo Date: Mon, 15 Mar 2021 17:10:09 -0500 Subject: [PATCH 4/9] For the time out on tasks, I'm going to only use Hangfire scheduled tasks on prod. --- API/Services/TaskScheduler.cs | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/API/Services/TaskScheduler.cs b/API/Services/TaskScheduler.cs index 70bca9121..28241aaea 100644 --- a/API/Services/TaskScheduler.cs +++ b/API/Services/TaskScheduler.cs @@ -6,6 +6,8 @@ using API.Helpers.Converters; using API.Interfaces; using API.Interfaces.Services; using Hangfire; +using Microsoft.AspNetCore.Hosting; +using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; namespace API.Services @@ -29,7 +31,7 @@ namespace API.Services public TaskScheduler(ICacheService cacheService, ILogger logger, IScannerService scannerService, IUnitOfWork unitOfWork, IMetadataService metadataService, IBackupService backupService, ICleanupService cleanupService, - IDirectoryService directoryService) + IDirectoryService directoryService, IWebHostEnvironment env) { _cacheService = cacheService; _logger = logger; @@ -39,8 +41,18 @@ namespace API.Services _backupService = backupService; _cleanupService = cleanupService; _directoryService = directoryService; + + if (!env.IsDevelopment()) + { + ScheduleTasks(); + } + else + { + RecurringJob.RemoveIfExists("scan-libraries"); + RecurringJob.RemoveIfExists("backup"); + RecurringJob.RemoveIfExists("cleanup"); + } - ScheduleTasks(); } public void ScheduleTasks() @@ -78,8 +90,7 @@ namespace API.Services _logger.LogInformation("Enqueuing library scan for: {LibraryId}", libraryId); BackgroundJob.Enqueue(() => _scannerService.ScanLibrary(libraryId, forceUpdate)); - //BackgroundJob.Enqueue(() => _cleanupService.Cleanup()); // When we do a scan, force cache to re-unpack in case page numbers change - RecurringJob.Trigger("cleanup"); // TODO: Alternate way to trigger jobs. Test this out and see if we should switch. + BackgroundJob.Enqueue(() => _cleanupService.Cleanup()); // When we do a scan, force cache to re-unpack in case page numbers change } public void CleanupChapters(int[] chapterIds) From 0a85555f383e7103a47e9fdb3a36a3c87fd0488c Mon Sep 17 00:00:00 2001 From: Joseph Milazzo Date: Wed, 17 Mar 2021 14:29:50 -0500 Subject: [PATCH 5/9] Temp commit to record attempts. Stream APIs are implemented and working. Added some new test cases based on deployed Kavita server testing. --- API.Tests/ParserTest.cs | 7 +- API/API.csproj | 5 + API/Comparators/ChapterSortComparer.cs | 21 +++- API/Controllers/SeriesController.cs | 9 ++ API/DTOs/InProgressChapterDto.cs | 23 +++++ API/DTOs/SeriesDto.cs | 2 + API/Data/SeriesRepository.cs | 97 ++++++------------- API/Data/UnitOfWork.cs | 10 +- API/Data/VolumeRepository.cs | 97 ++++++++++++++++++- .../ApplicationServiceExtensions.cs | 9 +- API/Extensions/EnumerableExtensions.cs | 21 ++++ API/Helpers/AutoMapperProfiles.cs | 1 + API/Interfaces/IVolumeRepository.cs | 1 + API/Program.cs | 4 +- API/Services/Tasks/ScannerService.cs | 2 +- API/Startup.cs | 28 +++++- 16 files changed, 247 insertions(+), 90 deletions(-) create mode 100644 API/DTOs/InProgressChapterDto.cs create mode 100644 API/Extensions/EnumerableExtensions.cs diff --git a/API.Tests/ParserTest.cs b/API.Tests/ParserTest.cs index 6c1b5cfd5..0d13f7628 100644 --- a/API.Tests/ParserTest.cs +++ b/API.Tests/ParserTest.cs @@ -49,6 +49,7 @@ namespace API.Tests [InlineData("Kedouin Makoto - Corpse Party Musume, Chapter 12", "0")] [InlineData("VanDread-v01-c001[MD].zip", "1")] [InlineData("Ichiban_Ushiro_no_Daimaou_v04_ch27_[VISCANS].zip", "4")] + [InlineData("Mob Psycho 100 v02 (2019) (Digital) (Shizu).cbz", "2")] public void ParseVolumeTest(string filename, string expected) { Assert.Equal(expected, ParseVolume(filename)); @@ -100,6 +101,9 @@ namespace API.Tests [InlineData("Kedouin Makoto - Corpse Party Musume, Chapter 09", "Kedouin Makoto - Corpse Party Musume")] [InlineData("Goblin Slayer Side Story - Year One 025.5", "Goblin Slayer Side Story - Year One")] [InlineData("Goblin Slayer - Brand New Day 006.5 (2019) (Digital) (danke-Empire)", "Goblin Slayer - Brand New Day")] + [InlineData("Yumekui-Merry_DKThias_Chapter11v2.zip", "Yumekui-Merry")] + [InlineData("Yumekui-Merry DKThiasScanlations Chapter51v2", "Yumekui-Merry")] + [InlineData("Yumekui-Merry_DKThiasScanlations&RenzokuseiScans_Chapter61", "Yumekui-Merry")] public void ParseSeriesTest(string filename, string expected) { Assert.Equal(expected, ParseSeries(filename)); @@ -130,6 +134,7 @@ namespace API.Tests [InlineData("Yumekui_Merry_v01_c01[Bakayarou-Kuu].rar", "1")] [InlineData("Yumekui-Merry_DKThias_Chapter11v2.zip", "11")] [InlineData("Yumekui-Merry DKThiasScanlations Chapter51v2", "51")] + [InlineData("Yumekui-Merry_DKThiasScanlations&RenzokuseiScans_Chapter61", "61")] [InlineData("Goblin Slayer Side Story - Year One 017.5", "17.5")] [InlineData("Beelzebub_53[KSH].zip", "53")] [InlineData("Black Bullet - v4 c20.5 [batoto]", "20.5")] @@ -139,7 +144,7 @@ namespace API.Tests [InlineData("Vol 1", "0")] [InlineData("VanDread-v01-c001[MD].zip", "1")] [InlineData("Goblin Slayer Side Story - Year One 025.5", "25.5")] - //[InlineData("[Tempus Edax Rerum] Epigraph of the Closed Curve - Chapter 6.zip", "6")] + [InlineData("Mob Psycho 100 v02 (2019) (Digital) (Shizu).cbz", "0")] public void ParseChaptersTest(string filename, string expected) { Assert.Equal(expected, ParseChapter(filename)); diff --git a/API/API.csproj b/API/API.csproj index 008455438..0f6d90fb6 100644 --- a/API/API.csproj +++ b/API/API.csproj @@ -6,12 +6,17 @@ true + + false + + + diff --git a/API/Comparators/ChapterSortComparer.cs b/API/Comparators/ChapterSortComparer.cs index 725622bec..1798afe7e 100644 --- a/API/Comparators/ChapterSortComparer.cs +++ b/API/Comparators/ChapterSortComparer.cs @@ -2,15 +2,26 @@ namespace API.Comparators { - public class ChapterSortComparer : IComparer + public class ChapterSortComparer : IComparer { - public int Compare(int x, int y) + // public int Compare(int x, int y) + // { + // if (x == 0 && y == 0) return 0; + // // if x is 0, it comes second + // if (x == 0) return 1; + // // if y is 0, it comes second + // if (y == 0) return -1; + // + // return x.CompareTo(y); + // } + + public int Compare(float x, float y) { - if (x == 0 && y == 0) return 0; + if (x == 0.0 && y == 0.0) return 0; // if x is 0, it comes second - if (x == 0) return 1; + if (x == 0.0) return 1; // if y is 0, it comes second - if (y == 0) return -1; + if (y == 0.0) return -1; return x.CompareTo(y); } diff --git a/API/Controllers/SeriesController.cs b/API/Controllers/SeriesController.cs index 611fff59b..c1e4c9b73 100644 --- a/API/Controllers/SeriesController.cs +++ b/API/Controllers/SeriesController.cs @@ -140,6 +140,15 @@ namespace API.Controllers return Ok(await _unitOfWork.SeriesRepository.GetInProgress(user.Id, libraryId, limit)); } + [HttpGet("continue-reading")] + public async Task>> GetContinueReading(int libraryId = 0, int limit = 20) + { + var user = await _unitOfWork.UserRepository.GetUserByUsernameAsync(User.GetUsername()); + return Ok(await _unitOfWork.VolumeRepository.GetContinueReading(user.Id, libraryId, limit)); + } + + + } } \ No newline at end of file diff --git a/API/DTOs/InProgressChapterDto.cs b/API/DTOs/InProgressChapterDto.cs new file mode 100644 index 000000000..f0a0096ef --- /dev/null +++ b/API/DTOs/InProgressChapterDto.cs @@ -0,0 +1,23 @@ +namespace API.DTOs +{ + public class InProgressChapterDto + { + public int Id { get; init; } + /// + /// Range of chapters. Chapter 2-4 -> "2-4". Chapter 2 -> "2". + /// + public string Range { get; init; } + /// + /// Smallest number of the Range. + /// + public string Number { get; init; } + /// + /// Total number of pages in all MangaFiles + /// + public int Pages { get; init; } + public int SeriesId { get; init; } + public int LibraryId { get; init; } + public string SeriesName { get; init; } + + } +} \ No newline at end of file diff --git a/API/DTOs/SeriesDto.cs b/API/DTOs/SeriesDto.cs index 593870309..b3057baac 100644 --- a/API/DTOs/SeriesDto.cs +++ b/API/DTOs/SeriesDto.cs @@ -21,5 +21,7 @@ /// Review from logged in user. Calculated at API-time. /// public string UserReview { get; set; } + + public int LibraryId { get; set; } } } \ No newline at end of file diff --git a/API/Data/SeriesRepository.cs b/API/Data/SeriesRepository.cs index 29e3aaadb..8c1949edc 100644 --- a/API/Data/SeriesRepository.cs +++ b/API/Data/SeriesRepository.cs @@ -280,87 +280,52 @@ namespace API.Data } /// - /// Returns a list of Series that were added within 2 weeks. + /// Returns a list of Series that were added, ordered by Created desc /// /// Library to restrict to, if 0, will apply to all libraries - /// + /// How many series to pick. /// public async Task> GetRecentlyAdded(int libraryId, int limit) { - // TODO: Remove 2 week condition - var twoWeeksAgo = DateTime.Today.Subtract(TimeSpan.FromDays(14)); - _logger.LogDebug("2 weeks from today is: {Date}", twoWeeksAgo); return await _context.Series - .Where(s => s.Created > twoWeeksAgo && (libraryId <= 0 || s.LibraryId == libraryId)) + .Where(s => (libraryId <= 0 || s.LibraryId == libraryId)) .Take(limit) - .OrderBy(s => s.Created) + .OrderByDescending(s => s.Created) .AsNoTracking() .ProjectTo(_mapper.ConfigurationProvider) .ToListAsync(); - } + /// + /// + /// + /// + /// + /// + /// public async Task> GetInProgress(int userId, int libraryId, int limit) { - //&& (libraryId <= 0 || s.Series.LibraryId == libraryId) - var twoWeeksAgo = DateTime.Today.Subtract(TimeSpan.FromDays(14)); - _logger.LogInformation("GetInProgress"); - _logger.LogDebug("2 weeks from today is: {Date}", twoWeeksAgo); - // var series = await _context.Series - // .Join(_context.AppUserProgresses, s => s.Id, progress => progress.SeriesId, (s, progress) => new - // { - // Series = s, - // Progress = progress - // }) - // .DefaultIfEmpty() - // .Where(s => s.Series.Created > twoWeeksAgo - // && s.Progress.AppUserId == userId - // && s.Progress.PagesRead > s.Series.Pages) - // .Take(limit) - // .OrderBy(s => s.Series.Created) - // .AsNoTracking() - // .Select(s => s.Series) - // .ProjectTo(_mapper.ConfigurationProvider) - // .ToListAsync(); + //var twoWeeksAgo = DateTime.Today.Subtract(TimeSpan.FromDays(14)); // TODO: Think about moving this to a setting var series = await _context.Series - .Where(s => s.Created > twoWeeksAgo) // && (libraryId <= 0 || s.LibraryId == libraryId) - .AsNoTracking() - .ProjectTo(_mapper.ConfigurationProvider) - .ToListAsync(); - - await AddSeriesModifiers(userId, series); - - return series.Where(s => s.PagesRead > 0).Take(limit).ToList(); - } - - - public async Task> GetSeriesStream(int userId) - { - // Testing out In Progress to figure out how to design generalized solution - var userProgress = await _context.AppUserProgresses - .Where(p => p.AppUserId == userId && p.PagesRead > 0) - .AsNoTracking() - .ToListAsync(); - if (!userProgress.Any()) return new SeriesDto[] {}; - - var seriesIds = userProgress.Select(p => p.SeriesId).ToList(); - /* - *select P.*, S.Name, S.Pages from AppUserProgresses AS P - LEFT join Series as "S" on s.Id = P.SeriesId - where AppUserId = 1 AND P.PagesRead > 0 AND P.PagesRead < S.Pages - * - */ - - - - - // var series = await _context.Series - // .Where(s => seriesIds.Contains(s.Id) && s.Pages) // I need a join - - - - - return new SeriesDto[] {}; + .Join(_context.AppUserProgresses, s => s.Id, progress => progress.SeriesId, (s, progress) => new + { + Series = s, + progress.PagesRead, + progress.AppUserId, + progress.LastModified + }) + .Where(s => s.AppUserId == userId + && s.PagesRead > 0 + && s.PagesRead < s.Series.Pages + && (libraryId <= 0 || s.Series.LibraryId == libraryId) ) + .Take(limit) + .OrderByDescending(s => s.LastModified) + .AsNoTracking() + .Select(s => s.Series) + .Distinct() + .ProjectTo(_mapper.ConfigurationProvider) + .ToListAsync(); + return series; } } } \ No newline at end of file diff --git a/API/Data/UnitOfWork.cs b/API/Data/UnitOfWork.cs index 4b8ffac81..aa3c9ab5f 100644 --- a/API/Data/UnitOfWork.cs +++ b/API/Data/UnitOfWork.cs @@ -12,21 +12,21 @@ namespace API.Data private readonly DataContext _context; private readonly IMapper _mapper; private readonly UserManager _userManager; - private readonly ILogger _seriesLogger; + private readonly ILogger _logger; - public UnitOfWork(DataContext context, IMapper mapper, UserManager userManager, ILogger seriesLogger) + public UnitOfWork(DataContext context, IMapper mapper, UserManager userManager, ILogger logger) { _context = context; _mapper = mapper; _userManager = userManager; - _seriesLogger = seriesLogger; + _logger = logger; } - public ISeriesRepository SeriesRepository => new SeriesRepository(_context, _mapper, _seriesLogger); + public ISeriesRepository SeriesRepository => new SeriesRepository(_context, _mapper, _logger); public IUserRepository UserRepository => new UserRepository(_context, _userManager); public ILibraryRepository LibraryRepository => new LibraryRepository(_context, _mapper); - public IVolumeRepository VolumeRepository => new VolumeRepository(_context, _mapper); + public IVolumeRepository VolumeRepository => new VolumeRepository(_context, _mapper, _logger); public ISettingsRepository SettingsRepository => new SettingsRepository(_context, _mapper); diff --git a/API/Data/VolumeRepository.cs b/API/Data/VolumeRepository.cs index 6b9e541ea..4461c5dfb 100644 --- a/API/Data/VolumeRepository.cs +++ b/API/Data/VolumeRepository.cs @@ -1,12 +1,15 @@ using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; +using API.Comparators; using API.DTOs; using API.Entities; +using API.Extensions; using API.Interfaces; using AutoMapper; using AutoMapper.QueryableExtensions; using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; namespace API.Data { @@ -14,11 +17,13 @@ namespace API.Data { private readonly DataContext _context; private readonly IMapper _mapper; + private readonly ILogger _logger; - public VolumeRepository(DataContext context, IMapper mapper) + public VolumeRepository(DataContext context, IMapper mapper, ILogger logger) { _context = context; _mapper = mapper; + _logger = logger; } public void Update(Volume volume) @@ -84,5 +89,95 @@ namespace API.Data .AsNoTracking() .ToListAsync(); } + + /// + /// Gets the first (ordered) volume/chapter in a series where the user has progress on it. Only completed volumes/chapters, next entity shouldn't + /// have any read progress on it. + /// + /// + /// + /// + /// + public async Task> GetContinueReading(int userId, int libraryId, int limit) + { + _logger.LogInformation("Get Continue Reading"); + var progress = await _context.Chapter + .Join(_context.AppUserProgresses, chapter => chapter.Id, progress => progress.ChapterId, + (chapter, progress) => + new + { + Chapter = chapter, + Progress = progress + }) + .Join(_context.Series, arg => arg.Progress.SeriesId, series => series.Id, (arg, series) => + new + { + arg.Chapter, + arg.Progress, + Series = series + }) + .AsNoTracking() + .Where(arg => arg.Progress.AppUserId == userId + && arg.Progress.PagesRead < arg.Chapter.Pages) + .OrderByDescending(d => d.Progress.LastModified) + .Take(limit) + .ToListAsync(); + + return progress + .OrderBy(c => float.Parse(c.Chapter.Number), new ChapterSortComparer()) + .DistinctBy(p => p.Series.Id) + .Select(arg => new InProgressChapterDto() + { + Id = arg.Chapter.Id, + Number = arg.Chapter.Number, + Range = arg.Chapter.Range, + SeriesId = arg.Progress.SeriesId, + SeriesName = arg.Series.Name, + LibraryId = arg.Series.LibraryId, + Pages = arg.Chapter.Pages, + }); + + // var chapters = await _context.Chapter + // .Join(_context.AppUserProgresses, chapter => chapter.Id, progress => progress.ChapterId, (chapter, progress) => + // new + // { + // Chapter = chapter, + // Progress = progress + // }) + // .Where(arg => arg.Progress.AppUserId == userId && arg.Progress.PagesRead < arg.Chapter.Pages) + // .Join(_context.Series, arg => arg.Progress.SeriesId, series => series.Id, (arg, series) => + // new + // { + // arg.Chapter, + // arg.Progress, + // Series = series + // }) + // .AsNoTracking() + // //.OrderBy(s => s.Chapter.Number) + // .GroupBy(p => p.Series.Id) + // .Select(g => g.FirstOrDefault()) + // .Select(arg => new InProgressChapterDto() + // { + // Id = arg.Chapter.Id, + // Number = arg.Chapter.Number, + // Range = arg.Chapter.Range, + // SeriesId = arg.Progress.SeriesId, + // SeriesName = arg.Series.Name, + // LibraryId = arg.Series.LibraryId, + // Pages = arg.Chapter.Pages, + // }) + // + // //.OrderBy(c => float.Parse(c.Number)) //can't convert to SQL + // + // .ToListAsync(); + // + // + // return chapters; + + + // return chapters + // .OrderBy(c => float.Parse(c.Number), new ChapterSortComparer()) + // .DistinctBy(c => c.SeriesName); + } } } \ No newline at end of file diff --git a/API/Extensions/ApplicationServiceExtensions.cs b/API/Extensions/ApplicationServiceExtensions.cs index f8b10a442..6c5d4ec34 100644 --- a/API/Extensions/ApplicationServiceExtensions.cs +++ b/API/Extensions/ApplicationServiceExtensions.cs @@ -7,6 +7,7 @@ using API.Services.Tasks; using AutoMapper; using Hangfire; using Hangfire.LiteDB; +using Hangfire.MemoryStorage; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; @@ -45,15 +46,7 @@ namespace API.Extensions var loggingSection = config.GetSection("Logging"); loggingBuilder.AddFile(loggingSection); }); - - services.AddHangfire(configuration => configuration - .UseSimpleAssemblyNameTypeSerializer() - .UseRecommendedSerializerSettings() - .UseLiteDbStorage()); - // Add the processing server as IHostedService - services.AddHangfireServer(); - return services; } diff --git a/API/Extensions/EnumerableExtensions.cs b/API/Extensions/EnumerableExtensions.cs new file mode 100644 index 000000000..b8293436e --- /dev/null +++ b/API/Extensions/EnumerableExtensions.cs @@ -0,0 +1,21 @@ +using System; +using System.Collections.Generic; + +namespace API.Extensions +{ + public static class EnumerableExtensions + { + public static IEnumerable DistinctBy + (this IEnumerable source, Func keySelector) + { + var seenKeys = new HashSet(); + foreach (var element in source) + { + if (seenKeys.Add(keySelector(element))) + { + yield return element; + } + } + } + } +} \ No newline at end of file diff --git a/API/Helpers/AutoMapperProfiles.cs b/API/Helpers/AutoMapperProfiles.cs index f5d670b59..85b1aaf9a 100644 --- a/API/Helpers/AutoMapperProfiles.cs +++ b/API/Helpers/AutoMapperProfiles.cs @@ -20,6 +20,7 @@ namespace API.Helpers CreateMap(); CreateMap(); + //CreateMap(); CreateMap(); diff --git a/API/Interfaces/IVolumeRepository.cs b/API/Interfaces/IVolumeRepository.cs index faf18abb8..bec554fde 100644 --- a/API/Interfaces/IVolumeRepository.cs +++ b/API/Interfaces/IVolumeRepository.cs @@ -13,5 +13,6 @@ namespace API.Interfaces Task> GetFilesForChapter(int chapterId); Task> GetChaptersAsync(int volumeId); Task GetChapterCoverImageAsync(int chapterId); + Task> GetContinueReading(int userId, int libraryId, int limit); } } \ No newline at end of file diff --git a/API/Program.cs b/API/Program.cs index c38736407..e3d8cae60 100644 --- a/API/Program.cs +++ b/API/Program.cs @@ -43,9 +43,9 @@ namespace API logger.LogError(ex, "An error occurred during migration"); } - // Load all tasks from DI (TODO: This is not working) + // Load all tasks from DI (TODO: This is not working - WarmupServicesStartupTask is Null) var startupTasks = host.Services.GetServices(); - + // Execute all the tasks foreach (var startupTask in startupTasks) { diff --git a/API/Services/Tasks/ScannerService.cs b/API/Services/Tasks/ScannerService.cs index 61842eac8..e63144113 100644 --- a/API/Services/Tasks/ScannerService.cs +++ b/API/Services/Tasks/ScannerService.cs @@ -66,7 +66,7 @@ namespace API.Services.Tasks //[DisableConcurrentExecution(5)] [AutomaticRetry(Attempts = 0, LogEvents = false, OnAttemptsExceeded = AttemptsExceededAction.Delete)] - public void ScanLibrary(int libraryId, bool forceUpdate) + public async void ScanLibrary(int libraryId, bool forceUpdate) { _forceUpdate = forceUpdate; var sw = Stopwatch.StartNew(); diff --git a/API/Startup.cs b/API/Startup.cs index 55b7f6f9e..2174e5ed2 100644 --- a/API/Startup.cs +++ b/API/Startup.cs @@ -6,6 +6,8 @@ using API.Extensions; using API.Middleware; using API.Services; using Hangfire; +using Hangfire.LiteDB; +using Hangfire.MemoryStorage; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Http; @@ -25,10 +27,12 @@ namespace API public class Startup { private readonly IConfiguration _config; + private readonly IWebHostEnvironment _env; - public Startup(IConfiguration config) + public Startup(IConfiguration config, IWebHostEnvironment env) { _config = config; + _env = env; } // This method gets called by the runtime. Use this method to add services to the container. @@ -63,6 +67,24 @@ namespace API services.AddResponseCaching(); + if (_env.IsDevelopment()) + { + services.AddHangfire(configuration => configuration + .UseSimpleAssemblyNameTypeSerializer() + .UseRecommendedSerializerSettings() + .UseMemoryStorage()); + } + else + { + services.AddHangfire(configuration => configuration + .UseSimpleAssemblyNameTypeSerializer() + .UseRecommendedSerializerSettings() + .UseLiteDbStorage()); + } + + // Add the processing server as IHostedService + services.AddHangfireServer(); + // services // .AddStartupTask() @@ -127,6 +149,10 @@ namespace API }); applicationLifetime.ApplicationStopping.Register(OnShutdown); + applicationLifetime.ApplicationStarted.Register(() => + { + Console.WriteLine("Kavita - v0.3"); + }); } private void OnShutdown() From e60d84a2c81d476cf43a70c342f1ebfc257b2cf3 Mon Sep 17 00:00:00 2001 From: Joseph Milazzo Date: Wed, 17 Mar 2021 14:31:17 -0500 Subject: [PATCH 6/9] Cleanup --- API/Data/VolumeRepository.cs | 48 ++----------------------------- API/Helpers/AutoMapperProfiles.cs | 3 +- 2 files changed, 4 insertions(+), 47 deletions(-) diff --git a/API/Data/VolumeRepository.cs b/API/Data/VolumeRepository.cs index 4461c5dfb..35119efa8 100644 --- a/API/Data/VolumeRepository.cs +++ b/API/Data/VolumeRepository.cs @@ -101,8 +101,8 @@ namespace API.Data public async Task> GetContinueReading(int userId, int libraryId, int limit) { _logger.LogInformation("Get Continue Reading"); - var progress = await _context.Chapter - .Join(_context.AppUserProgresses, chapter => chapter.Id, progress => progress.ChapterId, + var chapters = await _context.Chapter + .Join(_context.AppUserProgresses, c => c.Id, p => p.ChapterId, (chapter, progress) => new { @@ -123,7 +123,7 @@ namespace API.Data .Take(limit) .ToListAsync(); - return progress + return chapters .OrderBy(c => float.Parse(c.Chapter.Number), new ChapterSortComparer()) .DistinctBy(p => p.Series.Id) .Select(arg => new InProgressChapterDto() @@ -136,48 +136,6 @@ namespace API.Data LibraryId = arg.Series.LibraryId, Pages = arg.Chapter.Pages, }); - - // var chapters = await _context.Chapter - // .Join(_context.AppUserProgresses, chapter => chapter.Id, progress => progress.ChapterId, (chapter, progress) => - // new - // { - // Chapter = chapter, - // Progress = progress - // }) - // .Where(arg => arg.Progress.AppUserId == userId && arg.Progress.PagesRead < arg.Chapter.Pages) - // .Join(_context.Series, arg => arg.Progress.SeriesId, series => series.Id, (arg, series) => - // new - // { - // arg.Chapter, - // arg.Progress, - // Series = series - // }) - // .AsNoTracking() - // //.OrderBy(s => s.Chapter.Number) - // .GroupBy(p => p.Series.Id) - // .Select(g => g.FirstOrDefault()) - // .Select(arg => new InProgressChapterDto() - // { - // Id = arg.Chapter.Id, - // Number = arg.Chapter.Number, - // Range = arg.Chapter.Range, - // SeriesId = arg.Progress.SeriesId, - // SeriesName = arg.Series.Name, - // LibraryId = arg.Series.LibraryId, - // Pages = arg.Chapter.Pages, - // }) - // - // //.OrderBy(c => float.Parse(c.Number)) //can't convert to SQL - // - // .ToListAsync(); - // - // - // return chapters; - - - // return chapters - // .OrderBy(c => float.Parse(c.Number), new ChapterSortComparer()) - // .DistinctBy(c => c.SeriesName); } } } \ No newline at end of file diff --git a/API/Helpers/AutoMapperProfiles.cs b/API/Helpers/AutoMapperProfiles.cs index 85b1aaf9a..328a27ade 100644 --- a/API/Helpers/AutoMapperProfiles.cs +++ b/API/Helpers/AutoMapperProfiles.cs @@ -20,8 +20,7 @@ namespace API.Helpers CreateMap(); CreateMap(); - //CreateMap(); - + CreateMap(); CreateMap() From 4c6758b9b0942524f6647aeebf04ccd51e4ab7ef Mon Sep 17 00:00:00 2001 From: Joseph Milazzo Date: Wed, 17 Mar 2021 17:21:59 -0500 Subject: [PATCH 7/9] Can't figure out how to implement StartupTask --- API/Program.cs | 4 +--- API/Startup.cs | 10 +++------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/API/Program.cs b/API/Program.cs index e3d8cae60..aa6c98e56 100644 --- a/API/Program.cs +++ b/API/Program.cs @@ -43,10 +43,8 @@ namespace API logger.LogError(ex, "An error occurred during migration"); } - // Load all tasks from DI (TODO: This is not working - WarmupServicesStartupTask is Null) + // Load all tasks from DI and initialize them (TODO: This is not working - WarmupServicesStartupTask is Null) var startupTasks = host.Services.GetServices(); - - // Execute all the tasks foreach (var startupTask in startupTasks) { await startupTask.ExecuteAsync(); diff --git a/API/Startup.cs b/API/Startup.cs index 2174e5ed2..c9a6a8eca 100644 --- a/API/Startup.cs +++ b/API/Startup.cs @@ -1,8 +1,8 @@ using System; using System.IO.Compression; using System.Linq; -using API.Data; using API.Extensions; +using API.Interfaces.Services; using API.Middleware; using API.Services; using Hangfire; @@ -18,9 +18,7 @@ using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Hosting; -using Microsoft.Extensions.Logging; using Microsoft.OpenApi.Models; -using IApplicationLifetime = Microsoft.AspNetCore.Hosting.IApplicationLifetime; namespace API { @@ -85,10 +83,8 @@ namespace API // Add the processing server as IHostedService services.AddHangfireServer(); - - // services - // .AddStartupTask() - // //.TryAddSingleton(services); + //services.AddStartupTask(services). + services.AddTransient().TryAddSingleton(services); } // This method gets called by the runtime. Use this method to configure the HTTP request pipeline. From a5069158fa10fc15f92e4f7ec7990e7897eb4e93 Mon Sep 17 00:00:00 2001 From: Joseph Milazzo Date: Wed, 17 Mar 2021 17:47:06 -0500 Subject: [PATCH 8/9] Removed tests. For those cases, I was unable to find a good solution. Users will have to manually map or rename. --- API.Tests/ParserTest.cs | 9 +-------- API/Parser/Parser.cs | 3 ++- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/API.Tests/ParserTest.cs b/API.Tests/ParserTest.cs index 0d13f7628..0190e6dc7 100644 --- a/API.Tests/ParserTest.cs +++ b/API.Tests/ParserTest.cs @@ -101,9 +101,6 @@ namespace API.Tests [InlineData("Kedouin Makoto - Corpse Party Musume, Chapter 09", "Kedouin Makoto - Corpse Party Musume")] [InlineData("Goblin Slayer Side Story - Year One 025.5", "Goblin Slayer Side Story - Year One")] [InlineData("Goblin Slayer - Brand New Day 006.5 (2019) (Digital) (danke-Empire)", "Goblin Slayer - Brand New Day")] - [InlineData("Yumekui-Merry_DKThias_Chapter11v2.zip", "Yumekui-Merry")] - [InlineData("Yumekui-Merry DKThiasScanlations Chapter51v2", "Yumekui-Merry")] - [InlineData("Yumekui-Merry_DKThiasScanlations&RenzokuseiScans_Chapter61", "Yumekui-Merry")] public void ParseSeriesTest(string filename, string expected) { Assert.Equal(expected, ParseSeries(filename)); @@ -144,7 +141,6 @@ namespace API.Tests [InlineData("Vol 1", "0")] [InlineData("VanDread-v01-c001[MD].zip", "1")] [InlineData("Goblin Slayer Side Story - Year One 025.5", "25.5")] - [InlineData("Mob Psycho 100 v02 (2019) (Digital) (Shizu).cbz", "0")] public void ParseChaptersTest(string filename, string expected) { Assert.Equal(expected, ParseChapter(filename)); @@ -306,10 +302,7 @@ namespace API.Tests Chapters = "6.5", Filename = "Goblin Slayer - Brand New Day 006.5 (2019) (Digital) (danke-Empire).cbz", Format = MangaFormat.Archive, FullFilePath = filepath }); - - - - + foreach (var file in expected.Keys) { diff --git a/API/Parser/Parser.cs b/API/Parser/Parser.cs index 1e860ab28..27b6309c1 100644 --- a/API/Parser/Parser.cs +++ b/API/Parser/Parser.cs @@ -124,9 +124,10 @@ namespace API.Parser RegexOptions.IgnoreCase | RegexOptions.Compiled), // [Suihei Kiki]_Kasumi_Otoko_no_Ko_[Taruby]_v1.1.zip new Regex( - @"v\d+\.(?\d+(?:.\d+|-\d+)?)", RegexOptions.IgnoreCase | RegexOptions.Compiled), + // Mob Psycho 100 + // Hinowa ga CRUSH! 018 (2019) (Digital) (LuCaZ).cbz, Hinowa ga CRUSH! 018.5 (2019) (Digital) (LuCaZ).cbz new Regex( @"^(?!Vol)(?.*) (?\d+(?:.\d+|-\d+)?)(?: \(\d{4}\))?", From c312e0706ff8a4e4544b9884506a1719e1a4df0f Mon Sep 17 00:00:00 2001 From: Joseph Milazzo Date: Wed, 17 Mar 2021 18:07:10 -0500 Subject: [PATCH 9/9] Removed a leftover code for async --- API/Services/Tasks/ScannerService.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/API/Services/Tasks/ScannerService.cs b/API/Services/Tasks/ScannerService.cs index e63144113..61842eac8 100644 --- a/API/Services/Tasks/ScannerService.cs +++ b/API/Services/Tasks/ScannerService.cs @@ -66,7 +66,7 @@ namespace API.Services.Tasks //[DisableConcurrentExecution(5)] [AutomaticRetry(Attempts = 0, LogEvents = false, OnAttemptsExceeded = AttemptsExceededAction.Delete)] - public async void ScanLibrary(int libraryId, bool forceUpdate) + public void ScanLibrary(int libraryId, bool forceUpdate) { _forceUpdate = forceUpdate; var sw = Stopwatch.StartNew();