diff --git a/API.Benchmark/CleanTitleBenchmark.cs b/API.Benchmark/CleanTitleBenchmark.cs index a32b4beb2..90310a9ef 100644 --- a/API.Benchmark/CleanTitleBenchmark.cs +++ b/API.Benchmark/CleanTitleBenchmark.cs @@ -8,15 +8,15 @@ using BenchmarkDotNet.Order; namespace API.Benchmark; [MemoryDiagnoser] -public class CleanTitleBenchmarks +public static class CleanTitleBenchmarks { private static IList _names; [GlobalSetup] - public void LoadData() => _names = File.ReadAllLines("Data/Comics.txt"); + public static void LoadData() => _names = File.ReadAllLines("Data/Comics.txt"); [Benchmark] - public void TestCleanTitle() + public static void TestCleanTitle() { foreach (var name in _names) { diff --git a/API.Tests/Parser/MangaParserTests.cs b/API.Tests/Parser/MangaParserTests.cs index 010e5ea3f..f96620378 100644 --- a/API.Tests/Parser/MangaParserTests.cs +++ b/API.Tests/Parser/MangaParserTests.cs @@ -189,6 +189,7 @@ public class MangaParserTests [InlineData("Манга Глава 2", "Манга")] [InlineData("Манга Глава 2-2", "Манга")] [InlineData("Манга Том 1 3-4 Глава", "Манга")] + [InlineData("Esquire 6권 2021년 10월호", "Esquire")] public void ParseSeriesTest(string filename, string expected) { Assert.Equal(expected, API.Services.Tasks.Scanner.Parser.Parser.ParseSeries(filename)); diff --git a/API/API.csproj b/API/API.csproj index 21dd48ce7..e38ee8b77 100644 --- a/API/API.csproj +++ b/API/API.csproj @@ -72,9 +72,11 @@ + + @@ -83,15 +85,15 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive - - + + diff --git a/API/DTOs/Filtering/FilterDto.cs b/API/DTOs/Filtering/FilterDto.cs index a8c7aa1cc..f6c47f71f 100644 --- a/API/DTOs/Filtering/FilterDto.cs +++ b/API/DTOs/Filtering/FilterDto.cs @@ -99,8 +99,10 @@ public class FilterDto /// An optional name string to filter by. Empty string will ignore. /// public string SeriesNameQuery { get; init; } = string.Empty; + #nullable enable /// /// An optional release year to filter by. Null will ignore. You can pass 0 for an individual field to ignore it. /// public Range? ReleaseYearRange { get; init; } = null; + #nullable disable } diff --git a/API/Data/Repositories/SeriesRepository.cs b/API/Data/Repositories/SeriesRepository.cs index e656de29c..d9ac6779e 100644 --- a/API/Data/Repositories/SeriesRepository.cs +++ b/API/Data/Repositories/SeriesRepository.cs @@ -56,6 +56,7 @@ internal class RecentlyAddedSeries public interface ISeriesRepository { + void Add(Series series); void Attach(Series series); void Update(Series series); void Remove(Series series); @@ -136,6 +137,11 @@ public class SeriesRepository : ISeriesRepository _mapper = mapper; } + public void Add(Series series) + { + _context.Series.Add(series); + } + public void Attach(Series series) { _context.Series.Attach(series); diff --git a/API/Extensions/ApplicationServiceExtensions.cs b/API/Extensions/ApplicationServiceExtensions.cs index cb4c871c9..c696e4858 100644 --- a/API/Extensions/ApplicationServiceExtensions.cs +++ b/API/Extensions/ApplicationServiceExtensions.cs @@ -59,12 +59,11 @@ public static class ApplicationServiceExtensions services.AddScoped(); services.AddScoped(); - services.AddSqLite(config, env); + services.AddSqLite(env); services.AddSignalR(opt => opt.EnableDetailedErrors = true); } - private static void AddSqLite(this IServiceCollection services, IConfiguration config, - IHostEnvironment env) + private static void AddSqLite(this IServiceCollection services, IHostEnvironment env) { services.AddDbContext(options => { diff --git a/API/Logging/LogLevelOptions.cs b/API/Logging/LogLevelOptions.cs index 66bdbe423..e7e97268b 100644 --- a/API/Logging/LogLevelOptions.cs +++ b/API/Logging/LogLevelOptions.cs @@ -46,12 +46,16 @@ public static class LogLevelOptions .MinimumLevel.Override("Microsoft.Hosting.Lifetime", MicrosoftHostingLifetimeLogLevelSwitch) .MinimumLevel.Override("Hangfire", HangfireLogLevelSwitch) .MinimumLevel.Override("Microsoft.AspNetCore.Hosting.Internal.WebHost", AspNetCoreLogLevelSwitch) + // Suppress noisy loggers that add no value + .MinimumLevel.Override("Microsoft.AspNetCore.ResponseCaching.ResponseCachingMiddleware", LogEventLevel.Error) + .MinimumLevel.Override("Microsoft.AspNetCore", LogEventLevel.Error) .Enrich.FromLogContext() + .Enrich.WithThreadId() .WriteTo.Console() .WriteTo.File(LogFile, shared: true, rollingInterval: RollingInterval.Day, - outputTemplate: "[{Timestamp:yyyy-MM-dd HH:mm:ss.fff zzz} {CorrelationId}] [{Level}] {Message:lj}{NewLine}{Exception}"); + outputTemplate: "[{Timestamp:yyyy-MM-dd HH:mm:ss.fff zzz} {CorrelationId} {ThreadId}] [{Level}] {SourceContext} {Message:lj}{NewLine}{Exception}"); } public static void SwitchLogLevel(string level) @@ -60,9 +64,9 @@ public static class LogLevelOptions { case "Debug": LogLevelSwitch.MinimumLevel = LogEventLevel.Debug; - MicrosoftLogLevelSwitch.MinimumLevel = LogEventLevel.Information; - MicrosoftHostingLifetimeLogLevelSwitch.MinimumLevel = LogEventLevel.Debug; - AspNetCoreLogLevelSwitch.MinimumLevel = LogEventLevel.Debug; + MicrosoftLogLevelSwitch.MinimumLevel = LogEventLevel.Warning; // This is DB output information, Inf shows the SQL + MicrosoftHostingLifetimeLogLevelSwitch.MinimumLevel = LogEventLevel.Information; + AspNetCoreLogLevelSwitch.MinimumLevel = LogEventLevel.Warning; break; case "Information": LogLevelSwitch.MinimumLevel = LogEventLevel.Error; @@ -74,7 +78,7 @@ public static class LogLevelOptions LogLevelSwitch.MinimumLevel = LogEventLevel.Verbose; MicrosoftLogLevelSwitch.MinimumLevel = LogEventLevel.Information; MicrosoftHostingLifetimeLogLevelSwitch.MinimumLevel = LogEventLevel.Debug; - AspNetCoreLogLevelSwitch.MinimumLevel = LogEventLevel.Debug; + AspNetCoreLogLevelSwitch.MinimumLevel = LogEventLevel.Information; break; case "Warning": LogLevelSwitch.MinimumLevel = LogEventLevel.Warning; diff --git a/API/Services/Tasks/Scanner/LibraryWatcher.cs b/API/Services/Tasks/Scanner/LibraryWatcher.cs index 6dfc7abc5..24b938153 100644 --- a/API/Services/Tasks/Scanner/LibraryWatcher.cs +++ b/API/Services/Tasks/Scanner/LibraryWatcher.cs @@ -40,12 +40,11 @@ public class LibraryWatcher : ILibraryWatcher private readonly ILogger _logger; private readonly IScannerService _scannerService; - private readonly Dictionary> _watcherDictionary = new (); + private static readonly Dictionary> WatcherDictionary = new (); /// /// This is just here to prevent GC from Disposing our watchers /// - private readonly IList _fileWatchers = new List(); - private IList _libraryFolders = new List(); + private static readonly IList FileWatchers = new List(); /// /// The amount of time until the Schedule ScanFolder task should be executed /// @@ -68,13 +67,14 @@ public class LibraryWatcher : ILibraryWatcher { _logger.LogInformation("[LibraryWatcher] Starting file watchers"); - _libraryFolders = (await _unitOfWork.LibraryRepository.GetLibraryDtosAsync()) + var libraryFolders = (await _unitOfWork.LibraryRepository.GetLibraryDtosAsync()) .SelectMany(l => l.Folders) .Distinct() .Select(Parser.Parser.NormalizePath) .Where(_directoryService.Exists) .ToList(); - foreach (var libraryFolder in _libraryFolders) + + foreach (var libraryFolder in libraryFolders) { _logger.LogDebug("[LibraryWatcher] Watching {FolderPath}", libraryFolder); var watcher = new FileSystemWatcher(libraryFolder); @@ -87,21 +87,21 @@ public class LibraryWatcher : ILibraryWatcher watcher.Filter = "*.*"; watcher.IncludeSubdirectories = true; watcher.EnableRaisingEvents = true; - _fileWatchers.Add(watcher); - if (!_watcherDictionary.ContainsKey(libraryFolder)) + FileWatchers.Add(watcher); + if (!WatcherDictionary.ContainsKey(libraryFolder)) { - _watcherDictionary.Add(libraryFolder, new List()); + WatcherDictionary.Add(libraryFolder, new List()); } - _watcherDictionary[libraryFolder].Add(watcher); + WatcherDictionary[libraryFolder].Add(watcher); } - _logger.LogInformation("[LibraryWatcher] Watching {Count} folders", _fileWatchers.Count); + _logger.LogInformation("[LibraryWatcher] Watching {Count} folders", FileWatchers.Count); } public void StopWatching() { _logger.LogInformation("[LibraryWatcher] Stopping watching folders"); - foreach (var fileSystemWatcher in _watcherDictionary.Values.SelectMany(watcher => watcher)) + foreach (var fileSystemWatcher in WatcherDictionary.Values.SelectMany(watcher => watcher)) { fileSystemWatcher.EnableRaisingEvents = false; fileSystemWatcher.Changed -= OnChanged; @@ -110,12 +110,13 @@ public class LibraryWatcher : ILibraryWatcher fileSystemWatcher.Error -= OnError; fileSystemWatcher.Dispose(); } - _fileWatchers.Clear(); - _watcherDictionary.Clear(); + FileWatchers.Clear(); + WatcherDictionary.Clear(); } public async Task RestartWatching() { + _logger.LogDebug("[LibraryWatcher] Restarting watcher"); StopWatching(); await StartWatching(); } @@ -160,7 +161,7 @@ public class LibraryWatcher : ILibraryWatcher /// This is public only because Hangfire will invoke it. Do not call external to this class. /// File or folder that changed /// If the change is on a directory and not a file - public void ProcessChange(string filePath, bool isDirectoryChange = false) + public async Task ProcessChange(string filePath, bool isDirectoryChange = false) { var sw = Stopwatch.StartNew(); _logger.LogDebug("[LibraryWatcher] Processing change of {FilePath}", filePath); @@ -174,26 +175,33 @@ public class LibraryWatcher : ILibraryWatcher return; } - var fullPath = GetFolder(filePath, _libraryFolders); + var libraryFolders = (await _unitOfWork.LibraryRepository.GetLibraryDtosAsync()) + .SelectMany(l => l.Folders) + .Distinct() + .Select(Parser.Parser.NormalizePath) + .Where(_directoryService.Exists) + .ToList(); + + var fullPath = GetFolder(filePath, libraryFolders); + _logger.LogDebug("Folder path: {FolderPath}", fullPath); if (string.IsNullOrEmpty(fullPath)) { _logger.LogDebug("[LibraryWatcher] Change from {FilePath} could not find root level folder, ignoring change", filePath); return; } - // Check if this task has already enqueued or is being processed, before enquing + // Check if this task has already enqueued or is being processed, before enqueing var alreadyScheduled = TaskScheduler.HasAlreadyEnqueuedTask(ScannerService.Name, "ScanFolder", new object[] {fullPath}); - _logger.LogDebug("[LibraryWatcher] {FullPath} already enqueued: {Value}", fullPath, alreadyScheduled); if (!alreadyScheduled) { - _logger.LogDebug("[LibraryWatcher] Scheduling ScanFolder for {Folder}", fullPath); + _logger.LogInformation("[LibraryWatcher] Scheduling ScanFolder for {Folder}", fullPath); BackgroundJob.Schedule(() => _scannerService.ScanFolder(fullPath), _queueWaitTime); } else { - _logger.LogDebug("[LibraryWatcher] Skipped scheduling ScanFolder for {Folder} as a job already queued", + _logger.LogInformation("[LibraryWatcher] Skipped scheduling ScanFolder for {Folder} as a job already queued", fullPath); } } @@ -207,18 +215,17 @@ public class LibraryWatcher : ILibraryWatcher private string GetFolder(string filePath, IList libraryFolders) { var parentDirectory = _directoryService.GetParentDirectoryName(filePath); - if (string.IsNullOrEmpty(parentDirectory)) - { - return string.Empty; - } + _logger.LogDebug("[LibraryWatcher] Parent Directory: {ParentDirectory}", parentDirectory); if (string.IsNullOrEmpty(parentDirectory)) return string.Empty; // We need to find the library this creation belongs to // Multiple libraries can point to the same base folder. In this case, we need use FirstOrDefault var libraryFolder = libraryFolders.FirstOrDefault(f => parentDirectory.Contains(f)); + _logger.LogDebug("[LibraryWatcher] Library Folder: {LibraryFolder}", libraryFolder); if (string.IsNullOrEmpty(libraryFolder)) return string.Empty; var rootFolder = _directoryService.GetFoldersTillRoot(libraryFolder, filePath).ToList(); + _logger.LogDebug("[LibraryWatcher] Root Folders: {RootFolders}", rootFolder); if (!rootFolder.Any()) return string.Empty; // Select the first folder and join with library folder, this should give us the folder to scan. diff --git a/API/Services/Tasks/Scanner/ProcessSeries.cs b/API/Services/Tasks/Scanner/ProcessSeries.cs index 545031abc..807ef8f31 100644 --- a/API/Services/Tasks/Scanner/ProcessSeries.cs +++ b/API/Services/Tasks/Scanner/ProcessSeries.cs @@ -44,9 +44,9 @@ public class ProcessSeries : IProcessSeries private readonly IMetadataService _metadataService; private readonly IWordCountAnalyzerService _wordCountAnalyzerService; - private IList _genres; - private IList _people; - private IList _tags; + private volatile IList _genres; + private volatile IList _people; + private volatile IList _tags; @@ -108,6 +108,7 @@ public class ProcessSeries : IProcessSeries { seriesAdded = true; series = DbFactory.Series(firstInfo.Series, firstInfo.LocalizedSeries); + _unitOfWork.SeriesRepository.Add(series); } if (series.LibraryId == 0) series.LibraryId = library.Id; @@ -156,7 +157,6 @@ public class ProcessSeries : IProcessSeries await UpdateSeriesFolderPath(parsedInfos, library, series); series.LastFolderScanned = DateTime.Now; - _unitOfWork.SeriesRepository.Attach(series); if (_unitOfWork.HasChanges()) { @@ -167,7 +167,7 @@ public class ProcessSeries : IProcessSeries catch (Exception ex) { await _unitOfWork.RollbackAsync(); - _logger.LogCritical(ex, "[ScannerService] There was an issue writing to the for series {@SeriesName}", series); + _logger.LogCritical(ex, "[ScannerService] There was an issue writing to the database for series {@SeriesName}", series.Name); await _eventHub.SendMessageAsync(MessageFactory.Error, MessageFactory.ErrorEvent($"There was an issue writing to the DB for Series {series}", diff --git a/API/Startup.cs b/API/Startup.cs index 905322159..1291a702c 100644 --- a/API/Startup.cs +++ b/API/Startup.cs @@ -268,7 +268,10 @@ public class Startup }); app.UseSerilogRequestLogging(opts - => opts.EnrichDiagnosticContext = LogEnricher.EnrichFromRequest); + => + { + opts.EnrichDiagnosticContext = LogEnricher.EnrichFromRequest; + }); app.Use(async (context, next) => { diff --git a/entrypoint.sh b/entrypoint.sh index 155cf62e7..ef42f34a4 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -4,7 +4,6 @@ if [ ! -f "/kavita/config/appsettings.json" ]; then echo "Kavita configuration file does not exist, creating..." echo '{ "TokenKey": "super secret unguessable key", - }, "Port": 5000 }' >> /kavita/config/appsettings.json fi