diff --git a/API/Data/MigrateConfigFiles.cs b/API/Data/MigrateConfigFiles.cs index 2436e3820..752b03192 100644 --- a/API/Data/MigrateConfigFiles.cs +++ b/API/Data/MigrateConfigFiles.cs @@ -26,8 +26,6 @@ namespace API.Data "temp" }; - private static readonly string ConfigDirectory = Path.Join(Directory.GetCurrentDirectory(), "config"); - /// /// In v0.4.8 we moved all config files to config/ to match with how docker was setup. This will move all config files from current directory @@ -66,8 +64,8 @@ namespace API.Data Console.WriteLine( "Migrating files from pre-v0.4.8. All Kavita config files are now located in config/"); - Console.WriteLine($"Creating {ConfigDirectory}"); - DirectoryService.ExistOrCreate(ConfigDirectory); + Console.WriteLine($"Creating {DirectoryService.ConfigDirectory}"); + DirectoryService.ExistOrCreate(DirectoryService.ConfigDirectory); try { @@ -116,13 +114,13 @@ namespace API.Data foreach (var folderToMove in AppFolders) { - if (new DirectoryInfo(Path.Join(ConfigDirectory, folderToMove)).Exists) continue; + if (new DirectoryInfo(Path.Join(DirectoryService.ConfigDirectory, folderToMove)).Exists) continue; try { DirectoryService.CopyDirectoryToDirectory( Path.Join(Directory.GetCurrentDirectory(), folderToMove), - Path.Join(ConfigDirectory, folderToMove)); + Path.Join(DirectoryService.ConfigDirectory, folderToMove)); } catch (Exception) { @@ -144,7 +142,7 @@ namespace API.Data { try { - fileInfo.CopyTo(Path.Join(ConfigDirectory, fileInfo.Name)); + fileInfo.CopyTo(Path.Join(DirectoryService.ConfigDirectory, fileInfo.Name)); } catch (Exception) { diff --git a/API/Services/DirectoryService.cs b/API/Services/DirectoryService.cs index b9560084f..241d71422 100644 --- a/API/Services/DirectoryService.cs +++ b/API/Services/DirectoryService.cs @@ -266,7 +266,7 @@ namespace API.Services /// /// An optional string to prepend to the target file's name /// - public bool CopyFilesToDirectory(IEnumerable filePaths, string directoryPath, string prepend = "") + public static bool CopyFilesToDirectory(IEnumerable filePaths, string directoryPath, string prepend = "", ILogger logger = null) { ExistOrCreate(directoryPath); string currentFile = null; @@ -282,19 +282,24 @@ namespace API.Services } else { - _logger.LogWarning("Tried to copy {File} but it doesn't exist", file); + logger?.LogWarning("Tried to copy {File} but it doesn't exist", file); } } } catch (Exception ex) { - _logger.LogError(ex, "Unable to copy {File} to {DirectoryPath}", currentFile, directoryPath); + logger?.LogError(ex, "Unable to copy {File} to {DirectoryPath}", currentFile, directoryPath); return false; } return true; } + public bool CopyFilesToDirectory(IEnumerable filePaths, string directoryPath, string prepend = "") + { + return CopyFilesToDirectory(filePaths, directoryPath, prepend, _logger); + } + public IEnumerable ListDirectory(string rootPath) { if (!Directory.Exists(rootPath)) return ImmutableList.Empty; diff --git a/API/Services/MetadataService.cs b/API/Services/MetadataService.cs index 1b5ab0d69..5d86f0766 100644 --- a/API/Services/MetadataService.cs +++ b/API/Services/MetadataService.cs @@ -218,14 +218,18 @@ namespace API.Services var stopwatch = Stopwatch.StartNew(); var totalTime = 0L; _logger.LogInformation("[MetadataService] Refreshing Library {LibraryName}. Total Items: {TotalSize}. Total Chunks: {TotalChunks} with {ChunkSize} size", library.Name, chunkInfo.TotalSize, chunkInfo.TotalChunks, chunkInfo.ChunkSize); + await _messageHub.Clients.All.SendAsync(SignalREvents.ScanLibraryProgress, + MessageFactory.RefreshMetadataProgressEvent(library.Id, 0F)); - for (var chunk = 1; chunk <= chunkInfo.TotalChunks; chunk++) + var i = 0; + for (var chunk = 1; chunk <= chunkInfo.TotalChunks; chunk++, i++) { if (chunkInfo.TotalChunks == 0) continue; totalTime += stopwatch.ElapsedMilliseconds; stopwatch.Restart(); _logger.LogInformation("[MetadataService] Processing chunk {ChunkNumber} / {TotalChunks} with size {ChunkSize}. Series ({SeriesStart} - {SeriesEnd}", chunk, chunkInfo.TotalChunks, chunkInfo.ChunkSize, chunk * chunkInfo.ChunkSize, (chunk + 1) * chunkInfo.ChunkSize); + var nonLibrarySeries = await _unitOfWork.SeriesRepository.GetFullSeriesForLibraryIdAsync(library.Id, new UserParams() { @@ -233,6 +237,7 @@ namespace API.Services PageSize = chunkInfo.ChunkSize }); _logger.LogDebug("[MetadataService] Fetched {SeriesCount} series for refresh", nonLibrarySeries.Count); + Parallel.ForEach(nonLibrarySeries, series => { try @@ -275,8 +280,14 @@ namespace API.Services "[MetadataService] Processed {SeriesStart} - {SeriesEnd} out of {TotalSeries} series in {ElapsedScanTime} milliseconds for {LibraryName}", chunk * chunkInfo.ChunkSize, (chunk * chunkInfo.ChunkSize) + nonLibrarySeries.Count, chunkInfo.TotalSize, stopwatch.ElapsedMilliseconds, library.Name); } + var progress = Math.Max(0F, Math.Min(100F, i * 1F / chunkInfo.TotalChunks)); + await _messageHub.Clients.All.SendAsync(SignalREvents.ScanLibraryProgress, + MessageFactory.RefreshMetadataProgressEvent(library.Id, progress)); } + await _messageHub.Clients.All.SendAsync(SignalREvents.ScanLibraryProgress, + MessageFactory.RefreshMetadataProgressEvent(library.Id, 100F)); + _logger.LogInformation("[MetadataService] Updated metadata for {SeriesNumber} series in library {LibraryName} in {ElapsedMilliseconds} milliseconds total", chunkInfo.TotalSize, library.Name, totalTime); } diff --git a/API/Services/Tasks/BackupService.cs b/API/Services/Tasks/BackupService.cs index bf1f26711..5f268be5d 100644 --- a/API/Services/Tasks/BackupService.cs +++ b/API/Services/Tasks/BackupService.cs @@ -9,7 +9,6 @@ using API.Extensions; using API.Interfaces; using API.Interfaces.Services; using Hangfire; -using Kavita.Common.EnvironmentInfo; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; @@ -20,8 +19,6 @@ namespace API.Services.Tasks private readonly IUnitOfWork _unitOfWork; private readonly ILogger _logger; private readonly IDirectoryService _directoryService; - private readonly string _tempDirectory = DirectoryService.TempDirectory; - private readonly string _logDirectory = DirectoryService.LogDirectory; private readonly IList _backupFiles; @@ -35,30 +32,16 @@ namespace API.Services.Tasks var loggingSection = config.GetLoggingFileName(); var files = LogFiles(maxRollingFiles, loggingSection); - if (new OsInfo(Array.Empty()).IsDocker) + + _backupFiles = new List() { - _backupFiles = new List() - { - "data/appsettings.json", - "data/Hangfire.db", - "data/Hangfire-log.db", - "data/kavita.db", - "data/kavita.db-shm", // This wont always be there - "data/kavita.db-wal" // This wont always be there - }; - } - else - { - _backupFiles = new List() - { - "appsettings.json", - "Hangfire.db", - "Hangfire-log.db", - "kavita.db", - "kavita.db-shm", // This wont always be there - "kavita.db-wal" // This wont always be there - }; - } + "appsettings.json", + "Hangfire.db", // This is not used atm + "Hangfire-log.db", // This is not used atm + "kavita.db", + "kavita.db-shm", // This wont always be there + "kavita.db-wal" // This wont always be there + }; foreach (var file in files.Select(f => (new FileInfo(f)).Name).ToList()) { @@ -72,7 +55,7 @@ namespace API.Services.Tasks var fi = new FileInfo(logFileName); var files = maxRollingFiles > 0 - ? DirectoryService.GetFiles(_logDirectory, $@"{Path.GetFileNameWithoutExtension(fi.Name)}{multipleFileRegex}\.log") + ? DirectoryService.GetFiles(DirectoryService.LogDirectory, $@"{Path.GetFileNameWithoutExtension(fi.Name)}{multipleFileRegex}\.log") : new[] {"kavita.log"}; return files; } @@ -93,7 +76,7 @@ namespace API.Services.Tasks return; } - var dateString = DateTime.Now.ToShortDateString().Replace("/", "_"); + var dateString = $"{DateTime.Now.ToShortDateString()}_{DateTime.Now.ToLongTimeString()}".Replace("/", "_").Replace(":", "_"); var zipPath = Path.Join(backupDirectory, $"kavita_backup_{dateString}.zip"); if (File.Exists(zipPath)) @@ -102,7 +85,7 @@ namespace API.Services.Tasks return; } - var tempDirectory = Path.Join(_tempDirectory, dateString); + var tempDirectory = Path.Join(DirectoryService.TempDirectory, dateString); DirectoryService.ExistOrCreate(tempDirectory); DirectoryService.ClearDirectory(tempDirectory); diff --git a/API/Services/Tasks/ScannerService.cs b/API/Services/Tasks/ScannerService.cs index c2abe341f..250045a47 100644 --- a/API/Services/Tasks/ScannerService.cs +++ b/API/Services/Tasks/ScannerService.cs @@ -360,14 +360,13 @@ namespace API.Services.Tasks Series existingSeries; try { - existingSeries = allSeries.SingleOrDefault(s => - (s.NormalizedName.Equals(key.NormalizedName) || Parser.Parser.Normalize(s.OriginalName).Equals(key.NormalizedName)) - && (s.Format == key.Format || s.Format == MangaFormat.Unknown)); + existingSeries = allSeries.SingleOrDefault(s => FindSeries(s, key)); } catch (Exception e) { + // NOTE: If I ever want to put Duplicates table, this is where it can go _logger.LogCritical(e, "[ScannerService] There are multiple series that map to normalized key {Key}. You can manually delete the entity via UI and rescan to fix it. This will be skipped", key.NormalizedName); - var duplicateSeries = allSeries.Where(s => s.NormalizedName == key.NormalizedName || Parser.Parser.Normalize(s.OriginalName) == key.NormalizedName).ToList(); + var duplicateSeries = allSeries.Where(s => FindSeries(s, key)); foreach (var series in duplicateSeries) { _logger.LogCritical("[ScannerService] Duplicate Series Found: {Key} maps with {Series}", key.Name, series.OriginalName); @@ -378,21 +377,20 @@ namespace API.Services.Tasks if (existingSeries != null) continue; - existingSeries = DbFactory.Series(infos[0].Series); - existingSeries.Format = key.Format; - newSeries.Add(existingSeries); + var s = DbFactory.Series(infos[0].Series); + s.Format = key.Format; + s.LibraryId = library.Id; // We have to manually set this since we aren't adding the series to the Library's series. + newSeries.Add(s); } var i = 0; foreach(var series in newSeries) { + _logger.LogDebug("[ScannerService] Processing series {SeriesName}", series.OriginalName); + UpdateSeries(series, parsedSeries); + _unitOfWork.SeriesRepository.Attach(series); try { - _logger.LogDebug("[ScannerService] Processing series {SeriesName}", series.OriginalName); - UpdateVolumes(series, ParseScannedFiles.GetInfosByName(parsedSeries, series).ToArray()); - series.Pages = series.Volumes.Sum(v => v.Pages); - series.LibraryId = library.Id; // We have to manually set this since we aren't adding the series to the Library's series. - _unitOfWork.SeriesRepository.Attach(series); await _unitOfWork.CommitAsync(); _logger.LogInformation( "[ScannerService] Added {NewSeries} series in {ElapsedScanTime} milliseconds for {LibraryName}", @@ -403,7 +401,7 @@ namespace API.Services.Tasks } catch (Exception ex) { - _logger.LogCritical(ex, "[ScannerService] There was a critical exception adding new series entry for {SeriesName} with a duplicate index key: {IndexKey}", + _logger.LogCritical(ex, "[ScannerService] There was a critical exception adding new series entry for {SeriesName} with a duplicate index key: {IndexKey} ", series.Name, $"{series.Name}_{series.NormalizedName}_{series.LocalizedName}_{series.LibraryId}_{series.Format}"); } @@ -418,13 +416,19 @@ namespace API.Services.Tasks newSeries.Count, stopwatch.ElapsedMilliseconds, library.Name); } + private static bool FindSeries(Series series, ParsedSeries parsedInfoKey) + { + return (series.NormalizedName.Equals(parsedInfoKey.NormalizedName) || Parser.Parser.Normalize(series.OriginalName).Equals(parsedInfoKey.NormalizedName)) + && (series.Format == parsedInfoKey.Format || series.Format == MangaFormat.Unknown); + } + private void UpdateSeries(Series series, Dictionary> parsedSeries) { try { _logger.LogInformation("[ScannerService] Processing series {SeriesName}", series.OriginalName); - var parsedInfos = ParseScannedFiles.GetInfosByName(parsedSeries, series).ToArray(); + var parsedInfos = ParseScannedFiles.GetInfosByName(parsedSeries, series); UpdateVolumes(series, parsedInfos); series.Pages = series.Volumes.Sum(v => v.Pages); @@ -491,7 +495,7 @@ namespace API.Services.Tasks /// Series not found on disk or can't be parsed /// /// the updated existingSeries - public static IList RemoveMissingSeries(IList existingSeries, IEnumerable missingSeries, out int removeCount) + public static IEnumerable RemoveMissingSeries(IList existingSeries, IEnumerable missingSeries, out int removeCount) { var existingCount = existingSeries.Count; var missingList = missingSeries.ToList(); @@ -505,7 +509,7 @@ namespace API.Services.Tasks return existingSeries; } - private void UpdateVolumes(Series series, ParserInfo[] parsedInfos) + private void UpdateVolumes(Series series, IList parsedInfos) { var startingVolumeCount = series.Volumes.Count; // Add new volumes and update chapters per volume @@ -559,7 +563,7 @@ namespace API.Services.Tasks /// /// /// - private void UpdateChapters(Volume volume, ParserInfo[] parsedInfos) + private void UpdateChapters(Volume volume, IList parsedInfos) { // Add new chapters foreach (var info in parsedInfos) diff --git a/API/SignalR/MessageFactory.cs b/API/SignalR/MessageFactory.cs index 0bab0b4da..c0f069f18 100644 --- a/API/SignalR/MessageFactory.cs +++ b/API/SignalR/MessageFactory.cs @@ -60,6 +60,20 @@ namespace API.SignalR }; } + public static SignalRMessage RefreshMetadataProgressEvent(int libraryId, float progress) + { + return new SignalRMessage() + { + Name = SignalREvents.RefreshMetadataProgress, + Body = new + { + LibraryId = libraryId, + Progress = progress, + EventTime = DateTime.Now + } + }; + } + public static SignalRMessage RefreshMetadataEvent(int libraryId, int seriesId) diff --git a/API/SignalR/SignalREvents.cs b/API/SignalR/SignalREvents.cs index 0f97ad493..09a6e8f24 100644 --- a/API/SignalR/SignalREvents.cs +++ b/API/SignalR/SignalREvents.cs @@ -4,7 +4,14 @@ { public const string UpdateVersion = "UpdateVersion"; public const string ScanSeries = "ScanSeries"; + /// + /// Event during Refresh Metadata for cover image change + /// public const string RefreshMetadata = "RefreshMetadata"; + /// + /// Event sent out during Refresh Metadata for progress tracking + /// + public const string RefreshMetadataProgress = "RefreshMetadataProgress"; public const string ScanLibrary = "ScanLibrary"; public const string SeriesAdded = "SeriesAdded"; public const string SeriesRemoved = "SeriesRemoved"; diff --git a/UI/Web/src/app/_services/message-hub.service.ts b/UI/Web/src/app/_services/message-hub.service.ts index 1365746ed..1a1c4d8f7 100644 --- a/UI/Web/src/app/_services/message-hub.service.ts +++ b/UI/Web/src/app/_services/message-hub.service.ts @@ -16,6 +16,7 @@ export enum EVENTS { UpdateAvailable = 'UpdateAvailable', ScanSeries = 'ScanSeries', RefreshMetadata = 'RefreshMetadata', + RefreshMetadataProgress = 'RefreshMetadataProgress', SeriesAdded = 'SeriesAdded', SeriesRemoved = 'SeriesRemoved', ScanLibraryProgress = 'ScanLibraryProgress', @@ -89,6 +90,13 @@ export class MessageHubService { this.scanLibrary.emit(resp.body); }); + this.hubConnection.on(EVENTS.RefreshMetadataProgress, resp => { + this.messagesSource.next({ + event: EVENTS.RefreshMetadataProgress, + payload: resp.body + }); + }); + this.hubConnection.on(EVENTS.SeriesAddedToCollection, resp => { this.messagesSource.next({ event: EVENTS.SeriesAddedToCollection, diff --git a/UI/Web/src/app/admin/_modals/library-access-modal/library-access-modal.component.html b/UI/Web/src/app/admin/_modals/library-access-modal/library-access-modal.component.html index de4001387..e97c96179 100644 --- a/UI/Web/src/app/admin/_modals/library-access-modal/library-access-modal.component.html +++ b/UI/Web/src/app/admin/_modals/library-access-modal/library-access-modal.component.html @@ -6,17 +6,24 @@ - - - - - {{library.data.name}} - - - - There are no libraries setup yet. - + + + + {{selectAll ? 'Deselect' : 'Select'}} All + + + + + + {{library.name}} + + + + There are no libraries setup yet. + +