From cd8a1d28922a913efbc5135bb4364bd7f4f9bcef Mon Sep 17 00:00:00 2001 From: Joseph Milazzo Date: Sat, 9 Jan 2021 16:14:28 -0600 Subject: [PATCH] Added a basic cache service to handle interations with the underlying cache implementation. Refactored some code to be more robust. --- API/Controllers/ReaderController.cs | 22 ++++---- .../ApplicationServiceExtensions.cs | 1 + API/Extensions/ZipArchiveExtensions.cs | 19 +++++++ API/IO/ImageProvider.cs | 5 +- API/Interfaces/ICacheService.cs | 21 ++++++++ API/Interfaces/IDirectoryService.cs | 17 +++++- API/Services/CacheService.cs | 46 ++++++++++++++++ API/Services/DirectoryService.cs | 52 ++++++++++++------- 8 files changed, 149 insertions(+), 34 deletions(-) create mode 100644 API/Extensions/ZipArchiveExtensions.cs create mode 100644 API/Interfaces/ICacheService.cs create mode 100644 API/Services/CacheService.cs diff --git a/API/Controllers/ReaderController.cs b/API/Controllers/ReaderController.cs index 4ad88bb40..e426e87a8 100644 --- a/API/Controllers/ReaderController.cs +++ b/API/Controllers/ReaderController.cs @@ -11,16 +11,19 @@ namespace API.Controllers { private readonly ISeriesRepository _seriesRepository; private readonly IDirectoryService _directoryService; + private readonly ICacheService _cacheService; - public ReaderController(ISeriesRepository seriesRepository, IDirectoryService directoryService) + public ReaderController(ISeriesRepository seriesRepository, IDirectoryService directoryService, ICacheService cacheService) { _seriesRepository = seriesRepository; _directoryService = directoryService; + _cacheService = cacheService; } [HttpGet("info")] public async Task> GetInformation(int volumeId) { + // TODO: This will be refactored out. No longer needed. Volume volume = await _seriesRepository.GetVolumeAsync(volumeId); // Assume we always get first Manga File @@ -28,24 +31,21 @@ namespace API.Controllers { return BadRequest("There are no files in the volume to read."); } + + _cacheService.Ensure(volumeId); - var filepath = volume.Files.ElementAt(0).FilePath; + return Ok(volume.Files.Select(x => x.NumberOfPages).Sum()); - var extractPath = _directoryService.ExtractArchive(filepath, volumeId); - if (string.IsNullOrEmpty(extractPath)) - { - return BadRequest("There file is no longer there or has no images. Please rescan."); - } - - // NOTE: I'm starting to think this should actually cache the information about Volume/Manga file in the DB. - // It will be updated each time this is called which is on open of a manga. - return Ok(_directoryService.ListFiles(extractPath).Count()); } [HttpGet("image")] public async Task> GetImage(int volumeId, int page) { // Temp let's iterate the directory each call to get next image + _cacheService.Ensure(volumeId); + + + var files = _directoryService.ListFiles(_directoryService.GetExtractPath(volumeId)); var path = files.ElementAt(page); var file = await _directoryService.ReadImageAsync(path); diff --git a/API/Extensions/ApplicationServiceExtensions.cs b/API/Extensions/ApplicationServiceExtensions.cs index 2e0bb342c..2fe5008e7 100644 --- a/API/Extensions/ApplicationServiceExtensions.cs +++ b/API/Extensions/ApplicationServiceExtensions.cs @@ -19,6 +19,7 @@ namespace API.Extensions services.AddScoped(); services.AddScoped(); services.AddScoped(); + services.AddScoped(); services.AddScoped(); services.AddScoped(); services.AddScoped(); diff --git a/API/Extensions/ZipArchiveExtensions.cs b/API/Extensions/ZipArchiveExtensions.cs new file mode 100644 index 000000000..a871162e8 --- /dev/null +++ b/API/Extensions/ZipArchiveExtensions.cs @@ -0,0 +1,19 @@ +using System.IO; +using System.IO.Compression; +using System.Linq; + +namespace API.Extensions +{ + public static class ZipArchiveExtensions + { + /// + /// Checks if archive has one or more files. Excludes directory entries. + /// + /// + /// + public static bool HasFiles(this ZipArchive archive) + { + return archive.Entries.Any(x => Path.HasExtension(x.FullName)); + } + } +} \ No newline at end of file diff --git a/API/IO/ImageProvider.cs b/API/IO/ImageProvider.cs index 9af4f06ca..a85caf0df 100644 --- a/API/IO/ImageProvider.cs +++ b/API/IO/ImageProvider.cs @@ -2,6 +2,7 @@ using System.IO; using System.IO.Compression; using System.Linq; +using API.Extensions; using NetVips; namespace API.IO @@ -21,7 +22,7 @@ namespace API.IO if (!File.Exists(filepath) || !Parser.Parser.IsArchive(filepath)) return Array.Empty(); using ZipArchive archive = ZipFile.OpenRead(filepath); - if (archive.Entries.Count <= 0) return Array.Empty(); + if (!archive.HasFiles()) return Array.Empty(); @@ -48,7 +49,7 @@ namespace API.IO } } - return ExtractEntryToImage(entry); + return ExtractEntryToImage(entry); } private static byte[] ExtractEntryToImage(ZipArchiveEntry entry) diff --git a/API/Interfaces/ICacheService.cs b/API/Interfaces/ICacheService.cs new file mode 100644 index 000000000..809733050 --- /dev/null +++ b/API/Interfaces/ICacheService.cs @@ -0,0 +1,21 @@ +using API.Entities; + +namespace API.Interfaces +{ + public interface ICacheService + { + /// + /// Ensures the cache is created for the given volume and if not, will create it. + /// + /// + void Ensure(int volumeId); + + bool Cleanup(Volume volume); + + //bool CleanupAll(); + + string GetCachePath(int volumeId); + + + } +} \ No newline at end of file diff --git a/API/Interfaces/IDirectoryService.cs b/API/Interfaces/IDirectoryService.cs index 664f7a631..da328f502 100644 --- a/API/Interfaces/IDirectoryService.cs +++ b/API/Interfaces/IDirectoryService.cs @@ -1,4 +1,6 @@ using System.Collections.Generic; +using System.IO; +using System.IO.Compression; using System.Threading.Tasks; using API.DTOs; @@ -16,7 +18,7 @@ namespace API.Interfaces /// /// Lists out top-level files for a given directory. - /// TODO: Implement ability to provide a filter for file types + /// TODO: Implement ability to provide a filter for file types (done in another implementation on DirectoryService) /// /// Absolute path /// List of folder names @@ -34,6 +36,7 @@ namespace API.Interfaces /// Extracts an archive to a temp cache directory. Returns path to new directory. If temp cache directory already exists, /// will return that without performing an extraction. Returns empty string if there are any invalidations which would /// prevent operations to perform correctly (missing archivePath file, empty archive, etc). + /// Deprecated. /// /// A valid file to an archive file. /// Id of volume being extracted. @@ -42,11 +45,23 @@ namespace API.Interfaces /// /// Returns the path a volume would be extracted to. + /// Deprecated. /// /// /// string GetExtractPath(int volumeId); Task ReadImageAsync(string imagePath); + + /// + /// Extracts an archive to a temp cache directory. Returns path to new directory. If temp cache directory already exists, + /// will return that without performing an extraction. Returns empty string if there are any invalidations which would + /// prevent operations to perform correctly (missing archivePath file, empty archive, etc). + /// + /// A valid file to an archive file. + /// Path to extract to + /// + string ExtractArchive(string archivePath, string extractPath); + } } \ No newline at end of file diff --git a/API/Services/CacheService.cs b/API/Services/CacheService.cs new file mode 100644 index 000000000..3bc4778e5 --- /dev/null +++ b/API/Services/CacheService.cs @@ -0,0 +1,46 @@ +using System.IO; +using API.Entities; +using API.Interfaces; + +namespace API.Services +{ + public class CacheService : ICacheService + { + private readonly IDirectoryService _directoryService; + private readonly ISeriesRepository _seriesRepository; + + public CacheService(IDirectoryService directoryService, ISeriesRepository seriesRepository) + { + _directoryService = directoryService; + _seriesRepository = seriesRepository; + } + + public async void Ensure(int volumeId) + { + Volume volume = await _seriesRepository.GetVolumeAsync(volumeId); + foreach (var file in volume.Files) + { + var extractPath = GetCachePath(volumeId); + if (file.Chapter > 0) + { + extractPath = Path.Join(extractPath, file.Chapter + ""); + } + + _directoryService.ExtractArchive(file.FilePath, extractPath); + } + } + + public bool Cleanup(Volume volume) + { + throw new System.NotImplementedException(); + } + + public string GetCachePath(int volumeId) + { + // TODO: Make this an absolute path, no ..'s in it. + return Path.Join(Directory.GetCurrentDirectory(), $"../cache/{volumeId}/"); + } + + + } +} \ No newline at end of file diff --git a/API/Services/DirectoryService.cs b/API/Services/DirectoryService.cs index 53e311c57..7809aee7a 100644 --- a/API/Services/DirectoryService.cs +++ b/API/Services/DirectoryService.cs @@ -11,6 +11,7 @@ using System.Threading; using System.Threading.Tasks; using API.DTOs; using API.Entities; +using API.Extensions; using API.Interfaces; using API.IO; using API.Parser; @@ -43,11 +44,11 @@ namespace API.Services /// Regex version of search pattern (ie \.mp3|\.mp4) /// SearchOption to use, defaults to TopDirectoryOnly /// List of file paths - public static IEnumerable GetFiles(string path, + private static IEnumerable GetFiles(string path, string searchPatternExpression = "", SearchOption searchOption = SearchOption.TopDirectoryOnly) { - Regex reSearchPattern = new Regex(searchPatternExpression, RegexOptions.IgnoreCase); + var reSearchPattern = new Regex(searchPatternExpression, RegexOptions.IgnoreCase); return Directory.EnumerateFiles(path, "*", searchOption) .Where(file => reSearchPattern.IsMatch(Path.GetExtension(file))); @@ -89,9 +90,8 @@ namespace API.Services return; } - ConcurrentBag tempBag; ConcurrentBag newBag = new ConcurrentBag(); - if (_scannedSeries.TryGetValue(info.Series, out tempBag)) + if (_scannedSeries.TryGetValue(info.Series, out var tempBag)) { var existingInfos = tempBag.ToArray(); foreach (var existingInfo in existingInfos) @@ -118,7 +118,7 @@ namespace API.Services if (series == null) { - series = new Series() + series = new Series { Name = seriesName, OriginalName = seriesName, @@ -159,11 +159,10 @@ namespace API.Services { ICollection volumes = new List(); IList existingVolumes = _seriesRepository.GetVolumes(series.Id).ToList(); - Volume existingVolume = null; foreach (var info in infos) { - existingVolume = existingVolumes.SingleOrDefault(v => v.Name == info.Volumes); + var existingVolume = existingVolumes.SingleOrDefault(v => v.Name == info.Volumes); if (existingVolume != null) { var existingFile = existingVolume.Files.SingleOrDefault(f => f.FilePath == info.FullFilePath); @@ -177,14 +176,6 @@ namespace API.Services { existingVolume.Files.Add(CreateMangaFile(info)); } - // existingVolume.Files = new List() - // { - // new MangaFile() - // { - // FilePath = info.FullFilePath, - // Chapter = Int32.Parse(info.Chapters) - // } - // }; if (forceUpdate || existingVolume.CoverImage == null || existingVolumes.Count == 0) { @@ -301,13 +292,37 @@ namespace API.Services using ZipArchive archive = ZipFile.OpenRead(archivePath); - if (archive.Entries.Count <= 0) return ""; + if (!archive.HasFiles()) return ""; archive.ExtractToDirectory(extractPath); _logger.LogInformation($"Extracting archive to {extractPath}"); return extractPath; } + + public string ExtractArchive(string archivePath, string extractPath) + { + if (!File.Exists(archivePath) || !Parser.Parser.IsArchive(archivePath)) + { + _logger.LogError($"Archive {archivePath} could not be found."); + return ""; + } + + if (Directory.Exists(extractPath)) + { + _logger.LogDebug($"Archive {archivePath} has already been extracted. Returning existing folder."); + return extractPath; + } + + using ZipArchive archive = ZipFile.OpenRead(archivePath); + + if (!archive.HasFiles()) return ""; + + archive.ExtractToDirectory(extractPath); + _logger.LogDebug($"Extracting archive to {extractPath}"); + + return extractPath; + } private int GetNumberOfPagesFromArchive(string archivePath) { @@ -320,6 +335,7 @@ namespace API.Services using ZipArchive archive = ZipFile.OpenRead(archivePath); return archive.Entries.Count(e => Parser.Parser.IsImage(e.FullName)); } + public async Task ReadImageAsync(string imagePath) { @@ -347,7 +363,6 @@ namespace API.Services { //Count of files traversed and timer for diagnostic output int fileCount = 0; - //var sw = Stopwatch.StartNew(); // Determine whether to parallelize file processing on each folder based on processor count. int procCount = Environment.ProcessorCount; @@ -434,9 +449,6 @@ namespace API.Services foreach (string str in subDirs) dirs.Push(str); } - - // For diagnostic purposes. - //Console.WriteLine("Processed {0} files in {1} milliseconds", fileCount, sw.ElapsedMilliseconds); } }