using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using API.Data;
using API.DTOs.Reader;
using API.Entities;
using API.Entities.Enums;
using API.Extensions;
using Kavita.Common;
using Microsoft.Extensions.Logging;
using NetVips;
namespace API.Services;
#nullable enable
public interface ICacheService
{
///
/// Ensures the cache is created for the given chapter and if not, will create it. Should be called before any other
/// cache operations (except cleanup).
///
///
/// Extracts a PDF into images for a different reading experience
/// Chapter for the passed chapterId. Side-effect from ensuring cache.
Task Ensure(int chapterId, bool extractPdfToImages = false);
///
/// Clears cache directory of all volumes. This can be invoked from deleting a library or a series.
///
/// Volumes that belong to that library. Assume the library might have been deleted before this invocation.
void CleanupChapters(IEnumerable chapterIds);
void CleanupBookmarks(IEnumerable seriesIds);
string GetCachedPagePath(int chapterId, int page);
string GetCachePath(int chapterId);
string GetBookmarkCachePath(int seriesId);
IEnumerable GetCachedPages(int chapterId);
IEnumerable GetCachedFileDimensions(string cachePath);
string GetCachedBookmarkPagePath(int seriesId, int page);
string GetCachedFile(Chapter chapter);
public void ExtractChapterFiles(string extractPath, IReadOnlyList files, bool extractPdfImages = false);
Task CacheBookmarkForSeries(int userId, int seriesId);
void CleanupBookmarkCache(int seriesId);
}
public class CacheService : ICacheService
{
private readonly ILogger _logger;
private readonly IUnitOfWork _unitOfWork;
private readonly IDirectoryService _directoryService;
private readonly IReadingItemService _readingItemService;
private readonly IBookmarkService _bookmarkService;
private static readonly ConcurrentDictionary ExtractLocks = new();
public CacheService(ILogger logger, IUnitOfWork unitOfWork,
IDirectoryService directoryService, IReadingItemService readingItemService,
IBookmarkService bookmarkService)
{
_logger = logger;
_unitOfWork = unitOfWork;
_directoryService = directoryService;
_readingItemService = readingItemService;
_bookmarkService = bookmarkService;
}
public IEnumerable GetCachedPages(int chapterId)
{
var path = GetCachePath(chapterId);
return _directoryService.GetFilesWithExtension(path, Tasks.Scanner.Parser.Parser.ImageFileExtensions)
.OrderByNatural(Path.GetFileNameWithoutExtension);
}
///
/// For a given path, scan all files (in reading order) and generate File Dimensions for it. Path must exist
///
///
///
public IEnumerable GetCachedFileDimensions(string cachePath)
{
var sw = Stopwatch.StartNew();
var files = _directoryService.GetFilesWithExtension(cachePath, Tasks.Scanner.Parser.Parser.ImageFileExtensions)
.OrderByNatural(Path.GetFileNameWithoutExtension)
.ToArray();
if (files.Length == 0)
{
return ArraySegment.Empty;
}
var dimensions = new List();
var originalCacheSize = Cache.MaxFiles;
try
{
Cache.MaxFiles = 0;
for (var i = 0; i < files.Length; i++)
{
var file = files[i];
using var image = Image.NewFromFile(file, memory: false, access: Enums.Access.SequentialUnbuffered);
dimensions.Add(new FileDimensionDto()
{
PageNumber = i,
Height = image.Height,
Width = image.Width,
IsWide = image.Width > image.Height,
FileName = file.Replace(cachePath, string.Empty)
});
}
}
catch (Exception ex)
{
_logger.LogError(ex, "There was an error calculating image dimensions for {CachePath}", cachePath);
}
finally
{
Cache.MaxFiles = originalCacheSize;
}
_logger.LogDebug("File Dimensions call for {Length} images took {Time}ms", dimensions.Count, sw.ElapsedMilliseconds);
return dimensions;
}
public string GetCachedBookmarkPagePath(int seriesId, int page)
{
// Calculate what chapter the page belongs to
var path = GetBookmarkCachePath(seriesId);
var files = _directoryService.GetFilesWithExtension(path, Tasks.Scanner.Parser.Parser.ImageFileExtensions);
files = files
.AsEnumerable()
.OrderByNatural(Path.GetFileNameWithoutExtension)
.ToArray();
if (files.Length == 0)
{
return string.Empty;
}
// Since array is 0 based, we need to keep that in account (only affects last image)
return page == files.Length ? files[page - 1] : files[page];
}
///
/// Returns the full path to the cached file. If the file does not exist, will fallback to the original.
///
///
///
public string GetCachedFile(Chapter chapter)
{
var extractPath = GetCachePath(chapter.Id);
var path = Path.Join(extractPath, _directoryService.FileSystem.Path.GetFileName(chapter.Files.First().FilePath));
if (!(_directoryService.FileSystem.FileInfo.New(path).Exists))
{
path = chapter.Files.First().FilePath;
}
return path;
}
///
/// Caches the files for the given chapter to CacheDirectory
///
///
/// Defaults to false. Extract pdf file into images rather than copying just the pdf file
/// This will always return the Chapter for the chapterId
public async Task Ensure(int chapterId, bool extractPdfToImages = false)
{
_directoryService.ExistOrCreate(_directoryService.CacheDirectory);
var chapter = await _unitOfWork.ChapterRepository.GetChapterAsync(chapterId);
var extractPath = GetCachePath(chapterId);
SemaphoreSlim extractLock = ExtractLocks.GetOrAdd(chapterId, id => new SemaphoreSlim(1,1));
await extractLock.WaitAsync();
try {
if(_directoryService.Exists(extractPath)) return chapter;
var files = chapter?.Files.ToList();
ExtractChapterFiles(extractPath, files, extractPdfToImages);
} finally {
extractLock.Release();
}
return chapter;
}
///
/// This is an internal method for cache service for extracting chapter files to disk. The code is structured
/// for cache service, but can be re-used (download bookmarks)
///
///
///
/// Defaults to false, if true, will extract the images from the PDF renderer and not move the pdf file
///
public void ExtractChapterFiles(string extractPath, IReadOnlyList? files, bool extractPdfImages = false)
{
if (files == null) return;
var removeNonImages = true;
var fileCount = files.Count;
var extraPath = string.Empty;
var extractDi = _directoryService.FileSystem.DirectoryInfo.New(extractPath);
if (files.Count > 0 && files[0].Format == MangaFormat.Image)
{
// Check if all the files are Images. If so, do a directory copy, else do the normal copy
if (files.All(f => f.Format == MangaFormat.Image))
{
_directoryService.ExistOrCreate(extractPath);
_directoryService.CopyFilesToDirectory(files.Select(f => f.FilePath), extractPath);
}
else
{
foreach (var file in files)
{
if (fileCount > 1)
{
extraPath = file.Id + string.Empty;
}
_readingItemService.Extract(file.FilePath, Path.Join(extractPath, extraPath), MangaFormat.Image, files.Count);
}
_directoryService.Flatten(extractDi.FullName);
}
}
foreach (var file in files)
{
if (fileCount > 1)
{
extraPath = file.Id + string.Empty;
}
switch (file.Format)
{
case MangaFormat.Archive:
_readingItemService.Extract(file.FilePath, Path.Join(extractPath, extraPath), file.Format);
break;
case MangaFormat.Epub:
case MangaFormat.Pdf:
{
if (!_directoryService.FileSystem.File.Exists(files[0].FilePath))
{
_logger.LogError("{File} does not exist on disk", files[0].FilePath);
throw new KavitaException($"{files[0].FilePath} does not exist on disk");
}
if (extractPdfImages)
{
_readingItemService.Extract(file.FilePath, Path.Join(extractPath, extraPath), file.Format);
break;
}
removeNonImages = false;
_directoryService.ExistOrCreate(extractPath);
_directoryService.CopyFileToDirectory(files[0].FilePath, extractPath);
break;
}
}
}
_directoryService.Flatten(extractDi.FullName);
if (removeNonImages)
{
_directoryService.RemoveNonImages(extractDi.FullName);
}
}
///
/// Removes the cached files and folders for a set of chapterIds
///
///
public void CleanupChapters(IEnumerable chapterIds)
{
foreach (var chapter in chapterIds)
{
_directoryService.ClearAndDeleteDirectory(GetCachePath(chapter));
}
}
///
/// Removes the cached files and folders for a set of chapterIds
///
///
public void CleanupBookmarks(IEnumerable seriesIds)
{
foreach (var series in seriesIds)
{
_directoryService.ClearAndDeleteDirectory(GetBookmarkCachePath(series));
}
}
///
/// Returns the cache path for a given Chapter. Should be cacheDirectory/{chapterId}/
///
///
///
public string GetCachePath(int chapterId)
{
return _directoryService.FileSystem.Path.GetFullPath(_directoryService.FileSystem.Path.Join(_directoryService.CacheDirectory, $"{chapterId}/"));
}
///
/// Returns the cache path for a given series' bookmarks. Should be cacheDirectory/{seriesId_bookmarks}/
///
///
///
public string GetBookmarkCachePath(int seriesId)
{
return _directoryService.FileSystem.Path.GetFullPath(_directoryService.FileSystem.Path.Join(_directoryService.CacheDirectory, $"{seriesId}_bookmarks/"));
}
///
/// Returns the absolute path of a cached page.
///
/// Chapter id with Files populated.
/// Page number to look for
/// Page filepath or empty if no files found.
public string GetCachedPagePath(int chapterId, int page)
{
// Calculate what chapter the page belongs to
var path = GetCachePath(chapterId);
// NOTE: We can optimize this by extracting and renaming, so we don't need to scan for the files and can do a direct access
var files = _directoryService.GetFilesWithExtension(path, Tasks.Scanner.Parser.Parser.ImageFileExtensions)
//.OrderByNatural(Path.GetFileNameWithoutExtension) // This is already done in GetPageFromFiles
.ToArray();
return GetPageFromFiles(files, page);
}
public async Task CacheBookmarkForSeries(int userId, int seriesId)
{
var destDirectory = _directoryService.FileSystem.Path.Join(_directoryService.CacheDirectory, seriesId + "_bookmarks");
if (_directoryService.Exists(destDirectory)) return _directoryService.GetFiles(destDirectory).Count();
var bookmarkDtos = await _unitOfWork.UserRepository.GetBookmarkDtosForSeries(userId, seriesId);
var files = (await _bookmarkService.GetBookmarkFilesById(bookmarkDtos.Select(b => b.Id))).ToList();
_directoryService.CopyFilesToDirectory(files, destDirectory,
Enumerable.Range(1, files.Count).Select(i => i + string.Empty).ToList());
return files.Count;
}
///
/// Clears a cached bookmarks for a series id folder
///
///
public void CleanupBookmarkCache(int seriesId)
{
var destDirectory = _directoryService.FileSystem.Path.Join(_directoryService.CacheDirectory, seriesId + "_bookmarks");
if (!_directoryService.Exists(destDirectory)) return;
_directoryService.ClearAndDeleteDirectory(destDirectory);
}
///
/// Returns either the file or an empty string
///
///
///
///
public static string GetPageFromFiles(string[] files, int pageNum)
{
files = files
.AsEnumerable()
.OrderByNatural(Path.GetFileNameWithoutExtension)
.ToArray();
if (files.Length == 0)
{
return string.Empty;
}
if (pageNum < 0)
{
pageNum = 0;
}
// Since array is 0 based, we need to keep that in account (only affects last image)
return pageNum >= files.Length ? files[Math.Min(pageNum - 1, files.Length - 1)] : files[pageNum];
}
}