using System; using System.Collections; using System.Collections.Generic; using System.Collections.Immutable; using System.IO; using System.IO.Abstractions; using System.Linq; using System.Text.RegularExpressions; using System.Threading.Tasks; using API.Comparators; using Microsoft.Extensions.Logging; namespace API.Services { public interface IDirectoryService { IFileSystem FileSystem { get; } string CacheDirectory { get; } string CoverImageDirectory { get; } string LogDirectory { get; } string TempDirectory { get; } string ConfigDirectory { get; } /// /// Original BookmarkDirectory. Only used for resetting directory. Use for actual path. /// string BookmarkDirectory { get; } /// /// Lists out top-level folders for a given directory. Filters out System and Hidden folders. /// /// Absolute path of directory to scan. /// List of folder names IEnumerable ListDirectory(string rootPath); Task ReadFileAsync(string path); bool CopyFilesToDirectory(IEnumerable filePaths, string directoryPath, string prepend = ""); bool Exists(string directory); void CopyFileToDirectory(string fullFilePath, string targetDirectory); int TraverseTreeParallelForEach(string root, Action action, string searchPattern, ILogger logger); bool IsDriveMounted(string path); bool IsDirectoryEmpty(string path); long GetTotalSize(IEnumerable paths); void ClearDirectory(string directoryPath); void ClearAndDeleteDirectory(string directoryPath); string[] GetFilesWithExtension(string path, string searchPatternExpression = ""); bool CopyDirectoryToDirectory(string sourceDirName, string destDirName, string searchPattern = ""); Dictionary FindHighestDirectoriesFromFiles(IEnumerable libraryFolders, IList filePaths); IEnumerable GetFoldersTillRoot(string rootPath, string fullPath); IEnumerable GetFiles(string path, string fileNameRegex = "", SearchOption searchOption = SearchOption.TopDirectoryOnly); bool ExistOrCreate(string directoryPath); void DeleteFiles(IEnumerable files); void RemoveNonImages(string directoryName); void Flatten(string directoryName); Task CheckWriteAccess(string directoryName); } public class DirectoryService : IDirectoryService { public IFileSystem FileSystem { get; } public string CacheDirectory { get; } public string CoverImageDirectory { get; } public string LogDirectory { get; } public string TempDirectory { get; } public string ConfigDirectory { get; } public string BookmarkDirectory { get; } private readonly ILogger _logger; private static readonly Regex ExcludeDirectories = new Regex( @"@eaDir|\.DS_Store", RegexOptions.Compiled | RegexOptions.IgnoreCase); public static readonly string BackupDirectory = Path.Join(Directory.GetCurrentDirectory(), "config", "backups"); public DirectoryService(ILogger logger, IFileSystem fileSystem) { _logger = logger; FileSystem = fileSystem; CoverImageDirectory = FileSystem.Path.Join(FileSystem.Directory.GetCurrentDirectory(), "config", "covers"); CacheDirectory = FileSystem.Path.Join(FileSystem.Directory.GetCurrentDirectory(), "config", "cache"); LogDirectory = FileSystem.Path.Join(FileSystem.Directory.GetCurrentDirectory(), "config", "logs"); TempDirectory = FileSystem.Path.Join(FileSystem.Directory.GetCurrentDirectory(), "config", "temp"); ConfigDirectory = FileSystem.Path.Join(FileSystem.Directory.GetCurrentDirectory(), "config"); BookmarkDirectory = FileSystem.Path.Join(FileSystem.Directory.GetCurrentDirectory(), "config", "bookmarks"); } /// /// Given a set of regex search criteria, get files in the given path. /// /// This will always exclude patterns /// Directory to search /// Regex version of search pattern (ie \.mp3|\.mp4). Defaults to * meaning all files. /// SearchOption to use, defaults to TopDirectoryOnly /// List of file paths private IEnumerable GetFilesWithCertainExtensions(string path, string searchPatternExpression = "", SearchOption searchOption = SearchOption.TopDirectoryOnly) { if (!FileSystem.Directory.Exists(path)) return ImmutableList.Empty; var reSearchPattern = new Regex(searchPatternExpression, RegexOptions.IgnoreCase); return FileSystem.Directory.EnumerateFiles(path, "*", searchOption) .Where(file => reSearchPattern.IsMatch(FileSystem.Path.GetExtension(file)) && !FileSystem.Path.GetFileName(file).StartsWith(Parser.Parser.MacOsMetadataFileStartsWith)); } /// /// Returns a list of folders from end of fullPath to rootPath. If a file is passed at the end of the fullPath, it will be ignored. /// /// Example) (C:/Manga/, C:/Manga/Love Hina/Specials/Omake/) returns [Omake, Specials, Love Hina] /// /// /// /// public IEnumerable GetFoldersTillRoot(string rootPath, string fullPath) { var separator = FileSystem.Path.AltDirectorySeparatorChar; if (fullPath.Contains(FileSystem.Path.DirectorySeparatorChar)) { fullPath = fullPath.Replace(FileSystem.Path.DirectorySeparatorChar, FileSystem.Path.AltDirectorySeparatorChar); } if (rootPath.Contains(Path.DirectorySeparatorChar)) { rootPath = rootPath.Replace(FileSystem.Path.DirectorySeparatorChar, FileSystem.Path.AltDirectorySeparatorChar); } var path = fullPath.EndsWith(separator) ? fullPath.Substring(0, fullPath.Length - 1) : fullPath; var root = rootPath.EndsWith(separator) ? rootPath.Substring(0, rootPath.Length - 1) : rootPath; var paths = new List(); // If a file is at the end of the path, remove it before we start processing folders if (FileSystem.Path.GetExtension(path) != string.Empty) { path = path.Substring(0, path.LastIndexOf(separator)); } while (FileSystem.Path.GetDirectoryName(path) != Path.GetDirectoryName(root)) { //var folder = new DirectoryInfo(path).Name; var folder = FileSystem.DirectoryInfo.FromDirectoryName(path).Name; paths.Add(folder); path = path.Substring(0, path.LastIndexOf(separator)); } return paths; } /// /// Does Directory Exist /// /// /// public bool Exists(string directory) { var di = FileSystem.DirectoryInfo.FromDirectoryName(directory); return di.Exists; } /// /// Get files given a path. /// /// This will automatically filter out restricted files, like MacOsMetadata files /// /// An optional regex string to search against. Will use file path to match against. /// Defaults to top level directory only, can be given all to provide recursive searching /// public IEnumerable GetFiles(string path, string fileNameRegex = "", SearchOption searchOption = SearchOption.TopDirectoryOnly) { // TODO: Refactor this and GetFilesWithCertainExtensions to use same implementation if (!FileSystem.Directory.Exists(path)) return ImmutableList.Empty; if (fileNameRegex != string.Empty) { var reSearchPattern = new Regex(fileNameRegex, RegexOptions.IgnoreCase); return FileSystem.Directory.EnumerateFiles(path, "*", searchOption) .Where(file => { var fileName = FileSystem.Path.GetFileName(file); return reSearchPattern.IsMatch(fileName) && !fileName.StartsWith(Parser.Parser.MacOsMetadataFileStartsWith); }); } return FileSystem.Directory.EnumerateFiles(path, "*", searchOption).Where(file => !FileSystem.Path.GetFileName(file).StartsWith(Parser.Parser.MacOsMetadataFileStartsWith)); } /// /// Copies a file into a directory. Does not maintain parent folder of file. /// Will create target directory if doesn't exist. Automatically overwrites what is there. /// /// /// public void CopyFileToDirectory(string fullFilePath, string targetDirectory) { try { var fileInfo = FileSystem.FileInfo.FromFileName(fullFilePath); if (fileInfo.Exists) { ExistOrCreate(targetDirectory); fileInfo.CopyTo(FileSystem.Path.Join(targetDirectory, fileInfo.Name), true); } } catch (Exception ex) { _logger.LogError(ex, "There was a critical error when copying {File} to {Directory}", fullFilePath, targetDirectory); } } /// /// Copies all files and subdirectories within a directory to a target location /// /// Directory to copy from. Does not copy the parent folder /// Destination to copy to. Will be created if doesn't exist /// Defaults to all files /// If was successful /// Thrown when source directory does not exist public bool CopyDirectoryToDirectory(string sourceDirName, string destDirName, string searchPattern = "") { if (string.IsNullOrEmpty(sourceDirName)) return false; // Get the subdirectories for the specified directory. var dir = FileSystem.DirectoryInfo.FromDirectoryName(sourceDirName); if (!dir.Exists) { throw new DirectoryNotFoundException( "Source directory does not exist or could not be found: " + sourceDirName); } var dirs = dir.GetDirectories(); // If the destination directory doesn't exist, create it. ExistOrCreate(destDirName); // Get the files in the directory and copy them to the new location. var files = GetFilesWithExtension(dir.FullName, searchPattern).Select(n => FileSystem.FileInfo.FromFileName(n)); foreach (var file in files) { var tempPath = FileSystem.Path.Combine(destDirName, file.Name); file.CopyTo(tempPath, false); } // If copying subdirectories, copy them and their contents to new location. foreach (var subDir in dirs) { var tempPath = FileSystem.Path.Combine(destDirName, subDir.Name); CopyDirectoryToDirectory(subDir.FullName, tempPath); } return true; } /// /// Checks if the root path of a path exists or not. /// /// /// public bool IsDriveMounted(string path) { return FileSystem.DirectoryInfo.FromDirectoryName(FileSystem.Path.GetPathRoot(path) ?? string.Empty).Exists; } /// /// Checks if the root path of a path is empty or not. /// /// /// public bool IsDirectoryEmpty(string path) { return FileSystem.Directory.Exists(path) && !FileSystem.Directory.EnumerateFileSystemEntries(path).Any(); } public string[] GetFilesWithExtension(string path, string searchPatternExpression = "") { // TODO: Use GitFiles instead if (searchPatternExpression != string.Empty) { return GetFilesWithCertainExtensions(path, searchPatternExpression).ToArray(); } return !FileSystem.Directory.Exists(path) ? Array.Empty() : FileSystem.Directory.GetFiles(path); } /// /// Returns the total number of bytes for a given set of full file paths /// /// /// Total bytes public long GetTotalSize(IEnumerable paths) { return paths.Sum(path => FileSystem.FileInfo.FromFileName(path).Length); } /// /// Returns true if the path exists and is a directory. If path does not exist, this will create it. Returns false in all fail cases. /// /// /// public bool ExistOrCreate(string directoryPath) { var di = FileSystem.DirectoryInfo.FromDirectoryName(directoryPath); if (di.Exists) return true; try { FileSystem.Directory.CreateDirectory(directoryPath); } catch (Exception) { return false; } return true; } /// /// Deletes all files within the directory, then the directory itself. /// /// public void ClearAndDeleteDirectory(string directoryPath) { if (!FileSystem.Directory.Exists(directoryPath)) return; var di = FileSystem.DirectoryInfo.FromDirectoryName(directoryPath); ClearDirectory(directoryPath); di.Delete(true); } /// /// Deletes all files and folders within the directory path /// /// /// public void ClearDirectory(string directoryPath) { var di = FileSystem.DirectoryInfo.FromDirectoryName(directoryPath); if (!di.Exists) return; foreach (var file in di.EnumerateFiles()) { file.Delete(); } foreach (var dir in di.EnumerateDirectories()) { dir.Delete(true); } } /// /// Copies files to a destination directory. If the destination directory doesn't exist, this will create it. /// /// /// /// An optional string to prepend to the target file's name /// public bool CopyFilesToDirectory(IEnumerable filePaths, string directoryPath, string prepend = "") { ExistOrCreate(directoryPath); string currentFile = null; try { foreach (var file in filePaths) { currentFile = file; var fileInfo = FileSystem.FileInfo.FromFileName(file); if (fileInfo.Exists) { fileInfo.CopyTo(FileSystem.Path.Join(directoryPath, prepend + fileInfo.Name)); } else { _logger.LogWarning("Tried to copy {File} but it doesn't exist", file); } } } catch (Exception ex) { _logger.LogError(ex, "Unable to copy {File} to {DirectoryPath}", currentFile, directoryPath); return false; } return true; } /// /// Lists all directories in a root path. Will exclude Hidden or System directories. /// /// /// public IEnumerable ListDirectory(string rootPath) { if (!FileSystem.Directory.Exists(rootPath)) return ImmutableList.Empty; var di = FileSystem.DirectoryInfo.FromDirectoryName(rootPath); var dirs = di.GetDirectories() .Where(dir => !(dir.Attributes.HasFlag(FileAttributes.Hidden) || dir.Attributes.HasFlag(FileAttributes.System))) .Select(d => d.Name).ToImmutableList(); return dirs; } /// /// Reads a file's into byte[]. Returns empty array if file doesn't exist. /// /// /// public async Task ReadFileAsync(string path) { if (!FileSystem.File.Exists(path)) return Array.Empty(); return await FileSystem.File.ReadAllBytesAsync(path); } /// /// Finds the highest directories from a set of file paths. Does not return the root path, will always select the highest non-root path. /// /// If the file paths do not contain anything from libraryFolders, this returns an empty dictionary back /// List of top level folders which files belong to /// List of file paths that belong to libraryFolders /// public Dictionary FindHighestDirectoriesFromFiles(IEnumerable libraryFolders, IList filePaths) { var stopLookingForDirectories = false; var dirs = new Dictionary(); foreach (var folder in libraryFolders) { if (stopLookingForDirectories) break; foreach (var file in filePaths) { if (!file.Contains(folder)) continue; var parts = GetFoldersTillRoot(folder, file).ToList(); if (parts.Count == 0) { // Break from all loops, we done, just scan folder.Path (library root) dirs.Add(folder, string.Empty); stopLookingForDirectories = true; break; } var fullPath = Path.Join(folder, parts.Last()); if (!dirs.ContainsKey(fullPath)) { dirs.Add(fullPath, string.Empty); } } } return dirs; } /// /// Recursively scans files and applies an action on them. This uses as many cores the underlying PC has to speed /// up processing. /// /// Directory to scan /// Action to apply on file path /// Regex pattern to search against /// /// public int TraverseTreeParallelForEach(string root, Action action, string searchPattern, ILogger logger) { //Count of files traversed and timer for diagnostic output var fileCount = 0; // Determine whether to parallelize file processing on each folder based on processor count. //var procCount = Environment.ProcessorCount; // Data structure to hold names of subfolders to be examined for files. var dirs = new Stack(); if (!FileSystem.Directory.Exists(root)) { throw new ArgumentException("The directory doesn't exist"); } dirs.Push(root); while (dirs.Count > 0) { var currentDir = dirs.Pop(); IEnumerable subDirs; string[] files; try { subDirs = FileSystem.Directory.GetDirectories(currentDir).Where(path => ExcludeDirectories.Matches(path).Count == 0); } // Thrown if we do not have discovery permission on the directory. catch (UnauthorizedAccessException e) { Console.WriteLine(e.Message); logger.LogError(e, "Unauthorized access on {Directory}", currentDir); continue; } // Thrown if another process has deleted the directory after we retrieved its name. catch (DirectoryNotFoundException e) { Console.WriteLine(e.Message); logger.LogError(e, "Directory not found on {Directory}", currentDir); continue; } try { // TODO: Replace this with GetFiles - It's the same code files = GetFilesWithCertainExtensions(currentDir, searchPattern) .ToArray(); } catch (UnauthorizedAccessException e) { Console.WriteLine(e.Message); continue; } catch (DirectoryNotFoundException e) { Console.WriteLine(e.Message); continue; } catch (IOException e) { Console.WriteLine(e.Message); continue; } // Execute in parallel if there are enough files in the directory. // Otherwise, execute sequentially. Files are opened and processed // synchronously but this could be modified to perform async I/O. try { // if (files.Length < procCount) { // foreach (var file in files) { // action(file); // fileCount++; // } // } // else { // Parallel.ForEach(files, () => 0, (file, _, localCount) => // { action(file); // return ++localCount; // }, // (c) => { // Interlocked.Add(ref fileCount, c); // }); // } foreach (var file in files) { action(file); fileCount++; } } catch (AggregateException ae) { ae.Handle((ex) => { if (ex is UnauthorizedAccessException) { // Here we just output a message and go on. Console.WriteLine(ex.Message); _logger.LogError(ex, "Unauthorized access on file"); return true; } // Handle other exceptions here if necessary... return false; }); } // Push the subdirectories onto the stack for traversal. // This could also be done before handing the files. foreach (var str in subDirs) dirs.Push(str); } return fileCount; } /// /// Attempts to delete the files passed to it. Swallows exceptions. /// /// Full path of files to delete public void DeleteFiles(IEnumerable files) { foreach (var file in files) { try { FileSystem.FileInfo.FromFileName(file).Delete(); } catch (Exception) { /* Swallow exception */ } } } /// /// Returns the human-readable file size for an arbitrary, 64-bit file size /// The default format is "0.## XB", e.g. "4.2 KB" or "1.43 GB" /// /// https://www.somacon.com/p576.php /// /// public static string GetHumanReadableBytes(long bytes) { // Get absolute value var absoluteBytes = (bytes < 0 ? -bytes : bytes); // Determine the suffix and readable value string suffix; double readable; switch (absoluteBytes) { // Exabyte case >= 0x1000000000000000: suffix = "EB"; readable = (bytes >> 50); break; // Petabyte case >= 0x4000000000000: suffix = "PB"; readable = (bytes >> 40); break; // Terabyte case >= 0x10000000000: suffix = "TB"; readable = (bytes >> 30); break; // Gigabyte case >= 0x40000000: suffix = "GB"; readable = (bytes >> 20); break; // Megabyte case >= 0x100000: suffix = "MB"; readable = (bytes >> 10); break; // Kilobyte case >= 0x400: suffix = "KB"; readable = bytes; break; default: return bytes.ToString("0 B"); // Byte } // Divide by 1024 to get fractional value readable = (readable / 1024); // Return formatted number with suffix return readable.ToString("0.## ") + suffix; } /// /// Removes all files except images from the directory. Includes sub directories. /// /// Fully qualified directory public void RemoveNonImages(string directoryName) { DeleteFiles(GetFiles(directoryName, searchOption:SearchOption.AllDirectories).Where(file => !Parser.Parser.IsImage(file))); } /// /// Flattens all files in subfolders to the passed directory recursively. /// /// /// foo /// ├── 1.txt /// ├── 2.txt /// ├── 3.txt /// ├── 4.txt /// └── bar /// ├── 1.txt /// ├── 2.txt /// └── 5.txt /// /// becomes: /// foo /// ├── 1.txt /// ├── 2.txt /// ├── 3.txt /// ├── 4.txt /// ├── bar_1.txt /// ├── bar_2.txt /// └── bar_5.txt /// /// Fully qualified Directory name public void Flatten(string directoryName) { if (string.IsNullOrEmpty(directoryName) || !FileSystem.Directory.Exists(directoryName)) return; var directory = FileSystem.DirectoryInfo.FromDirectoryName(directoryName); var index = 0; FlattenDirectory(directory, directory, ref index); } /// /// Checks whether a directory has write permissions /// /// Fully qualified path /// public async Task CheckWriteAccess(string directoryName) { try { ExistOrCreate(directoryName); await FileSystem.File.WriteAllTextAsync( FileSystem.Path.Join(directoryName, "test.txt"), string.Empty); } catch (Exception ex) { ClearAndDeleteDirectory(directoryName); return false; } ClearAndDeleteDirectory(directoryName); return true; } private void FlattenDirectory(IDirectoryInfo root, IDirectoryInfo directory, ref int directoryIndex) { if (!root.FullName.Equals(directory.FullName)) { var fileIndex = 1; using var nc = new NaturalSortComparer(); foreach (var file in directory.EnumerateFiles().OrderBy(file => file.FullName, nc)) { if (file.Directory == null) continue; var paddedIndex = Parser.Parser.PadZeros(directoryIndex + ""); // We need to rename the files so that after flattening, they are in the order we found them var newName = $"{paddedIndex}_{Parser.Parser.PadZeros(fileIndex + "")}{file.Extension}"; var newPath = Path.Join(root.FullName, newName); if (!File.Exists(newPath)) file.MoveTo(newPath); fileIndex++; } directoryIndex++; } var sort = new NaturalSortComparer(); foreach (var subDirectory in directory.EnumerateDirectories().OrderBy(d => d.FullName, sort)) { FlattenDirectory(root, subDirectory, ref directoryIndex); } } } }