diff --git a/API/API.csproj b/API/API.csproj index d9bf4fb47..cdd2f9198 100644 --- a/API/API.csproj +++ b/API/API.csproj @@ -8,6 +8,9 @@ + + + @@ -25,4 +28,8 @@ + + + + diff --git a/API/Controllers/AccountController.cs b/API/Controllers/AccountController.cs index 3610925e9..3c25974ef 100644 --- a/API/Controllers/AccountController.cs +++ b/API/Controllers/AccountController.cs @@ -49,8 +49,6 @@ namespace API.Controllers if (!result.Succeeded) return BadRequest(result.Errors); - - // TODO: Need a way to store Roles in enum and configure from there var role = registerDto.IsAdmin ? PolicyConstants.AdminRole : PolicyConstants.PlebRole; var roleResult = await _userManager.AddToRoleAsync(user, role); diff --git a/API/Controllers/AdminController.cs b/API/Controllers/AdminController.cs index 173961a48..fa495b62e 100644 --- a/API/Controllers/AdminController.cs +++ b/API/Controllers/AdminController.cs @@ -1,9 +1,6 @@ -using System.Collections.Generic; -using System.Threading.Tasks; -using API.DTOs; +using System.Threading.Tasks; using API.Entities; using API.Interfaces; -using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Identity; using Microsoft.AspNetCore.Mvc; diff --git a/API/Controllers/LibraryController.cs b/API/Controllers/LibraryController.cs index a20bc1cda..82e88afb0 100644 --- a/API/Controllers/LibraryController.cs +++ b/API/Controllers/LibraryController.cs @@ -1,14 +1,13 @@ -using System.Collections.Generic; -using System.Collections.Immutable; +using System; +using System.Collections.Generic; using System.IO; -using System.Linq; using System.Threading.Tasks; using API.Data; using API.DTOs; using API.Entities; -using API.Extensions; using API.Interfaces; using AutoMapper; +using Hangfire; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Logging; @@ -18,23 +17,23 @@ namespace API.Controllers [Authorize] public class LibraryController : BaseApiController { - private readonly DataContext _context; private readonly IDirectoryService _directoryService; private readonly ILibraryRepository _libraryRepository; private readonly ILogger _logger; private readonly IUserRepository _userRepository; private readonly IMapper _mapper; + private readonly ITaskScheduler _taskScheduler; - public LibraryController(DataContext context, IDirectoryService directoryService, + public LibraryController(IDirectoryService directoryService, ILibraryRepository libraryRepository, ILogger logger, IUserRepository userRepository, - IMapper mapper) + IMapper mapper, ITaskScheduler taskScheduler) { - _context = context; _directoryService = directoryService; _libraryRepository = libraryRepository; _logger = logger; _userRepository = userRepository; _mapper = mapper; + _taskScheduler = taskScheduler; } /// @@ -84,5 +83,16 @@ namespace API.Controllers return BadRequest("Not Implemented"); } + + [Authorize(Policy = "RequireAdminRole")] + [HttpGet("scan")] + public async Task ScanLibrary(int libraryId) + { + var library = await _libraryRepository.GetLibraryForIdAsync(libraryId); + + BackgroundJob.Enqueue(() => _directoryService.ScanLibrary(library)); + + return Ok(); + } } } \ No newline at end of file diff --git a/API/Controllers/UsersController.cs b/API/Controllers/UsersController.cs index 28d1a3916..6102cc5e5 100644 --- a/API/Controllers/UsersController.cs +++ b/API/Controllers/UsersController.cs @@ -14,13 +14,11 @@ namespace API.Controllers [Authorize] public class UsersController : BaseApiController { - private readonly DataContext _context; private readonly IUserRepository _userRepository; private readonly ILibraryRepository _libraryRepository; - public UsersController(DataContext context, IUserRepository userRepository, ILibraryRepository libraryRepository) + public UsersController(IUserRepository userRepository, ILibraryRepository libraryRepository) { - _context = context; _userRepository = userRepository; _libraryRepository = libraryRepository; } @@ -43,7 +41,7 @@ namespace API.Controllers // TODO: We probably need to clean the folders before we insert var library = new Library { - Name = createLibraryDto.Name, // TODO: Ensure code handles Library name always being lowercase + Name = createLibraryDto.Name.ToLower(), Type = createLibraryDto.Type, AppUsers = new List() { user } }; diff --git a/API/DTOs/CreateLibraryDto.cs b/API/DTOs/CreateLibraryDto.cs index 3e3263d48..5dcdbf4e3 100644 --- a/API/DTOs/CreateLibraryDto.cs +++ b/API/DTOs/CreateLibraryDto.cs @@ -1,8 +1,6 @@ using System.Collections.Generic; using System.ComponentModel.DataAnnotations; -using System.Text.Json.Serialization; using API.Entities; -using Microsoft.EntityFrameworkCore.Storage.ValueConversion.Internal; namespace API.DTOs { diff --git a/API/DTOs/MemberDto.cs b/API/DTOs/MemberDto.cs index 6f09f1fc3..b404af389 100644 --- a/API/DTOs/MemberDto.cs +++ b/API/DTOs/MemberDto.cs @@ -1,7 +1,5 @@ using System; -using System.Collections; using System.Collections.Generic; -using API.Entities; namespace API.DTOs { diff --git a/API/Data/DataContext.cs b/API/Data/DataContext.cs index e29f5758a..9772251b7 100644 --- a/API/Data/DataContext.cs +++ b/API/Data/DataContext.cs @@ -1,5 +1,4 @@ -using System; -using API.Entities; +using API.Entities; using Microsoft.AspNetCore.Identity; using Microsoft.AspNetCore.Identity.EntityFrameworkCore; using Microsoft.EntityFrameworkCore; diff --git a/API/Data/LibraryRepository.cs b/API/Data/LibraryRepository.cs index 68e6371bd..b7bd978a1 100644 --- a/API/Data/LibraryRepository.cs +++ b/API/Data/LibraryRepository.cs @@ -37,6 +37,15 @@ namespace API.Data .Include(f => f.Folders) .ProjectTo(_mapper.ConfigurationProvider).ToListAsync(); } + + public async Task GetLibraryForIdAsync(int libraryId) + { + return await _context.Library + .Where(x => x.Id == libraryId) + .Include(f => f.Folders) + .ProjectTo(_mapper.ConfigurationProvider).SingleAsync(); + } + public async Task LibraryExists(string libraryName) { diff --git a/API/Data/Migrations/20201224155621_MiscCleanup.cs b/API/Data/Migrations/20201224155621_MiscCleanup.cs index 20e0a4dc9..78e66aea8 100644 --- a/API/Data/Migrations/20201224155621_MiscCleanup.cs +++ b/API/Data/Migrations/20201224155621_MiscCleanup.cs @@ -1,5 +1,4 @@ -using System; -using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Migrations; namespace API.Data.Migrations { diff --git a/API/Data/Migrations/DataContextModelSnapshot.cs b/API/Data/Migrations/DataContextModelSnapshot.cs index fd6f137ac..b7088bacf 100644 --- a/API/Data/Migrations/DataContextModelSnapshot.cs +++ b/API/Data/Migrations/DataContextModelSnapshot.cs @@ -1,9 +1,7 @@ // using System; -using API.Data; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Infrastructure; -using Microsoft.EntityFrameworkCore.Storage.ValueConversion; namespace API.Data.Migrations { diff --git a/API/Data/Seed.cs b/API/Data/Seed.cs index cd90243c4..8d7f619c3 100644 --- a/API/Data/Seed.cs +++ b/API/Data/Seed.cs @@ -12,8 +12,8 @@ namespace API.Data { var roles = new List { - new AppRole {Name = PolicyConstants.AdminRole}, - new AppRole {Name = PolicyConstants.PlebRole} + new() {Name = PolicyConstants.AdminRole}, + new() {Name = PolicyConstants.PlebRole} }; foreach (var role in roles) diff --git a/API/Extensions/ApplicationServiceExtensions.cs b/API/Extensions/ApplicationServiceExtensions.cs index 454c7c2f2..08077b5a1 100644 --- a/API/Extensions/ApplicationServiceExtensions.cs +++ b/API/Extensions/ApplicationServiceExtensions.cs @@ -3,6 +3,8 @@ using API.Helpers; using API.Interfaces; using API.Services; using AutoMapper; +using Hangfire; +using Hangfire.LiteDB; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; @@ -14,6 +16,7 @@ namespace API.Extensions public static IServiceCollection AddApplicationServices(this IServiceCollection services, IConfiguration config) { services.AddAutoMapper(typeof(AutoMapperProfiles).Assembly); + services.AddScoped(); services.AddScoped(); services.AddScoped(); services.AddScoped(); @@ -23,6 +26,14 @@ namespace API.Extensions options.UseSqlite(config.GetConnectionString("DefaultConnection")); }); + services.AddHangfire(configuration => configuration + .UseSimpleAssemblyNameTypeSerializer() + .UseRecommendedSerializerSettings() + .UseLiteDbStorage()); + + // Add the processing server as IHostedService + services.AddHangfireServer(); + return services; } } diff --git a/API/Hangfire-log.db b/API/Hangfire-log.db new file mode 100644 index 000000000..d8fc774c2 Binary files /dev/null and b/API/Hangfire-log.db differ diff --git a/API/Hangfire.db b/API/Hangfire.db new file mode 100644 index 000000000..db5987848 Binary files /dev/null and b/API/Hangfire.db differ diff --git a/API/Helpers/AutoMapperProfiles.cs b/API/Helpers/AutoMapperProfiles.cs index 4e203ab04..08e16b68a 100644 --- a/API/Helpers/AutoMapperProfiles.cs +++ b/API/Helpers/AutoMapperProfiles.cs @@ -1,5 +1,4 @@ -using System; -using System.Linq; +using System.Linq; using API.DTOs; using API.Entities; using AutoMapper; diff --git a/API/Interfaces/IDirectoryService.cs b/API/Interfaces/IDirectoryService.cs index 87a2b9f98..818aa9451 100644 --- a/API/Interfaces/IDirectoryService.cs +++ b/API/Interfaces/IDirectoryService.cs @@ -1,10 +1,12 @@ using System.Collections.Generic; -using System.Threading.Tasks; +using API.DTOs; namespace API.Interfaces { public interface IDirectoryService { IEnumerable ListDirectory(string rootPath); + + void ScanLibrary(LibraryDto library); } } \ No newline at end of file diff --git a/API/Interfaces/ILibraryRepository.cs b/API/Interfaces/ILibraryRepository.cs index 3f929efda..ae2cf88d8 100644 --- a/API/Interfaces/ILibraryRepository.cs +++ b/API/Interfaces/ILibraryRepository.cs @@ -17,6 +17,6 @@ namespace API.Interfaces /// Task LibraryExists(string libraryName); - Task> GetLibrariesForUserAsync(AppUser user); + public Task GetLibraryForIdAsync(int libraryId); } } \ No newline at end of file diff --git a/API/Interfaces/ITaskScheduler.cs b/API/Interfaces/ITaskScheduler.cs new file mode 100644 index 000000000..7f0a6312b --- /dev/null +++ b/API/Interfaces/ITaskScheduler.cs @@ -0,0 +1,7 @@ +namespace API.Interfaces +{ + public interface ITaskScheduler + { + + } +} \ No newline at end of file diff --git a/API/Interfaces/IUserRepository.cs b/API/Interfaces/IUserRepository.cs index 601913c89..31be6e52e 100644 --- a/API/Interfaces/IUserRepository.cs +++ b/API/Interfaces/IUserRepository.cs @@ -1,5 +1,4 @@ -using System.Collections; -using System.Collections.Generic; +using System.Collections.Generic; using System.Threading.Tasks; using API.DTOs; using API.Entities; diff --git a/API/Services/DirectoryService.cs b/API/Services/DirectoryService.cs index 49de3db48..8f09750b2 100644 --- a/API/Services/DirectoryService.cs +++ b/API/Services/DirectoryService.cs @@ -1,23 +1,35 @@ using System; using System.Collections.Generic; using System.Collections.Immutable; +using System.Diagnostics; using System.IO; using System.Linq; +using System.Security; +using System.Threading; using System.Threading.Tasks; +using API.DTOs; using API.Interfaces; +using Microsoft.Extensions.Logging; namespace API.Services { public class DirectoryService : IDirectoryService { - /// + private readonly ILogger _logger; + + public DirectoryService(ILogger logger) + { + _logger = logger; + } + + /// /// Lists out top-level folders for a given directory. Filters out System and Hidden folders. /// /// Absolute path /// List of folder names public IEnumerable ListDirectory(string rootPath) { - // TODO: Put some checks in here along with API to ensure that we aren't passed a file, folder exists, etc. + if (!Directory.Exists(rootPath)) return ImmutableList.Empty; var di = new DirectoryInfo(rootPath); var dirs = di.GetDirectories() @@ -27,5 +39,132 @@ namespace API.Services return dirs; } + + public void ScanLibrary(LibraryDto library) + { + foreach (var folderPath in library.Folders) + { + try { + TraverseTreeParallelForEach(folderPath, (f) => + { + // Exceptions are no-ops. + try { + // Do nothing with the data except read it. + //byte[] data = File.ReadAllBytes(f); + ProcessManga(f); + } + catch (FileNotFoundException) {} + catch (IOException) {} + catch (UnauthorizedAccessException) {} + catch (SecurityException) {} + // Display the filename. + Console.WriteLine(f); + }); + } + catch (ArgumentException) { + Console.WriteLine(@"The directory 'C:\Program Files' does not exist."); + } + } + } + + private static void ProcessManga(string filename) + { + Console.WriteLine($"Found {filename}"); + } + + public static void TraverseTreeParallelForEach(string root, Action action) + { + //Count of files traversed and timer for diagnostic output + int fileCount = 0; + var sw = Stopwatch.StartNew(); + + // Determine whether to parallelize file processing on each folder based on processor count. + int procCount = System.Environment.ProcessorCount; + + // Data structure to hold names of subfolders to be examined for files. + Stack dirs = new Stack(); + + if (!Directory.Exists(root)) { + throw new ArgumentException(); + } + dirs.Push(root); + + while (dirs.Count > 0) { + string currentDir = dirs.Pop(); + string[] subDirs = {}; + string[] files = {}; + + try { + subDirs = Directory.GetDirectories(currentDir); + } + // Thrown if we do not have discovery permission on the directory. + catch (UnauthorizedAccessException e) { + Console.WriteLine(e.Message); + continue; + } + // Thrown if another process has deleted the directory after we retrieved its name. + catch (DirectoryNotFoundException e) { + Console.WriteLine(e.Message); + continue; + } + + try { + files = Directory.GetFiles(currentDir); + } + catch (UnauthorizedAccessException e) { + Console.WriteLine(e.Message); + continue; + } + catch (DirectoryNotFoundException e) { + Console.WriteLine(e.Message); + continue; + } + catch (IOException e) { + Console.WriteLine(e.Message); + continue; + } + + // Execute in parallel if there are enough files in the directory. + // Otherwise, execute sequentially.Files are opened and processed + // synchronously but this could be modified to perform async I/O. + try { + if (files.Length < procCount) { + foreach (var file in files) { + action(file); + fileCount++; + } + } + else { + Parallel.ForEach(files, () => 0, (file, loopState, localCount) => + { action(file); + return ++localCount; + }, + (c) => { + Interlocked.Add(ref fileCount, c); + }); + } + } + catch (AggregateException ae) { + ae.Handle((ex) => { + if (ex is UnauthorizedAccessException) { + // Here we just output a message and go on. + Console.WriteLine(ex.Message); + return true; + } + // Handle other exceptions here if necessary... + + return false; + }); + } + + // Push the subdirectories onto the stack for traversal. + // This could also be done before handing the files. + foreach (string str in subDirs) + dirs.Push(str); + } + + // For diagnostic purposes. + Console.WriteLine("Processed {0} files in {1} milliseconds", fileCount, sw.ElapsedMilliseconds); + } } } \ No newline at end of file diff --git a/API/Services/TaskScheduler.cs b/API/Services/TaskScheduler.cs new file mode 100644 index 000000000..b76d10e40 --- /dev/null +++ b/API/Services/TaskScheduler.cs @@ -0,0 +1,17 @@ +using API.Interfaces; +using Hangfire; + +namespace API.Services +{ + public class TaskScheduler : ITaskScheduler + { + private readonly BackgroundJobServer _client; + + public TaskScheduler() + { + _client = new BackgroundJobServer(); + } + + + } +} \ No newline at end of file diff --git a/API/Startup.cs b/API/Startup.cs index c98d4064d..4ad56e85a 100644 --- a/API/Startup.cs +++ b/API/Startup.cs @@ -1,5 +1,7 @@ +using System; using API.Extensions; using API.Middleware; +using Hangfire; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; using Microsoft.Extensions.Configuration; @@ -33,7 +35,7 @@ namespace API } // This method gets called by the runtime. Use this method to configure the HTTP request pipeline. - public void Configure(IApplicationBuilder app, IWebHostEnvironment env) + public void Configure(IApplicationBuilder app, IBackgroundJobClient backgroundJobs, IWebHostEnvironment env) { app.UseMiddleware(); @@ -42,6 +44,9 @@ namespace API app.UseSwagger(); app.UseSwaggerUI(c => c.SwaggerEndpoint("/swagger/v1/swagger.json", "API v1")); } + + app.UseHangfireDashboard(); + backgroundJobs.Enqueue(() => Console.WriteLine("Hello world from Hangfire!")); app.UseHttpsRedirection(); @@ -57,6 +62,7 @@ namespace API app.UseEndpoints(endpoints => { endpoints.MapControllers(); + endpoints.MapHangfireDashboard(); }); } } diff --git a/API/appsettings.Development.json b/API/appsettings.Development.json index 280e18a03..5d8c460c5 100644 --- a/API/appsettings.Development.json +++ b/API/appsettings.Development.json @@ -1,13 +1,15 @@ { "ConnectionStrings": { - "DefaultConnection": "Data source=kavita.db" + "DefaultConnection": "Data source=kavita.db", + "HangfireConnection": "Data source=hangfire.db" }, "TokenKey": "super secret unguessable key", "Logging": { "LogLevel": { "Default": "Debug", "Microsoft": "Information", - "Microsoft.Hosting.Lifetime": "Information" + "Microsoft.Hosting.Lifetime": "Information", + "Hangfire": "Information" } } }