Merge pull request #16 from Kareadita/feature/scan-library

Library Scanning + Task Scheduling

API/API.csproj

@@ -8,6 +8,9 @@
 
   <ItemGroup>
     <PackageReference Include="AutoMapper.Extensions.Microsoft.DependencyInjection" Version="8.1.0" />
+    <PackageReference Include="Hangfire" Version="1.7.18" />
+    <PackageReference Include="Hangfire.AspNetCore" Version="1.7.18" />
+    <PackageReference Include="Hangfire.LiteDB" Version="0.4.0" />
     <PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="5.0.1" NoWarn="NU1605" />
     <PackageReference Include="Microsoft.AspNetCore.Authentication.OpenIdConnect" Version="5.0.1" NoWarn="NU1605" />
     <PackageReference Include="Microsoft.AspNetCore.Identity.EntityFrameworkCore" Version="5.0.1" />
@@ -25,4 +28,8 @@
     <PackageReference Include="System.IdentityModel.Tokens.Jwt" Version="6.8.0" />
   </ItemGroup>
 
+  <ItemGroup>
+    <Folder Include="Tasks" />
+  </ItemGroup>
+
 </Project>

API/Controllers/AccountController.cs

@@ -49,8 +49,6 @@ namespace API.Controllers
 
             if (!result.Succeeded) return BadRequest(result.Errors);
             
-
-            // TODO: Need a way to store Roles in enum and configure from there
             var role = registerDto.IsAdmin ? PolicyConstants.AdminRole : PolicyConstants.PlebRole;
             var roleResult = await _userManager.AddToRoleAsync(user, role);
 

API/Controllers/AdminController.cs

@@ -1,9 +1,6 @@
-using System.Collections.Generic;
+using System.Threading.Tasks;
-using System.Threading.Tasks;
-using API.DTOs;
 using API.Entities;
 using API.Interfaces;
-using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.Identity;
 using Microsoft.AspNetCore.Mvc;
 

API/Controllers/LibraryController.cs

@@ -1,14 +1,13 @@
-using System.Collections.Generic;
+using System;
-using System.Collections.Immutable;
+using System.Collections.Generic;
 using System.IO;
-using System.Linq;
 using System.Threading.Tasks;
 using API.Data;
 using API.DTOs;
 using API.Entities;
-using API.Extensions;
 using API.Interfaces;
 using AutoMapper;
+using Hangfire;
 using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.Extensions.Logging;
@@ -18,23 +17,23 @@ namespace API.Controllers
     [Authorize]
     public class LibraryController : BaseApiController
     {
-        private readonly DataContext _context;
         private readonly IDirectoryService _directoryService;
         private readonly ILibraryRepository _libraryRepository;
         private readonly ILogger<LibraryController> _logger;
         private readonly IUserRepository _userRepository;
         private readonly IMapper _mapper;
+        private readonly ITaskScheduler _taskScheduler;
 
-        public LibraryController(DataContext context, IDirectoryService directoryService, 
+        public LibraryController(IDirectoryService directoryService, 
             ILibraryRepository libraryRepository, ILogger<LibraryController> logger, IUserRepository userRepository,
-            IMapper mapper)
+            IMapper mapper, ITaskScheduler taskScheduler)
         {
-            _context = context;
             _directoryService = directoryService;
             _libraryRepository = libraryRepository;
             _logger = logger;
             _userRepository = userRepository;
             _mapper = mapper;
+            _taskScheduler = taskScheduler;
         }
 
         /// <summary>
@@ -84,5 +83,16 @@ namespace API.Controllers
 
             return BadRequest("Not Implemented");
         }
+
+        [Authorize(Policy = "RequireAdminRole")]
+        [HttpGet("scan")]
+        public async Task<ActionResult> ScanLibrary(int libraryId)
+        {
+            var library = await _libraryRepository.GetLibraryForIdAsync(libraryId);
+            
+            BackgroundJob.Enqueue(() => _directoryService.ScanLibrary(library));
+
+            return Ok();
+        }
     }
 }

API/Controllers/UsersController.cs

@@ -14,13 +14,11 @@ namespace API.Controllers
     [Authorize]
     public class UsersController : BaseApiController
     {
-        private readonly DataContext _context;
         private readonly IUserRepository _userRepository;
         private readonly ILibraryRepository _libraryRepository;
 
-        public UsersController(DataContext context, IUserRepository userRepository, ILibraryRepository libraryRepository)
+        public UsersController(IUserRepository userRepository, ILibraryRepository libraryRepository)
         {
-            _context = context;
             _userRepository = userRepository;
             _libraryRepository = libraryRepository;
         }
@@ -43,7 +41,7 @@ namespace API.Controllers
             // TODO: We probably need to clean the folders before we insert
             var library = new Library
             {
-                Name = createLibraryDto.Name, // TODO: Ensure code handles Library name always being lowercase
+                Name = createLibraryDto.Name.ToLower(),
                 Type = createLibraryDto.Type,
                 AppUsers = new List<AppUser>() { user }
             };

API/DTOs/CreateLibraryDto.cs

@@ -1,8 +1,6 @@
 using System.Collections.Generic;
 using System.ComponentModel.DataAnnotations;
-using System.Text.Json.Serialization;
 using API.Entities;
-using Microsoft.EntityFrameworkCore.Storage.ValueConversion.Internal;
 
 namespace API.DTOs
 {

API/DTOs/MemberDto.cs

@@ -1,7 +1,5 @@
 using System;
-using System.Collections;
 using System.Collections.Generic;
-using API.Entities;
 
 namespace API.DTOs
 {

API/Data/DataContext.cs

@@ -1,5 +1,4 @@
-using System;
+using API.Entities;
-using API.Entities;
 using Microsoft.AspNetCore.Identity;
 using Microsoft.AspNetCore.Identity.EntityFrameworkCore;
 using Microsoft.EntityFrameworkCore;

API/Data/LibraryRepository.cs

@@ -38,6 +38,15 @@ namespace API.Data
                 .ProjectTo<LibraryDto>(_mapper.ConfigurationProvider).ToListAsync();
         }
         
+        public async Task<LibraryDto> GetLibraryForIdAsync(int libraryId)
+        {
+            return await _context.Library
+                .Where(x => x.Id == libraryId)
+                .Include(f => f.Folders)
+                .ProjectTo<LibraryDto>(_mapper.ConfigurationProvider).SingleAsync();
+        }
+        
+
         public async Task<bool> LibraryExists(string libraryName)
         {
             return await _context.Library.AnyAsync(x => x.Name == libraryName);

API/Data/Migrations/20201224155621_MiscCleanup.cs

@@ -1,5 +1,4 @@
-using System;
+using Microsoft.EntityFrameworkCore.Migrations;
-using Microsoft.EntityFrameworkCore.Migrations;
 
 namespace API.Data.Migrations
 {

API/Data/Migrations/DataContextModelSnapshot.cs

@@ -1,9 +1,7 @@
 // <auto-generated />
 using System;
-using API.Data;
 using Microsoft.EntityFrameworkCore;
 using Microsoft.EntityFrameworkCore.Infrastructure;
-using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
 
 namespace API.Data.Migrations
 {

API/Data/Seed.cs

@@ -12,8 +12,8 @@ namespace API.Data
         {
             var roles = new List<AppRole>
             {
-                new AppRole {Name = PolicyConstants.AdminRole},
+                new() {Name = PolicyConstants.AdminRole},
-                new AppRole {Name = PolicyConstants.PlebRole}
+                new() {Name = PolicyConstants.PlebRole}
             };
 
             foreach (var role in roles)

API/Extensions/ApplicationServiceExtensions.cs

@@ -3,6 +3,8 @@ using API.Helpers;
 using API.Interfaces;
 using API.Services;
 using AutoMapper;
+using Hangfire;
+using Hangfire.LiteDB;
 using Microsoft.EntityFrameworkCore;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
@@ -14,6 +16,7 @@ namespace API.Extensions
         public static IServiceCollection AddApplicationServices(this IServiceCollection services, IConfiguration config)
         {
             services.AddAutoMapper(typeof(AutoMapperProfiles).Assembly);
+            services.AddScoped<ITaskScheduler, TaskScheduler>();
             services.AddScoped<IUserRepository, UserRepository>();
             services.AddScoped<ITokenService, TokenService>();
             services.AddScoped<IDirectoryService, DirectoryService>();
@@ -23,6 +26,14 @@ namespace API.Extensions
                 options.UseSqlite(config.GetConnectionString("DefaultConnection"));
             });
 
+            services.AddHangfire(configuration => configuration
+                .UseSimpleAssemblyNameTypeSerializer()
+                .UseRecommendedSerializerSettings()
+                .UseLiteDbStorage());
+            
+            // Add the processing server as IHostedService
+            services.AddHangfireServer();
+
             return services;
         }
     }

API/Helpers/AutoMapperProfiles.cs

@@ -1,5 +1,4 @@
-using System;
+using System.Linq;
-using System.Linq;
 using API.DTOs;
 using API.Entities;
 using AutoMapper;

API/Interfaces/IDirectoryService.cs

@@ -1,10 +1,12 @@
 using System.Collections.Generic;
-using System.Threading.Tasks;
+using API.DTOs;
 
 namespace API.Interfaces
 {
     public interface IDirectoryService
     {
         IEnumerable<string> ListDirectory(string rootPath);
+
+        void ScanLibrary(LibraryDto library);
     }
 }

API/Interfaces/ILibraryRepository.cs

@@ -17,6 +17,6 @@ namespace API.Interfaces
         /// <returns></returns>
         Task<bool> LibraryExists(string libraryName);
 
-        Task<IEnumerable<LibraryDto>> GetLibrariesForUserAsync(AppUser user);
+        public Task<LibraryDto> GetLibraryForIdAsync(int libraryId);
     }
 }

API/Interfaces/ITaskScheduler.cs

@@ -0,0 +1,7 @@
+namespace API.Interfaces
+{
+    public interface ITaskScheduler
+    {
+        
+    }
+}

API/Interfaces/IUserRepository.cs

@@ -1,5 +1,4 @@
-using System.Collections;
+using System.Collections.Generic;
-using System.Collections.Generic;
 using System.Threading.Tasks;
 using API.DTOs;
 using API.Entities;

API/Services/DirectoryService.cs

@@ -1,23 +1,35 @@
 using System;
 using System.Collections.Generic;
 using System.Collections.Immutable;
+using System.Diagnostics;
 using System.IO;
 using System.Linq;
+using System.Security;
+using System.Threading;
 using System.Threading.Tasks;
+using API.DTOs;
 using API.Interfaces;
+using Microsoft.Extensions.Logging;
 
 namespace API.Services
 {
     public class DirectoryService : IDirectoryService
     {
-        /// <summary>
+       private readonly ILogger<DirectoryService> _logger;
+
+       public DirectoryService(ILogger<DirectoryService> logger)
+       {
+          _logger = logger;
+       }
+
+       /// <summary>
         /// Lists out top-level folders for a given directory. Filters out System and Hidden folders.
         /// </summary>
         /// <param name="rootPath">Absolute path </param>
         /// <returns>List of folder names</returns>
         public IEnumerable<string> ListDirectory(string rootPath)
         {
-            // TODO: Put some checks in here along with API to ensure that we aren't passed a file, folder exists, etc.
+           if (!Directory.Exists(rootPath)) return ImmutableList<string>.Empty;
             
             var di = new DirectoryInfo(rootPath);
             var dirs = di.GetDirectories()
@@ -27,5 +39,132 @@ namespace API.Services
             
             return dirs;
         }
+
+        public void ScanLibrary(LibraryDto library)
+        {
+           foreach (var folderPath in library.Folders)
+           {
+              try {
+                 TraverseTreeParallelForEach(folderPath, (f) =>
+                 {
+                    // Exceptions are no-ops.
+                    try {
+                       // Do nothing with the data except read it.
+                       //byte[] data = File.ReadAllBytes(f);
+                       ProcessManga(f);
+                    }
+                    catch (FileNotFoundException) {}
+                    catch (IOException) {}
+                    catch (UnauthorizedAccessException) {}
+                    catch (SecurityException) {}
+                    // Display the filename.
+                    Console.WriteLine(f);
+                 });
+              }
+              catch (ArgumentException) {
+                 Console.WriteLine(@"The directory 'C:\Program Files' does not exist.");
+              }
+           }
+        }
+
+        private static void ProcessManga(string filename)
+        {
+            Console.WriteLine($"Found {filename}");
+        }
+        
+        public static void TraverseTreeParallelForEach(string root, Action<string> action)
+         {
+            //Count of files traversed and timer for diagnostic output
+            int fileCount = 0;
+            var sw = Stopwatch.StartNew();
+
+            // Determine whether to parallelize file processing on each folder based on processor count.
+            int procCount = System.Environment.ProcessorCount;
+
+            // Data structure to hold names of subfolders to be examined for files.
+            Stack<string> dirs = new Stack<string>();
+
+            if (!Directory.Exists(root)) {
+                   throw new ArgumentException();
+            }
+            dirs.Push(root);
+
+            while (dirs.Count > 0) {
+               string currentDir = dirs.Pop();
+               string[] subDirs = {};
+               string[] files = {};
+
+               try {
+                  subDirs = Directory.GetDirectories(currentDir);
+               }
+               // Thrown if we do not have discovery permission on the directory.
+               catch (UnauthorizedAccessException e) {
+                  Console.WriteLine(e.Message);
+                  continue;
+               }
+               // Thrown if another process has deleted the directory after we retrieved its name.
+               catch (DirectoryNotFoundException e) {
+                  Console.WriteLine(e.Message);
+                  continue;
+               }
+
+               try {
+                  files = Directory.GetFiles(currentDir);
+               }
+               catch (UnauthorizedAccessException e) {
+                  Console.WriteLine(e.Message);
+                  continue;
+               }
+               catch (DirectoryNotFoundException e) {
+                  Console.WriteLine(e.Message);
+                  continue;
+               }
+               catch (IOException e) {
+                  Console.WriteLine(e.Message);
+                  continue;
+               }
+
+               // Execute in parallel if there are enough files in the directory.
+               // Otherwise, execute sequentially.Files are opened and processed
+               // synchronously but this could be modified to perform async I/O.
+               try {
+                  if (files.Length < procCount) {
+                     foreach (var file in files) {
+                        action(file);
+                        fileCount++;
+                     }
+                  }
+                  else {
+                     Parallel.ForEach(files, () => 0, (file, loopState, localCount) =>
+                                                  { action(file);
+                                                    return ++localCount;
+                                                  },
+                                      (c) => {
+                                                Interlocked.Add(ref fileCount, c);
+                                      });
+                  }
+               }
+               catch (AggregateException ae) {
+                  ae.Handle((ex) => {
+                               if (ex is UnauthorizedAccessException) {
+                                  // Here we just output a message and go on.
+                                  Console.WriteLine(ex.Message);
+                                  return true;
+                               }
+                               // Handle other exceptions here if necessary...
+
+                               return false;
+                  });
+               }
+
+               // Push the subdirectories onto the stack for traversal.
+               // This could also be done before handing the files.
+               foreach (string str in subDirs)
+                  dirs.Push(str);
+            }
+
+            // For diagnostic purposes.
+            Console.WriteLine("Processed {0} files in {1} milliseconds", fileCount, sw.ElapsedMilliseconds);
+         }
     }
 }

API/Services/TaskScheduler.cs

@@ -0,0 +1,17 @@
+using API.Interfaces;
+using Hangfire;
+
+namespace API.Services
+{
+    public class TaskScheduler : ITaskScheduler
+    {
+        private readonly BackgroundJobServer _client;
+        
+        public TaskScheduler()
+        {
+            _client = new BackgroundJobServer();
+        }
+        
+        
+    }
+}

API/Startup.cs

@@ -1,5 +1,7 @@
+using System;
 using API.Extensions;
 using API.Middleware;
+using Hangfire;
 using Microsoft.AspNetCore.Builder;
 using Microsoft.AspNetCore.Hosting;
 using Microsoft.Extensions.Configuration;
@@ -33,7 +35,7 @@ namespace API
         }
 
         // This method gets called by the runtime. Use this method to configure the HTTP request pipeline.
-        public void Configure(IApplicationBuilder app, IWebHostEnvironment env)
+        public void Configure(IApplicationBuilder app, IBackgroundJobClient backgroundJobs, IWebHostEnvironment env)
         {
             app.UseMiddleware<ExceptionMiddleware>();
             
@@ -43,6 +45,9 @@ namespace API
                 app.UseSwaggerUI(c => c.SwaggerEndpoint("/swagger/v1/swagger.json", "API v1"));
             }
             
+            app.UseHangfireDashboard();
+            backgroundJobs.Enqueue(() => Console.WriteLine("Hello world from Hangfire!"));
+
             app.UseHttpsRedirection();
 
             app.UseRouting();
@@ -57,6 +62,7 @@ namespace API
             app.UseEndpoints(endpoints =>
             {
                 endpoints.MapControllers();
+                endpoints.MapHangfireDashboard();
             });
         }
     }

API/appsettings.Development.json

@@ -1,13 +1,15 @@
 {
   "ConnectionStrings": {
-    "DefaultConnection": "Data source=kavita.db"
+    "DefaultConnection": "Data source=kavita.db",
+    "HangfireConnection": "Data source=hangfire.db"
   },
   "TokenKey": "super secret unguessable key",
   "Logging": {
     "LogLevel": {
       "Default": "Debug",
       "Microsoft": "Information",
-      "Microsoft.Hosting.Lifetime": "Information"
+      "Microsoft.Hosting.Lifetime": "Information",
+      "Hangfire": "Information"
     }
   }
 }