mirror of
https://github.com/Kareadita/Kavita.git
synced 2025-07-09 03:04:19 -04:00
Merge pull request #16 from Kareadita/feature/scan-library
Library Scanning + Task Scheduling
This commit is contained in:
commit
1797a3fb64
@ -8,6 +8,9 @@
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="AutoMapper.Extensions.Microsoft.DependencyInjection" Version="8.1.0" />
|
||||
<PackageReference Include="Hangfire" Version="1.7.18" />
|
||||
<PackageReference Include="Hangfire.AspNetCore" Version="1.7.18" />
|
||||
<PackageReference Include="Hangfire.LiteDB" Version="0.4.0" />
|
||||
<PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="5.0.1" NoWarn="NU1605" />
|
||||
<PackageReference Include="Microsoft.AspNetCore.Authentication.OpenIdConnect" Version="5.0.1" NoWarn="NU1605" />
|
||||
<PackageReference Include="Microsoft.AspNetCore.Identity.EntityFrameworkCore" Version="5.0.1" />
|
||||
@ -25,4 +28,8 @@
|
||||
<PackageReference Include="System.IdentityModel.Tokens.Jwt" Version="6.8.0" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<Folder Include="Tasks" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
|
@ -49,8 +49,6 @@ namespace API.Controllers
|
||||
|
||||
if (!result.Succeeded) return BadRequest(result.Errors);
|
||||
|
||||
|
||||
// TODO: Need a way to store Roles in enum and configure from there
|
||||
var role = registerDto.IsAdmin ? PolicyConstants.AdminRole : PolicyConstants.PlebRole;
|
||||
var roleResult = await _userManager.AddToRoleAsync(user, role);
|
||||
|
||||
|
@ -1,9 +1,6 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Threading.Tasks;
|
||||
using API.DTOs;
|
||||
using System.Threading.Tasks;
|
||||
using API.Entities;
|
||||
using API.Interfaces;
|
||||
using Microsoft.AspNetCore.Authorization;
|
||||
using Microsoft.AspNetCore.Identity;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
|
||||
|
@ -1,14 +1,13 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
using API.Data;
|
||||
using API.DTOs;
|
||||
using API.Entities;
|
||||
using API.Extensions;
|
||||
using API.Interfaces;
|
||||
using AutoMapper;
|
||||
using Hangfire;
|
||||
using Microsoft.AspNetCore.Authorization;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.Extensions.Logging;
|
||||
@ -18,23 +17,23 @@ namespace API.Controllers
|
||||
[Authorize]
|
||||
public class LibraryController : BaseApiController
|
||||
{
|
||||
private readonly DataContext _context;
|
||||
private readonly IDirectoryService _directoryService;
|
||||
private readonly ILibraryRepository _libraryRepository;
|
||||
private readonly ILogger<LibraryController> _logger;
|
||||
private readonly IUserRepository _userRepository;
|
||||
private readonly IMapper _mapper;
|
||||
private readonly ITaskScheduler _taskScheduler;
|
||||
|
||||
public LibraryController(DataContext context, IDirectoryService directoryService,
|
||||
public LibraryController(IDirectoryService directoryService,
|
||||
ILibraryRepository libraryRepository, ILogger<LibraryController> logger, IUserRepository userRepository,
|
||||
IMapper mapper)
|
||||
IMapper mapper, ITaskScheduler taskScheduler)
|
||||
{
|
||||
_context = context;
|
||||
_directoryService = directoryService;
|
||||
_libraryRepository = libraryRepository;
|
||||
_logger = logger;
|
||||
_userRepository = userRepository;
|
||||
_mapper = mapper;
|
||||
_taskScheduler = taskScheduler;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@ -84,5 +83,16 @@ namespace API.Controllers
|
||||
|
||||
return BadRequest("Not Implemented");
|
||||
}
|
||||
|
||||
[Authorize(Policy = "RequireAdminRole")]
|
||||
[HttpGet("scan")]
|
||||
public async Task<ActionResult> ScanLibrary(int libraryId)
|
||||
{
|
||||
var library = await _libraryRepository.GetLibraryForIdAsync(libraryId);
|
||||
|
||||
BackgroundJob.Enqueue(() => _directoryService.ScanLibrary(library));
|
||||
|
||||
return Ok();
|
||||
}
|
||||
}
|
||||
}
|
@ -14,13 +14,11 @@ namespace API.Controllers
|
||||
[Authorize]
|
||||
public class UsersController : BaseApiController
|
||||
{
|
||||
private readonly DataContext _context;
|
||||
private readonly IUserRepository _userRepository;
|
||||
private readonly ILibraryRepository _libraryRepository;
|
||||
|
||||
public UsersController(DataContext context, IUserRepository userRepository, ILibraryRepository libraryRepository)
|
||||
public UsersController(IUserRepository userRepository, ILibraryRepository libraryRepository)
|
||||
{
|
||||
_context = context;
|
||||
_userRepository = userRepository;
|
||||
_libraryRepository = libraryRepository;
|
||||
}
|
||||
@ -43,7 +41,7 @@ namespace API.Controllers
|
||||
// TODO: We probably need to clean the folders before we insert
|
||||
var library = new Library
|
||||
{
|
||||
Name = createLibraryDto.Name, // TODO: Ensure code handles Library name always being lowercase
|
||||
Name = createLibraryDto.Name.ToLower(),
|
||||
Type = createLibraryDto.Type,
|
||||
AppUsers = new List<AppUser>() { user }
|
||||
};
|
||||
|
@ -1,8 +1,6 @@
|
||||
using System.Collections.Generic;
|
||||
using System.ComponentModel.DataAnnotations;
|
||||
using System.Text.Json.Serialization;
|
||||
using API.Entities;
|
||||
using Microsoft.EntityFrameworkCore.Storage.ValueConversion.Internal;
|
||||
|
||||
namespace API.DTOs
|
||||
{
|
||||
|
@ -1,7 +1,5 @@
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using API.Entities;
|
||||
|
||||
namespace API.DTOs
|
||||
{
|
||||
|
@ -1,5 +1,4 @@
|
||||
using System;
|
||||
using API.Entities;
|
||||
using API.Entities;
|
||||
using Microsoft.AspNetCore.Identity;
|
||||
using Microsoft.AspNetCore.Identity.EntityFrameworkCore;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
|
@ -37,6 +37,15 @@ namespace API.Data
|
||||
.Include(f => f.Folders)
|
||||
.ProjectTo<LibraryDto>(_mapper.ConfigurationProvider).ToListAsync();
|
||||
}
|
||||
|
||||
public async Task<LibraryDto> GetLibraryForIdAsync(int libraryId)
|
||||
{
|
||||
return await _context.Library
|
||||
.Where(x => x.Id == libraryId)
|
||||
.Include(f => f.Folders)
|
||||
.ProjectTo<LibraryDto>(_mapper.ConfigurationProvider).SingleAsync();
|
||||
}
|
||||
|
||||
|
||||
public async Task<bool> LibraryExists(string libraryName)
|
||||
{
|
||||
|
@ -1,5 +1,4 @@
|
||||
using System;
|
||||
using Microsoft.EntityFrameworkCore.Migrations;
|
||||
using Microsoft.EntityFrameworkCore.Migrations;
|
||||
|
||||
namespace API.Data.Migrations
|
||||
{
|
||||
|
@ -1,9 +1,7 @@
|
||||
// <auto-generated />
|
||||
using System;
|
||||
using API.Data;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.EntityFrameworkCore.Infrastructure;
|
||||
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
||||
|
||||
namespace API.Data.Migrations
|
||||
{
|
||||
|
@ -12,8 +12,8 @@ namespace API.Data
|
||||
{
|
||||
var roles = new List<AppRole>
|
||||
{
|
||||
new AppRole {Name = PolicyConstants.AdminRole},
|
||||
new AppRole {Name = PolicyConstants.PlebRole}
|
||||
new() {Name = PolicyConstants.AdminRole},
|
||||
new() {Name = PolicyConstants.PlebRole}
|
||||
};
|
||||
|
||||
foreach (var role in roles)
|
||||
|
@ -3,6 +3,8 @@ using API.Helpers;
|
||||
using API.Interfaces;
|
||||
using API.Services;
|
||||
using AutoMapper;
|
||||
using Hangfire;
|
||||
using Hangfire.LiteDB;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
@ -14,6 +16,7 @@ namespace API.Extensions
|
||||
public static IServiceCollection AddApplicationServices(this IServiceCollection services, IConfiguration config)
|
||||
{
|
||||
services.AddAutoMapper(typeof(AutoMapperProfiles).Assembly);
|
||||
services.AddScoped<ITaskScheduler, TaskScheduler>();
|
||||
services.AddScoped<IUserRepository, UserRepository>();
|
||||
services.AddScoped<ITokenService, TokenService>();
|
||||
services.AddScoped<IDirectoryService, DirectoryService>();
|
||||
@ -23,6 +26,14 @@ namespace API.Extensions
|
||||
options.UseSqlite(config.GetConnectionString("DefaultConnection"));
|
||||
});
|
||||
|
||||
services.AddHangfire(configuration => configuration
|
||||
.UseSimpleAssemblyNameTypeSerializer()
|
||||
.UseRecommendedSerializerSettings()
|
||||
.UseLiteDbStorage());
|
||||
|
||||
// Add the processing server as IHostedService
|
||||
services.AddHangfireServer();
|
||||
|
||||
return services;
|
||||
}
|
||||
}
|
||||
|
BIN
API/Hangfire-log.db
Normal file
BIN
API/Hangfire-log.db
Normal file
Binary file not shown.
BIN
API/Hangfire.db
Normal file
BIN
API/Hangfire.db
Normal file
Binary file not shown.
@ -1,5 +1,4 @@
|
||||
using System;
|
||||
using System.Linq;
|
||||
using System.Linq;
|
||||
using API.DTOs;
|
||||
using API.Entities;
|
||||
using AutoMapper;
|
||||
|
@ -1,10 +1,12 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Threading.Tasks;
|
||||
using API.DTOs;
|
||||
|
||||
namespace API.Interfaces
|
||||
{
|
||||
public interface IDirectoryService
|
||||
{
|
||||
IEnumerable<string> ListDirectory(string rootPath);
|
||||
|
||||
void ScanLibrary(LibraryDto library);
|
||||
}
|
||||
}
|
@ -17,6 +17,6 @@ namespace API.Interfaces
|
||||
/// <returns></returns>
|
||||
Task<bool> LibraryExists(string libraryName);
|
||||
|
||||
Task<IEnumerable<LibraryDto>> GetLibrariesForUserAsync(AppUser user);
|
||||
public Task<LibraryDto> GetLibraryForIdAsync(int libraryId);
|
||||
}
|
||||
}
|
7
API/Interfaces/ITaskScheduler.cs
Normal file
7
API/Interfaces/ITaskScheduler.cs
Normal file
@ -0,0 +1,7 @@
|
||||
namespace API.Interfaces
|
||||
{
|
||||
public interface ITaskScheduler
|
||||
{
|
||||
|
||||
}
|
||||
}
|
@ -1,5 +1,4 @@
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Generic;
|
||||
using System.Threading.Tasks;
|
||||
using API.DTOs;
|
||||
using API.Entities;
|
||||
|
@ -1,23 +1,35 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.Diagnostics;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Security;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using API.DTOs;
|
||||
using API.Interfaces;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace API.Services
|
||||
{
|
||||
public class DirectoryService : IDirectoryService
|
||||
{
|
||||
/// <summary>
|
||||
private readonly ILogger<DirectoryService> _logger;
|
||||
|
||||
public DirectoryService(ILogger<DirectoryService> logger)
|
||||
{
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Lists out top-level folders for a given directory. Filters out System and Hidden folders.
|
||||
/// </summary>
|
||||
/// <param name="rootPath">Absolute path </param>
|
||||
/// <returns>List of folder names</returns>
|
||||
public IEnumerable<string> ListDirectory(string rootPath)
|
||||
{
|
||||
// TODO: Put some checks in here along with API to ensure that we aren't passed a file, folder exists, etc.
|
||||
if (!Directory.Exists(rootPath)) return ImmutableList<string>.Empty;
|
||||
|
||||
var di = new DirectoryInfo(rootPath);
|
||||
var dirs = di.GetDirectories()
|
||||
@ -27,5 +39,132 @@ namespace API.Services
|
||||
|
||||
return dirs;
|
||||
}
|
||||
|
||||
public void ScanLibrary(LibraryDto library)
|
||||
{
|
||||
foreach (var folderPath in library.Folders)
|
||||
{
|
||||
try {
|
||||
TraverseTreeParallelForEach(folderPath, (f) =>
|
||||
{
|
||||
// Exceptions are no-ops.
|
||||
try {
|
||||
// Do nothing with the data except read it.
|
||||
//byte[] data = File.ReadAllBytes(f);
|
||||
ProcessManga(f);
|
||||
}
|
||||
catch (FileNotFoundException) {}
|
||||
catch (IOException) {}
|
||||
catch (UnauthorizedAccessException) {}
|
||||
catch (SecurityException) {}
|
||||
// Display the filename.
|
||||
Console.WriteLine(f);
|
||||
});
|
||||
}
|
||||
catch (ArgumentException) {
|
||||
Console.WriteLine(@"The directory 'C:\Program Files' does not exist.");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void ProcessManga(string filename)
|
||||
{
|
||||
Console.WriteLine($"Found {filename}");
|
||||
}
|
||||
|
||||
public static void TraverseTreeParallelForEach(string root, Action<string> action)
|
||||
{
|
||||
//Count of files traversed and timer for diagnostic output
|
||||
int fileCount = 0;
|
||||
var sw = Stopwatch.StartNew();
|
||||
|
||||
// Determine whether to parallelize file processing on each folder based on processor count.
|
||||
int procCount = System.Environment.ProcessorCount;
|
||||
|
||||
// Data structure to hold names of subfolders to be examined for files.
|
||||
Stack<string> dirs = new Stack<string>();
|
||||
|
||||
if (!Directory.Exists(root)) {
|
||||
throw new ArgumentException();
|
||||
}
|
||||
dirs.Push(root);
|
||||
|
||||
while (dirs.Count > 0) {
|
||||
string currentDir = dirs.Pop();
|
||||
string[] subDirs = {};
|
||||
string[] files = {};
|
||||
|
||||
try {
|
||||
subDirs = Directory.GetDirectories(currentDir);
|
||||
}
|
||||
// Thrown if we do not have discovery permission on the directory.
|
||||
catch (UnauthorizedAccessException e) {
|
||||
Console.WriteLine(e.Message);
|
||||
continue;
|
||||
}
|
||||
// Thrown if another process has deleted the directory after we retrieved its name.
|
||||
catch (DirectoryNotFoundException e) {
|
||||
Console.WriteLine(e.Message);
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
files = Directory.GetFiles(currentDir);
|
||||
}
|
||||
catch (UnauthorizedAccessException e) {
|
||||
Console.WriteLine(e.Message);
|
||||
continue;
|
||||
}
|
||||
catch (DirectoryNotFoundException e) {
|
||||
Console.WriteLine(e.Message);
|
||||
continue;
|
||||
}
|
||||
catch (IOException e) {
|
||||
Console.WriteLine(e.Message);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Execute in parallel if there are enough files in the directory.
|
||||
// Otherwise, execute sequentially.Files are opened and processed
|
||||
// synchronously but this could be modified to perform async I/O.
|
||||
try {
|
||||
if (files.Length < procCount) {
|
||||
foreach (var file in files) {
|
||||
action(file);
|
||||
fileCount++;
|
||||
}
|
||||
}
|
||||
else {
|
||||
Parallel.ForEach(files, () => 0, (file, loopState, localCount) =>
|
||||
{ action(file);
|
||||
return ++localCount;
|
||||
},
|
||||
(c) => {
|
||||
Interlocked.Add(ref fileCount, c);
|
||||
});
|
||||
}
|
||||
}
|
||||
catch (AggregateException ae) {
|
||||
ae.Handle((ex) => {
|
||||
if (ex is UnauthorizedAccessException) {
|
||||
// Here we just output a message and go on.
|
||||
Console.WriteLine(ex.Message);
|
||||
return true;
|
||||
}
|
||||
// Handle other exceptions here if necessary...
|
||||
|
||||
return false;
|
||||
});
|
||||
}
|
||||
|
||||
// Push the subdirectories onto the stack for traversal.
|
||||
// This could also be done before handing the files.
|
||||
foreach (string str in subDirs)
|
||||
dirs.Push(str);
|
||||
}
|
||||
|
||||
// For diagnostic purposes.
|
||||
Console.WriteLine("Processed {0} files in {1} milliseconds", fileCount, sw.ElapsedMilliseconds);
|
||||
}
|
||||
}
|
||||
}
|
17
API/Services/TaskScheduler.cs
Normal file
17
API/Services/TaskScheduler.cs
Normal file
@ -0,0 +1,17 @@
|
||||
using API.Interfaces;
|
||||
using Hangfire;
|
||||
|
||||
namespace API.Services
|
||||
{
|
||||
public class TaskScheduler : ITaskScheduler
|
||||
{
|
||||
private readonly BackgroundJobServer _client;
|
||||
|
||||
public TaskScheduler()
|
||||
{
|
||||
_client = new BackgroundJobServer();
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
}
|
@ -1,5 +1,7 @@
|
||||
using System;
|
||||
using API.Extensions;
|
||||
using API.Middleware;
|
||||
using Hangfire;
|
||||
using Microsoft.AspNetCore.Builder;
|
||||
using Microsoft.AspNetCore.Hosting;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
@ -33,7 +35,7 @@ namespace API
|
||||
}
|
||||
|
||||
// This method gets called by the runtime. Use this method to configure the HTTP request pipeline.
|
||||
public void Configure(IApplicationBuilder app, IWebHostEnvironment env)
|
||||
public void Configure(IApplicationBuilder app, IBackgroundJobClient backgroundJobs, IWebHostEnvironment env)
|
||||
{
|
||||
app.UseMiddleware<ExceptionMiddleware>();
|
||||
|
||||
@ -42,6 +44,9 @@ namespace API
|
||||
app.UseSwagger();
|
||||
app.UseSwaggerUI(c => c.SwaggerEndpoint("/swagger/v1/swagger.json", "API v1"));
|
||||
}
|
||||
|
||||
app.UseHangfireDashboard();
|
||||
backgroundJobs.Enqueue(() => Console.WriteLine("Hello world from Hangfire!"));
|
||||
|
||||
app.UseHttpsRedirection();
|
||||
|
||||
@ -57,6 +62,7 @@ namespace API
|
||||
app.UseEndpoints(endpoints =>
|
||||
{
|
||||
endpoints.MapControllers();
|
||||
endpoints.MapHangfireDashboard();
|
||||
});
|
||||
}
|
||||
}
|
||||
|
@ -1,13 +1,15 @@
|
||||
{
|
||||
"ConnectionStrings": {
|
||||
"DefaultConnection": "Data source=kavita.db"
|
||||
"DefaultConnection": "Data source=kavita.db",
|
||||
"HangfireConnection": "Data source=hangfire.db"
|
||||
},
|
||||
"TokenKey": "super secret unguessable key",
|
||||
"Logging": {
|
||||
"LogLevel": {
|
||||
"Default": "Debug",
|
||||
"Microsoft": "Information",
|
||||
"Microsoft.Hosting.Lifetime": "Information"
|
||||
"Microsoft.Hosting.Lifetime": "Information",
|
||||
"Hangfire": "Information"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
Loading…
x
Reference in New Issue
Block a user