Making the create show system for the crawler.

This commit is contained in:
Zoe Roux 2019-08-08 18:01:07 +02:00
parent 880434186e
commit 7f503cf3d3
11 changed files with 486 additions and 210 deletions

View File

@ -61,13 +61,14 @@ namespace Kyoo.InternalAPI
Debug.WriteLine("&ShowPath: " + ShowPath + " Show: " + ShowTitle + " season: " + Season + " episode: " + Episode);
if(!libraryManager.IsShowRegistered(ShowPath))
if (!libraryManager.IsShowRegistered(ShowPath, out long? showID))
{
Show show = await metadataProvider.GetShowFromName(ShowTitle);
Debug.WriteLine("&Show Name: " + show.Title + " Overview: " + show.Overview);
//long showID = libraryManager.RegisterShow(show);
Debug.WriteLine("&Should register show: " + ShowTitle);
Show show = await metadataProvider.GetShowFromName(ShowTitle, ShowPath);
showID = libraryManager.RegisterShow(show);
}
Debug.WriteLine("&Show ID: " + showID);
}
}
}

View File

@ -5,10 +5,15 @@ namespace Kyoo.InternalAPI
{
public interface ILibraryManager
{
//Read values
IEnumerable<Show> QueryShows(string selection);
//Check if value exists
bool IsEpisodeRegistered(string episodePath);
bool IsShowRegistered(string showPath);
bool IsShowRegistered(string showPath, out long? showID);
//Register values
long RegisterShow(Show show);
}
}

View File

@ -30,14 +30,14 @@ namespace Kyoo.InternalAPI
slug TEXT UNIQUE,
title TEXT,
aliases TEXT,
path TEXT,
path TEXT UNIQUE,
overview TEXT,
genres TEXT,
status TEXT,
startYear INTEGER,
endYear INTEGER,
imgPrimary TEXT,
imgThumb TEXT,
imgBanner TEXT,
imgLogo TEXT,
imgBackdrop TEXT,
externalIDs TEXT
@ -144,8 +144,10 @@ namespace Kyoo.InternalAPI
FOREIGN KEY(showID) REFERENCES shows(id)
);";
SQLiteCommand createCmd = new SQLiteCommand(createStatement, sqlConnection);
createCmd.ExecuteNonQuery();
using (SQLiteCommand createCmd = new SQLiteCommand(createStatement, sqlConnection))
{
createCmd.ExecuteNonQuery();
}
}
else
{
@ -165,35 +167,78 @@ namespace Kyoo.InternalAPI
{
string query = "SELECT * FROM shows;";
SQLiteCommand cmd = new SQLiteCommand(query, sqlConnection);
SQLiteDataReader reader = cmd.ExecuteReader();
using (SQLiteCommand cmd = new SQLiteCommand(query, sqlConnection))
{
SQLiteDataReader reader = cmd.ExecuteReader();
List<Show> shows = new List<Show>();
List<Show> shows = new List<Show>();
while (reader.Read())
shows.Add(Show.FromReader(reader));
while (reader.Read())
shows.Add(Show.FromReader(reader));
return shows;
return shows;
}
}
public bool IsEpisodeRegistered(string episodePath)
{
string query = "SELECT 1 FROM episodes WHERE path = $path;";
SQLiteCommand cmd = new SQLiteCommand(query, sqlConnection);
using (SQLiteCommand cmd = new SQLiteCommand(query, sqlConnection))
{
cmd.Parameters.AddWithValue("$path", episodePath);
cmd.Parameters.AddWithValue("$path", episodePath);
return cmd.ExecuteScalar() != null;
return cmd.ExecuteScalar() != null;
}
}
public bool IsShowRegistered(string showPath)
{
string query = "SELECT 1 FROM shows WHERE path = $path;";
SQLiteCommand cmd = new SQLiteCommand(query, sqlConnection);
using (SQLiteCommand cmd = new SQLiteCommand(query, sqlConnection))
{
cmd.Parameters.AddWithValue("$path", showPath);
cmd.Parameters.AddWithValue("$path", showPath);
return cmd.ExecuteScalar() != null;
}
}
return cmd.ExecuteScalar() != null;
public bool IsShowRegistered(string showPath, out long? showID)
{
string query = "SELECT 1 FROM shows WHERE path = $path;";
using (SQLiteCommand cmd = new SQLiteCommand(query, sqlConnection))
{
cmd.Parameters.AddWithValue("$path", showPath);
showID = cmd.ExecuteScalar() as long?;
return showID != null;
}
}
public long RegisterShow(Show show)
{
string query = "INSERT INTO shows (slug, title, aliases, path, overview, genres, startYear, endYear, imgPrimary, imgThumb, imgLogo, imgBackdrop, externalIDs) VALUES($slug, $title, $aliases, $path, $overview, $genres, $startYear, $endYear, $imgPrimary, $imgThumb, $imgLogo, $imgBackdrop, $externalIDs);";
Debug.WriteLine("&SQL QUERY:: " + query);
using (SQLiteCommand cmd = new SQLiteCommand(query, sqlConnection))
{
cmd.Parameters.AddWithValue("slug", show.Slug);
cmd.Parameters.AddWithValue("title", show.Title);
cmd.Parameters.AddWithValue("aliases", show.GetAliases());
cmd.Parameters.AddWithValue("path", show.Path);
cmd.Parameters.AddWithValue("overview", show.Overview);
cmd.Parameters.AddWithValue("genres", show.GetGenres());
cmd.Parameters.AddWithValue("status", show.Status);
cmd.Parameters.AddWithValue("startYear", show.StartYear);
cmd.Parameters.AddWithValue("endYear", show.EndYear);
cmd.Parameters.AddWithValue("imgPrimary", show.ImgPrimary);
cmd.Parameters.AddWithValue("imgThumb", show.ImgThumb);
cmd.Parameters.AddWithValue("imgLogo", show.ImgLogo);
cmd.Parameters.AddWithValue("imgBackdrop", show.ImgBackdrop);
cmd.Parameters.AddWithValue("externalIDs", show.ExternalIDs);
cmd.ExecuteNonQuery();
cmd.CommandText = "SELECT LAST_INSERT_ROWID()";
return (long)cmd.ExecuteScalar();
}
}
}
}

View File

@ -5,8 +5,10 @@ namespace Kyoo.InternalAPI
{
public interface IMetadataProvider
{
Task<Show> GetShowFromID(string externalIDs);
Task<Show> CompleteShow(Show show);
Task<Show> GetShowFromName(string showName);
Task<Show> GetShowFromName(string showName, string showPath);
Task<Show> GetImages(Show show);
}
}

View File

@ -1,164 +0,0 @@
using Kyoo.Models;
using Newtonsoft.Json;
using System;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Text;
using System.Threading.Tasks;
using System.Web;
namespace Kyoo.InternalAPI.MetadataProvider
{
public class ProviderTheTvDB : ProviderHelper, IMetadataProvider
{
public override string Provider => "TvDB";
private struct DataTvDB
{
public string seriesName;
public string overview;
public string slug;
public string network;
public string status;
public int id;
public string firstAired;
public string banner;
public string[] aliases;
}
async Task<string> Authentificate()
{
WebRequest request = WebRequest.Create("https://api.thetvdb.com/login");
request.Method = "POST";
request.Timeout = 12000;
request.ContentType = "application/json";
string json = "{ \"apikey\": \"IM2OXA8UHUIU0GH6\" }";
byte[] bytes = Encoding.ASCII.GetBytes(json);
request.ContentLength = bytes.Length;
using (Stream stream = request.GetRequestStream())
{
stream.Write(bytes, 0, bytes.Length);
}
HttpWebResponse response = (HttpWebResponse)await request.GetResponseAsync();
if(response.StatusCode == HttpStatusCode.OK)
{
Stream stream = response.GetResponseStream();
using (StreamReader reader = new StreamReader(stream))
{
string content = await reader.ReadToEndAsync();
stream.Close();
response.Close();
var obj = new { Token = "" };
return JsonConvert.DeserializeAnonymousType(content, obj).Token;
}
}
else
Debug.WriteLine("&Couldn't authentificate in TheTvDB API.\nError status: " + response.StatusCode + " Message: " + response.StatusDescription);
return null;
}
public async Task<Show> GetShowFromName(string showName)
{
string token = await Authentificate();
Debug.WriteLine("&Sucess, token = " + token);
if (token != null)
{
WebRequest request = WebRequest.Create("https://api.thetvdb.com/search/series?name=" + HttpUtility.HtmlEncode(showName));
request.Method = "GET";
request.Timeout = 12000;
request.ContentType = "application/json";
request.Headers.Add(HttpRequestHeader.Authorization, "Bearer " + token);
HttpWebResponse response = (HttpWebResponse)await request.GetResponseAsync();
if (response.StatusCode == HttpStatusCode.OK)
{
Stream stream = response.GetResponseStream();
using (StreamReader reader = new StreamReader(stream))
{
string content = await reader.ReadToEndAsync();
stream.Close();
response.Close();
var model = new { data = new DataTvDB[0] };
DataTvDB data = JsonConvert.DeserializeAnonymousType(content, model).data[0];
long? startYear = null;
if (!long.TryParse(data.firstAired?.Substring(4), out long year))
startYear = year;
Show show = new Show(-1, ToSlug(showName), data.seriesName, data.aliases?.ToList(), data.overview, null, startYear, null, null, null, null, null, null, string.Format("{0}={1}|", Provider, data.id));
return await GetShowFromID(show.ExternalIDs) ?? show;
}
}
else
{
Debug.WriteLine("&TheTvDB Provider couldn't work for this show: " + showName + ".\nError Code: " + response.StatusCode + " Message: " + response.StatusDescription);
response.Close();
}
}
return new Show() { Slug = ToSlug(showName), Title = showName };
}
public async Task<Show> GetShowFromID(string externalIDs)
{
string id = GetId(externalIDs);
if (id == null)
return null;
string token = await Authentificate();
if (token == null)
return null;
WebRequest request = WebRequest.Create("https://api.thetvdb.com/search/series/" + id);
request.Method = "GET";
request.Timeout = 12000;
request.ContentType = "application/json";
request.Headers.Add(HttpRequestHeader.Authorization, "Bearer " + token);
HttpWebResponse response = (HttpWebResponse)await request.GetResponseAsync();
if (response.StatusCode == HttpStatusCode.OK)
{
Stream stream = response.GetResponseStream();
using (StreamReader reader = new StreamReader(stream))
{
string content = await reader.ReadToEndAsync();
stream.Close();
response.Close();
var model = new { data = new DataTvDB[0] };
DataTvDB data = JsonConvert.DeserializeAnonymousType(content, model).data[0];
long? startYear = null;
if (!long.TryParse(data.firstAired?.Substring(4), out long year))
startYear = year;
Show show = new Show(-1, ToSlug(showName), data.seriesName, data.aliases?.ToList(), data.overview, null, startYear, null, null, null, null, null, null, string.Format("TvDB={0}|", data.id));
return await GetShowFromID(show.ExternalIDs) ?? show;
}
}
else
{
Debug.WriteLine("&TheTvDB Provider couldn't work for the show with the id: " + id + ".\nError Code: " + response.StatusCode + " Message: " + response.StatusDescription);
response.Close();
return null;
}
}
}
}

View File

@ -0,0 +1,93 @@
using Kyoo.Models;
using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Net;
using System.Text;
using System.Threading.Tasks;
namespace Kyoo.InternalAPI.MetadataProvider.TheTvDB
{
public class HelperTvDB : ProviderHelper
{
public override string Provider => "TvDB";
private string token;
private DateTime tokenDate;
public async Task<string> Authentificate()
{
if (tokenDate != null && tokenDate > DateTime.UtcNow.AddDays(-1))
return token;
WebRequest request = WebRequest.Create("https://api.thetvdb.com/login");
request.Method = "POST";
request.Timeout = 12000;
request.ContentType = "application/json";
string json = "{ \"apikey\": \"IM2OXA8UHUIU0GH6\" }";
byte[] bytes = Encoding.ASCII.GetBytes(json);
request.ContentLength = bytes.Length;
using (Stream stream = request.GetRequestStream())
{
stream.Write(bytes, 0, bytes.Length);
}
try
{
HttpWebResponse response = (HttpWebResponse)await request.GetResponseAsync();
if (response.StatusCode == HttpStatusCode.OK)
{
Stream stream = response.GetResponseStream();
using (StreamReader reader = new StreamReader(stream))
{
string content = await reader.ReadToEndAsync();
stream.Close();
response.Close();
var obj = new { Token = "" };
token = JsonConvert.DeserializeAnonymousType(content, obj).Token;
tokenDate = DateTime.UtcNow;
return token;
}
}
else
Debug.WriteLine("&Couldn't authentificate in TheTvDB API.\nError status: " + response.StatusCode + " Message: " + response.StatusDescription);
}
catch (WebException ex)
{
Debug.WriteLine("&Couldn't authentificate in TheTvDB API.\nError status: " + ex.Status);
return null;
}
return null;
}
public long? GetYear(string firstAired)
{
if (long.TryParse(firstAired?.Substring(0, 4), out long year))
return year;
return null;
}
public Status? GetStatus(string status)
{
switch (status)
{
case "Ended":
return Status.Finished;
case "Continuing":
return Status.Airing;
default:
return null;
}
}
}
}

View File

@ -0,0 +1,252 @@
using Kyoo.InternalAPI.MetadataProvider.TheTvDB;
using Kyoo.Models;
using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Net;
using System.Text;
using System.Threading.Tasks;
using System.Web;
namespace Kyoo.InternalAPI.MetadataProvider
{
public class ShowProviderTvDB : HelperTvDB, IMetadataProvider
{
private struct SearchTbDB
{
public string seriesName;
public string overview;
public string slug;
public string network;
public string status;
public int id;
public string firstAired;
public string banner;
public string[] aliases;
}
private struct DataTvDb
{
public string seriesName;
public string overview;
public string slug;
public string network;
public string status;
public int id;
public string seriesId;
public string imdbId;
public string zap2itId;
public string firstAired;
public string banner;
public string[] aliases;
public string[] genre;
public string added;
public string airsDayOfWeek;
public string airsTime;
public string lastUpdated;
public string runtime;
public string networkId;
public string rating;
public float siteRating;
public int siteRatingCount;
}
private struct RatingInfo
{
public float average;
public int count;
}
private struct ImageTvDb
{
public string fileName;
public int id;
public string keyType;
public int languageId;
public RatingInfo ratingsInfo;
public string resolution;
public string subKey;
public string thumbnail;
}
private struct ErrorsTvDB
{
public string[] invalidFilters;
public string invalidLanguage;
public string[] invalidQueryParams;
}
public async Task<Show> GetShowFromName(string showName, string showPath)
{
string token = await Authentificate();
if (token != null)
{
WebRequest request = WebRequest.Create("https://api.thetvdb.com/search/series?name=" + HttpUtility.HtmlEncode(showName));
request.Method = "GET";
request.Timeout = 12000;
request.ContentType = "application/json";
request.Headers.Add(HttpRequestHeader.Authorization, "Bearer " + token);
try
{
HttpWebResponse response = (HttpWebResponse)await request.GetResponseAsync();
if (response.StatusCode == HttpStatusCode.OK)
{
Stream stream = response.GetResponseStream();
using (StreamReader reader = new StreamReader(stream))
{
string content = await reader.ReadToEndAsync();
stream.Close();
response.Close();
var model = new { data = new SearchTbDB[0] };
SearchTbDB data = JsonConvert.DeserializeAnonymousType(content, model).data[0];
Show show = new Show(-1,
ToSlug(showName),
data.seriesName,
data.aliases,
showPath,
data.overview,
null, //genres (no info with this request)
GetStatus(data.status),
GetYear(data.firstAired),
null, //endYear
string.Format("{0}={1}|", Provider, data.id));
return await CompleteShow(show);
}
}
else
{
Debug.WriteLine("&TheTvDB Provider couldn't work for this show: " + showName + ".\nError Code: " + response.StatusCode + " Message: " + response.StatusDescription);
response.Close();
}
}
catch (WebException ex)
{
Debug.WriteLine("&TheTvDB Provider couldn't work for this show: " + showName + ".\nError Code: " + ex.Status);
}
}
return new Show() { Slug = ToSlug(showName), Title = showName };
}
public async Task<Show> CompleteShow(Show show)
{
string id = GetId(show.ExternalIDs);
if (id == null)
return show;
string token = await Authentificate();
if (token == null)
return show;
WebRequest request = WebRequest.Create("https://api.thetvdb.com/series/" + id);
request.Method = "GET";
request.Timeout = 12000;
request.ContentType = "application/json";
request.Headers.Add(HttpRequestHeader.Authorization, "Bearer " + token);
try
{
HttpWebResponse response = (HttpWebResponse)await request.GetResponseAsync();
if (response.StatusCode == HttpStatusCode.OK)
{
Stream stream = response.GetResponseStream();
using (StreamReader reader = new StreamReader(stream))
{
string content = await reader.ReadToEndAsync();
stream.Close();
response.Close();
var model = new { data = new DataTvDb(), errors = new ErrorsTvDB() };
DataTvDb data = JsonConvert.DeserializeAnonymousType(content, model).data;
show.Title = data.seriesName;
show.Aliases = data.aliases;
show.Overview = data.overview;
show.Genres = data.genre;
show.Status = GetStatus(data.status);
show.StartYear = GetYear(data.firstAired);
await GetImages(show);
return show;
}
}
else
{
Debug.WriteLine("&TheTvDB Provider couldn't work for the show with the id: " + id + ".\nError Code: " + response.StatusCode + " Message: " + response.StatusDescription);
response.Close();
return show;
}
}
catch(WebException ex)
{
Debug.WriteLine("&TheTvDB Provider couldn't work for the show with the id: " + id + ".\nError Code: " + ex.Status);
return show;
}
}
public async Task<Show> GetImages(Show show)
{
Debug.WriteLine("&Getting images for: " + show.Title);
string id = GetId(show.ExternalIDs);
if (id == null)
return show;
string token = await Authentificate();
if (token == null)
return show;
Dictionary<ImageType, string> imageTypes = new Dictionary<ImageType, string> { { ImageType.Poster, "poster" }, { ImageType.Background, "fanart" } };
foreach (KeyValuePair<ImageType, string> type in imageTypes)
{
WebRequest request = WebRequest.Create("https://api.thetvdb.com/series/" + id + "/images/query?keyType=" + type.Value);
request.Method = "GET";
request.Timeout = 12000;
request.ContentType = "application/json";
request.Headers.Add(HttpRequestHeader.Authorization, "Bearer " + token);
HttpWebResponse response = (HttpWebResponse)await request.GetResponseAsync();
if (response.StatusCode == HttpStatusCode.OK)
{
Stream stream = response.GetResponseStream();
using (StreamReader reader = new StreamReader(stream))
{
string content = await reader.ReadToEndAsync();
stream.Close();
response.Close();
var model = new { data = new ImageTvDb[0], error = new ErrorsTvDB() };
//Should implement language selection here
ImageTvDb data = JsonConvert.DeserializeAnonymousType(content, model).data.OrderByDescending(x => x.ratingsInfo.average).ThenByDescending(x => x.ratingsInfo.count).FirstOrDefault();
IEnumerable<ImageTvDb> datas = JsonConvert.DeserializeAnonymousType(content, model).data.OrderByDescending(x => x.ratingsInfo.average).ThenByDescending(x => x.ratingsInfo.count);
SetImage(show, "https://www.thetvdb.com/banners/" + data.fileName, type.Key);
}
}
else
{
Debug.WriteLine("&TheTvDB Provider couldn't get " + type + " for the show with the id: " + id + ".\nError Code: " + response.StatusCode + " Message: " + response.StatusDescription);
response.Close();
}
}
return show;
}
}
}

View File

@ -1,4 +1,5 @@
using System.Text;
using Kyoo.Models;
using System.Text;
using System.Text.RegularExpressions;
namespace Kyoo.InternalAPI.MetadataProvider
@ -11,7 +12,7 @@ namespace Kyoo.InternalAPI.MetadataProvider
{
if (externalIDs.Contains(Provider))
{
int startIndex = externalIDs.IndexOf(Provider) + Provider.Length;
int startIndex = externalIDs.IndexOf(Provider) + Provider.Length + 1; //The + 1 is for the '='
return externalIDs.Substring(startIndex, externalIDs.IndexOf('|', startIndex) - startIndex);
}
else
@ -44,5 +45,28 @@ namespace Kyoo.InternalAPI.MetadataProvider
return showTitle;
}
public enum ImageType { Poster, Background, Thumbnail, Logo }
public void SetImage(Show show, string imgUrl, ImageType type)
{
switch(type)
{
case ImageType.Poster:
show.ImgPrimary = imgUrl;
break;
case ImageType.Thumbnail:
show.ImgThumb = imgUrl;
break;
case ImageType.Logo:
show.ImgLogo = imgUrl;
break;
case ImageType.Background:
show.ImgBackdrop = imgUrl;
break;
default:
break;
}
}
}
}

View File

@ -1,12 +0,0 @@
namespace Kyoo.Models
{
public class Configuration
{
public string DatabasePath { get; set; }
public Configuration()
{
DatabasePath = @"C:\Projects\database.db";
}
}
}

View File

@ -9,8 +9,10 @@ namespace Kyoo.Models
public string Slug;
public string Title;
public List<string> Aliases;
public IEnumerable<string> Aliases;
public string Path;
public string Overview;
public IEnumerable<string> Genres;
public Status? Status;
public long? StartYear;
@ -18,27 +20,54 @@ namespace Kyoo.Models
public string ImgPrimary;
public string ImgThumb;
public string ImgBanner;
public string ImgLogo;
public string ImgBackdrop;
public string ExternalIDs;
public string GetAliases()
{
return string.Join('|', Aliases);
}
public string GetGenres()
{
return string.Join('|', Genres);
}
public Show() { }
public Show(long id, string slug, string title, List<string> aliases, string overview, Status? status, long? startYear, long? endYear, string imgPrimary, string imgThumb, string imgBanner, string imgLogo, string imgBackdrop, string externalIDs)
public Show(long id, string slug, string title, IEnumerable<string> aliases, string path, string overview, IEnumerable<string> genres, Status? status, long? startYear, long? endYear, string externalIDs)
{
this.id = id;
Slug = slug;
Title = title;
Aliases = aliases;
Path = path;
Overview = overview;
Genres = genres;
Status = status;
StartYear = startYear;
EndYear = endYear;
ExternalIDs = externalIDs;
}
public Show(long id, string slug, string title, IEnumerable<string> aliases, string path, string overview, IEnumerable<string> genres, Status? status, long? startYear, long? endYear, string imgPrimary, string imgThumb, string imgLogo, string imgBackdrop, string externalIDs)
{
this.id = id;
Slug = slug;
Title = title;
Aliases = aliases;
Path = path;
Overview = overview;
Genres = genres;
Status = status;
StartYear = startYear;
EndYear = endYear;
ImgPrimary = imgPrimary;
ImgThumb = imgThumb;
ImgBanner = imgBanner;
ImgLogo = imgLogo;
ImgBackdrop = imgBackdrop;
ExternalIDs = externalIDs;
@ -49,14 +78,15 @@ namespace Kyoo.Models
return new Show((long)reader["id"],
reader["slug"] as string,
reader["title"] as string,
(reader["aliases"] as string)?.Split('|').ToList() ?? null,
(reader["aliases"] as string)?.Split('|') ?? null,
reader["path"] as string,
reader["overview"] as string,
(reader["genres"] as string)?.Split('|') ?? null,
reader["status"] as Status?,
reader["startYear"] as long?,
reader["endYear"] as long?,
reader["imgPrimary"] as string,
reader["imgThumb"] as string,
reader["imgBanner"] as string,
reader["imgLogo"] as string,
reader["imgBackdrop"] as string,
reader["externalIDs"] as string);

View File

@ -32,7 +32,7 @@ namespace Kyoo
services.AddSingleton<ILibraryManager, LibraryManager>();
services.AddHostedService<Crawler>();
services.AddSingleton<IMetadataProvider, ProviderTheTvDB>(); //Shouldn't use it like that, it won't work with multiple providers.
services.AddSingleton<IMetadataProvider, ShowProviderTvDB>(); //Shouldn't use it like that, it won't work with multiple providers.
}
// This method gets called by the runtime. Use this method to configure the HTTP request pipeline.