Merge branch 'master' into add_number_plays_to_tails

dev/releases/3.2.yml

@@ -21,4 +21,16 @@ minor_release_name: "Nicole"
     - "[Technical] Bumped Python and SQLAlchemy versions"
     - "[Distribution] Removed build of arm/v7 image"
 3.2.1:
-    notes: []
+    commit: "5495d6e38d95c0c2128e1de9a9553b55b6be945b"
+    notes:
+    - "[Feature] Added setting for custom week offset"
+    - "[Feature] Added Musicbrainz album art fetching"
+    - "[Bugfix] Fixed album entity rows being marked as track entity rows"
+    - "[Bugfix] Fixed scrobbling of tracks when all artists have been removed by server parsing"
+    - "[Bugfix] Fixed Spotify import of multiple files"
+    - "[Bugfix] Fixed process control on FreeBSD"
+    - "[Bugfix] Fixed Spotify authentication thread blocking the process from terminating"
+    - "[Technical] Upgraded all third party modules to use requests module and send User Agent"
+3.2.2:
+    notes:
+    - "[Bugfix] Fixed Last.fm authentication"

maloja/__main__.py

@@ -160,6 +160,14 @@ def print_info():
 	except Exception:
 		print("Could not determine system information.")
 
+
+def print_settings():
+	print_header_info()
+	maxlen = max(len(k) for k in conf.malojaconfig)
+	for k in conf.malojaconfig:
+		print(col['lightblue'](k.ljust(maxlen+2)),conf.malojaconfig[k])
+
+
 @mainfunction({"l":"level","v":"version","V":"version"},flags=['version','include_images','prefer_existing'],shield=True)
 def main(*args,**kwargs):
 
@@ -180,7 +188,8 @@ def main(*args,**kwargs):
 		"apidebug":apidebug.run,				# maloja apidebug
 		"parsealbums":tasks.parse_albums,		# maloja parsealbums --strategy majority
 		# aux
-		"info":print_info
+		"info":print_info,
+		"settings":print_settings
 	}
 
 	if "version" in kwargs:

maloja/__pkginfo__.py

@@ -4,7 +4,7 @@
 # you know what f*ck it
 # this is hardcoded for now because of that damn project / package name discrepancy
 # i'll fix it one day
-VERSION = "3.2.0"
+VERSION = "3.2.1"
 HOMEPAGE = "https://github.com/krateng/maloja"
 
 

maloja/malojatime.py

@@ -214,8 +214,6 @@ class MTRangeWeek(MTRangeSingular):
 		# do this so we can construct the week with overflow (eg 2020/-3)
 		thisisoyear_firstday = date.fromisocalendar(year,1,1) + timedelta(days=malojaconfig['WEEK_OFFSET']-1)
 		self.firstday = thisisoyear_firstday + timedelta(days=7*(week-1))
-		self.firstday = date(self.firstday.year,self.firstday.month,self.firstday.day)
-		# for compatibility with pre python3.8 (https://bugs.python.org/issue32417)
 
 		self.lastday = self.firstday + timedelta(days=6)
 

maloja/pkg_global/conf.py

@@ -164,7 +164,7 @@ malojaconfig = Configuration(
 			"name":(tp.String(),												"Name",							"Generic Maloja User")
 		},
 		"Third Party Services":{
-			"metadata_providers":(tp.List(tp.String()),							"Metadata Providers",			['lastfm','spotify','deezer','musicbrainz'],	"Which metadata providers should be used in what order. Musicbrainz is rate-limited and should not be used first."),
+			"metadata_providers":(tp.List(tp.String()),							"Metadata Providers",			['lastfm','spotify','deezer','audiodb','musicbrainz'],	"Which metadata providers should be used in what order. Musicbrainz is rate-limited and should not be used first."),
 			"scrobble_lastfm":(tp.Boolean(),									"Proxy-Scrobble to Last.fm",	False),
 			"lastfm_api_key":(tp.String(),										"Last.fm API Key",				None),
 			"lastfm_api_secret":(tp.String(),									"Last.fm API Secret",			None),
@@ -190,7 +190,8 @@ malojaconfig = Configuration(
 			"delimiters_formal":(tp.Set(tp.String()),							"Formal Delimiters",			[";","/","|","␝","␞","␟"],																	"Delimiters used to tag multiple artists when only one tag field is available"),
 			"filters_remix":(tp.Set(tp.String()),								"Remix Filters",				["Remix", "Remix Edit", "Short Mix", "Extended Mix", "Soundtrack Version"],					"Filters used to recognize the remix artists in the title"),
 			"parse_remix_artists":(tp.Boolean(),								"Parse Remix Artists",			False),
-			"week_offset":(tp.Integer(),										"Week Begin Offset",			0,																							"Start of the week for the purpose of weekly statistics. 0 = Sunday, 6 = Saturday")
+			"week_offset":(tp.Integer(),										"Week Begin Offset",			0,																							"Start of the week for the purpose of weekly statistics. 0 = Sunday, 6 = Saturday"),
+			"timezone":(tp.Integer(),											"UTC Offset",					0)
 		},
 		"Web Interface":{
 			"default_range_startpage":(tp.Choice({'alltime':'All Time','year':'Year','month':"Month",'week':'Week'}),	"Default Range for Startpage Stats",	"year"),
@@ -200,12 +201,11 @@ malojaconfig = Configuration(
 			"display_art_icons":(tp.Boolean(),									"Display Album/Artist Icons",	True),
 			"default_album_artist":(tp.String(),								"Default Albumartist",			"Various Artists"),
 			"use_album_artwork_for_tracks":(tp.Boolean(),						"Use Album Artwork for tracks",	True),
-			"fancy_placeholder_art":(tp.Boolean(),								"Use fancy placeholder artwork",True),
+			"fancy_placeholder_art":(tp.Boolean(),								"Use fancy placeholder artwork",False),
 			"show_play_number_on_tiles":(tp.Boolean(),							"Show amount of plays on tails",			True),
 			"discourage_cpu_heavy_stats":(tp.Boolean(),							"Discourage CPU-heavy stats",	False,					"Prevent visitors from mindlessly clicking on CPU-heavy options. Does not actually disable them for malicious actors!"),
 			"use_local_images":(tp.Boolean(),									"Use Local Images",				True),
 			#"local_image_rotate":(tp.Integer(),									"Local Image Rotate",			3600),
-			"timezone":(tp.Integer(),											"UTC Offset",					0),
 			"time_format":(tp.String(),											"Time Format",					"%d. %b %Y %I:%M %p"),
 			"theme":(tp.String(),												"Theme",						"maloja")
 		}

maloja/thirdparty/__init__.py

@@ -7,15 +7,16 @@
 # pls don't sue me
 
 import xml.etree.ElementTree as ElementTree
-import json
+import requests
-import urllib.parse, urllib.request
+import urllib.parse
 import base64
 import time
 from doreah.logging import log
-from threading import BoundedSemaphore
+from threading import BoundedSemaphore, Thread
 
 from ..pkg_global.conf import malojaconfig
 from .. import database
+from ..__pkginfo__ import USER_AGENT
 
 
 services = {
@@ -51,6 +52,7 @@ def proxy_scrobble_all(artists,title,timestamp):
 def get_image_track_all(track):
 	with thirdpartylock:
 		for service in services["metadata"]:
+			if "track" not in service.metadata["enabled_entity_types"]: continue
 			try:
 				res = service.get_image_track(track)
 				if res:
@@ -63,6 +65,7 @@ def get_image_track_all(track):
 def get_image_artist_all(artist):
 	with thirdpartylock:
 		for service in services["metadata"]:
+			if "artist" not in service.metadata["enabled_entity_types"]: continue
 			try:
 				res = service.get_image_artist(artist)
 				if res:
@@ -75,6 +78,7 @@ def get_image_artist_all(artist):
 def get_image_album_all(album):
 	with thirdpartylock:
 		for service in services["metadata"]:
+			if "album" not in service.metadata["enabled_entity_types"]: continue
 			try:
 				res = service.get_image_album(album)
 				if res:
@@ -100,12 +104,17 @@ class GenericInterface:
 	scrobbleimport = {}
 	metadata = {}
 
+	useragent = USER_AGENT
+
 	def __init__(self):
 		# populate from settings file once on creation
 		# avoid constant disk access, restart on adding services is acceptable
 		for key in self.settings:
 			self.settings[key] = malojaconfig[self.settings[key]]
-		self.authorize()
+		t = Thread(target=self.authorize)
+		t.daemon = True
+		t.start()
+		#self.authorize()
 
 	# this makes sure that of every class we define, we immediately create an
 	# instance (de facto singleton). then each instance checks if the requirements
@@ -127,16 +136,6 @@ class GenericInterface:
 		return True
 		# per default. no authorization is necessary
 
-	# wrapper method
-	def request(self,url,data,responsetype):
-		response = urllib.request.urlopen(
-			url,
-			data=utf(data)
-		)
-		responsedata = response.read()
-		if responsetype == "xml":
-			data = ElementTree.fromstring(responsedata)
-			return data
 
 # proxy scrobbler
 class ProxyScrobbleInterface(GenericInterface,abstract=True):
@@ -155,11 +154,15 @@ class ProxyScrobbleInterface(GenericInterface,abstract=True):
 		)
 
 	def scrobble(self,artists,title,timestamp):
-		response = urllib.request.urlopen(
+		response = requests.post(
-			self.proxyscrobble["scrobbleurl"],
+			url=self.proxyscrobble["scrobbleurl"],
-			data=utf(self.proxyscrobble_postdata(artists,title,timestamp)))
+			data=self.proxyscrobble_postdata(artists,title,timestamp),
-		responsedata = response.read()
+			headers={
+				"User-Agent":self.useragent
+			}
+		)
 		if self.proxyscrobble["response_type"] == "xml":
+			responsedata = response.text
 			data = ElementTree.fromstring(responsedata)
 			return self.proxyscrobble_parse_response(data)
 
@@ -211,13 +214,15 @@ class MetadataInterface(GenericInterface,abstract=True):
 		artists, title = track
 		artiststring = urllib.parse.quote(", ".join(artists))
 		titlestring = urllib.parse.quote(title)
-		response = urllib.request.urlopen(
+		response = requests.get(
-			self.metadata["trackurl"].format(artist=artiststring,title=titlestring,**self.settings)
+			self.metadata["trackurl"].format(artist=artiststring,title=titlestring,**self.settings),
+			headers={
+				"User-Agent":self.useragent
+			}
 		)
 
-		responsedata = response.read()
 		if self.metadata["response_type"] == "json":
-			data = json.loads(responsedata)
+			data = response.json()
 			imgurl = self.metadata_parse_response_track(data)
 		else:
 			imgurl = None
@@ -227,13 +232,15 @@ class MetadataInterface(GenericInterface,abstract=True):
 
 	def get_image_artist(self,artist):
 		artiststring = urllib.parse.quote(artist)
-		response = urllib.request.urlopen(
+		response = requests.get(
-			self.metadata["artisturl"].format(artist=artiststring,**self.settings)
+			self.metadata["artisturl"].format(artist=artiststring,**self.settings),
+			headers={
+				"User-Agent":self.useragent
+			}
 		)
 
-		responsedata = response.read()
 		if self.metadata["response_type"] == "json":
-			data = json.loads(responsedata)
+			data = response.json()
 			imgurl = self.metadata_parse_response_artist(data)
 		else:
 			imgurl = None
@@ -245,13 +252,15 @@ class MetadataInterface(GenericInterface,abstract=True):
 		artists, title = album
 		artiststring = urllib.parse.quote(", ".join(artists or []))
 		titlestring = urllib.parse.quote(title)
-		response = urllib.request.urlopen(
+		response = requests.get(
-			self.metadata["albumurl"].format(artist=artiststring,title=titlestring,**self.settings)
+			self.metadata["albumurl"].format(artist=artiststring,title=titlestring,**self.settings),
+			headers={
+				"User-Agent":self.useragent
+			}
 		)
 
-		responsedata = response.read()
 		if self.metadata["response_type"] == "json":
-			data = json.loads(responsedata)
+			data = response.json()
 			imgurl = self.metadata_parse_response_album(data)
 		else:
 			imgurl = None

maloja/thirdparty/audiodb.py

@@ -16,6 +16,7 @@ class AudioDB(MetadataInterface):
 		#"response_parse_tree_track": ["tracks",0,"astrArtistThumb"],
 		"response_parse_tree_artist": ["artists",0,"strArtistThumb"],
 		"required_settings": ["api_key"],
+		"enabled_entity_types": ["artist"]
 	}
 
 	def get_image_track(self,track):

maloja/thirdparty/deezer.py

@@ -17,6 +17,7 @@ class Deezer(MetadataInterface):
 		"response_parse_tree_artist": ["data",0,"artist","picture_medium"],
 		"response_parse_tree_album": ["data",0,"album","cover_medium"],
 		"required_settings": [],
+		"enabled_entity_types": ["artist","album"]
 	}
 
 	delay = 1

maloja/thirdparty/lastfm.py

@@ -1,6 +1,7 @@
 from . import MetadataInterface, ProxyScrobbleInterface, utf
 import hashlib
-import urllib.parse, urllib.request
+import requests
+import xml.etree.ElementTree as ElementTree
 from doreah.logging import log
 
 class LastFM(MetadataInterface, ProxyScrobbleInterface):
@@ -31,6 +32,7 @@ class LastFM(MetadataInterface, ProxyScrobbleInterface):
 		#"response_parse_tree_artist": ["artist","image",-1,"#text"],
 		"response_parse_tree_album": ["album","image",-1,"#text"],
 		"required_settings": ["apikey"],
+		"enabled_entity_types": ["track","album"]
 	}
 
 	def get_image_artist(self,artist):
@@ -53,28 +55,39 @@ class LastFM(MetadataInterface, ProxyScrobbleInterface):
 		})
 
 	def authorize(self):
+		if all(self.settings[key] not in [None,"ASK",False] for key in ["username","password","apikey","secret"]):
 			try:
-			result = self.request(
+				response = requests.post(
-				self.proxyscrobble['scrobbleurl'],
+					url=self.proxyscrobble['scrobbleurl'],
-				self.query_compose({
+					params=self.query_compose({
 						"method":"auth.getMobileSession",
 						"username":self.settings["username"],
 						"password":self.settings["password"],
 						"api_key":self.settings["apikey"]
 					}),
-				responsetype="xml"
+					headers={
+						"User-Agent":self.useragent
+					}
 				)
-			self.settings["sk"] = result.find("session").findtext("key")
+
+				data = ElementTree.fromstring(response.text)
+				self.settings["sk"] = data.find("session").findtext("key")
 			except Exception as e:
-			pass
+				log("Error while authenticating with LastFM: " + repr(e))
-			#log("Error while authenticating with LastFM: " + repr(e))
 
 
-	# creates signature and returns full query string
+	# creates signature and returns full query
 	def query_compose(self,parameters):
 		m = hashlib.md5()
 		keys = sorted(str(k) for k in parameters)
 		m.update(utf("".join(str(k) + str(parameters[k]) for k in keys)))
 		m.update(utf(self.settings["secret"]))
 		sig = m.hexdigest()
-		return urllib.parse.urlencode(parameters) + "&api_sig=" + sig
+		return {**parameters,"api_sig":sig}
+
+	def handle_json_result_error(self,result):
+		if "track" in result and not result.get("track").get('album',{}):
+			return True
+
+		if "error" in result and result.get("error") == 6:
+			return True

maloja/thirdparty/maloja.py

@@ -1,5 +1,5 @@
 from . import ProxyScrobbleInterface, ImportInterface
-import urllib.request
+import requests
 from doreah.logging import log
 import json
 
@@ -32,8 +32,8 @@ class OtherMalojaInstance(ProxyScrobbleInterface, ImportInterface):
 	def get_remote_scrobbles(self):
 		url = f"{self.settings['instance']}/apis/mlj_1/scrobbles"
 
-		response = urllib.request.urlopen(url)
+		response = requests.get(url)
-		data = json.loads(response.read().decode('utf-8'))
+		data = response.json()
 
 		for scrobble in data['list']:
 			yield scrobble

maloja/thirdparty/musicbrainz.py

@@ -1,9 +1,7 @@
 from . import MetadataInterface
-import urllib.parse, urllib.request
+import requests
-import json
 import time
 import threading
-from ..__pkginfo__ import USER_AGENT
 
 class MusicBrainz(MetadataInterface):
 	name = "MusicBrainz"
@@ -11,15 +9,17 @@ class MusicBrainz(MetadataInterface):
 
 	# musicbrainz is rate-limited
 	lock = threading.Lock()
-	useragent = USER_AGENT
+
+
+	thumbnailsize_order = ['500','large','1200','250','small']
 
 	settings = {
 	}
 
 	metadata = {
 		"response_type":"json",
-		"response_parse_tree_track": ["images",0,"thumbnails","500"],
 		"required_settings": [],
+		"enabled_entity_types": ["album","track"]
 	}
 
 	def get_image_artist(self,artist):
@@ -27,36 +27,104 @@ class MusicBrainz(MetadataInterface):
 		# not supported
 
 	def get_image_album(self,album):
-		return None
-
-	def get_image_track(self,track):
 		self.lock.acquire()
 		try:
-			artists, title = track
+			artists, title = album
-			artiststring = ", ".join(artists) #Join artists collection into string
+			searchstr = f'release:"{title}"'
-			titlestring = title
+			for artist in artists:
-			querystr = urllib.parse.urlencode({
+				searchstr += f' artist:"{artist}"'
+			res = requests.get(**{
+				"url":"https://musicbrainz.org/ws/2/release",
+				"params":{
 					"fmt":"json",
-				"query":"{title} {artist}".format(artist=artiststring,title=titlestring)
+					"query":searchstr
-			})
+				},
-			req = urllib.request.Request(**{
-				"url":"https://musicbrainz.org/ws/2/release?" + querystr,
-				"method":"GET",
 				"headers":{
 					"User-Agent":self.useragent
 				}
 			})
-			response = urllib.request.urlopen(req)
+			data = res.json()
-			responsedata = response.read()
+			entity = data["releases"][0]
-			data = json.loads(responsedata)
+			coverartendpoint = "release"
-			mbid = data["releases"][0]["id"]
+			while True:
-			response = urllib.request.urlopen(
+				mbid = entity["id"]
-				"https://coverartarchive.org/release/{mbid}?fmt=json".format(mbid=mbid)
+				try:
+					response = requests.get(
+						f"https://coverartarchive.org/{coverartendpoint}/{mbid}",
+						params={
+							"fmt":"json"
+						},
+						headers={
+							"User-Agent":self.useragent
+						}
 					)
-			responsedata = response.read()
+					data = response.json()
-			data = json.loads(responsedata)
+					thumbnails = data['images'][0]['thumbnails']
-			imgurl = self.metadata_parse_response_track(data)
+					for size in self.thumbnailsize_order:
-			if imgurl is not None: imgurl = self.postprocess_url(imgurl)
+						if thumbnails.get(size) is not None:
+							imgurl = thumbnails.get(size)
+							continue
+				except:
+					imgurl = None
+				if imgurl is None:
+					entity = entity["release-group"]
+					# this will raise an error so we don't stay in the while loop forever
+					coverartendpoint = "release-group"
+					continue
+
+				imgurl = self.postprocess_url(imgurl)
+				return imgurl
+
+		except Exception:
+			return None
+		finally:
+			time.sleep(2)
+			self.lock.release()
+
+	def get_image_track(self,track):
+		self.lock.acquire()
+		try:
+			artists, title = track
+			searchstr = f'recording:"{title}"'
+			for artist in artists:
+				searchstr += f' artist:"{artist}"'
+			res = requests.get(**{
+				"url":"https://musicbrainz.org/ws/2/recording",
+				"params":{
+					"fmt":"json",
+					"query":searchstr
+				},
+				"headers":{
+					"User-Agent":self.useragent
+				}
+			})
+			data = res.json()
+			entity = data["recordings"][0]["releases"][0]
+			coverartendpoint = "release"
+			while True:
+				mbid = entity["id"]
+				try:
+					response = requests.get(
+						f"https://coverartarchive.org/{coverartendpoint}/{mbid}",
+						params={
+							"fmt":"json"
+						}
+					)
+					data = response.json()
+					thumbnails = data['images'][0]['thumbnails']
+					for size in self.thumbnailsize_order:
+						if thumbnails.get(size) is not None:
+							imgurl = thumbnails.get(size)
+							continue
+				except:
+					imgurl = None
+				if imgurl is None:
+					entity = entity["release-group"]
+					# this will raise an error so we don't stay in the while loop forever
+					coverartendpoint = "release-group"
+					continue
+
+				imgurl = self.postprocess_url(imgurl)
 				return imgurl
 
 		except Exception:

maloja/thirdparty/spotify.py

@@ -1,6 +1,5 @@
 from . import MetadataInterface, utf, b64
-import urllib.parse, urllib.request
+import requests
-import json
 from threading import Timer
 from doreah.logging import log
 
@@ -22,6 +21,7 @@ class Spotify(MetadataInterface):
 		"response_parse_tree_album": ["albums","items",0,"images",0,"url"],
 		"response_parse_tree_artist": ["artists","items",0,"images",0,"url"],
 		"required_settings": ["apiid","secret"],
+		"enabled_entity_types": ["artist","album","track"]
 	}
 
 	def authorize(self):
@@ -31,15 +31,14 @@ class Spotify(MetadataInterface):
 			try:
 				keys = {
 					"url":"https://accounts.spotify.com/api/token",
-					"method":"POST",
 					"headers":{
-						"Authorization":"Basic " + b64(utf(self.settings["apiid"] + ":" + self.settings["secret"])).decode("utf-8")
+						"Authorization":"Basic " + b64(utf(self.settings["apiid"] + ":" + self.settings["secret"])).decode("utf-8"),
+						"User-Agent": self.useragent
 					},
-					"data":bytes(urllib.parse.urlencode({"grant_type":"client_credentials"}),encoding="utf-8")
+					"data":{"grant_type":"client_credentials"}
 				}
-				req = urllib.request.Request(**keys)
+				res = requests.post(**keys)
-				response = urllib.request.urlopen(req)
+				responsedata = res.json()
-				responsedata = json.loads(response.read())
 				if "error" in responsedata:
 					log("Error authenticating with Spotify: " + responsedata['error_description'])
 					expire = 3600
@@ -47,6 +46,13 @@ class Spotify(MetadataInterface):
 					expire = responsedata.get("expires_in",3600)
 					self.settings["token"] = responsedata["access_token"]
 					#log("Successfully authenticated with Spotify")
-				Timer(expire,self.authorize).start()
+				t = Timer(expire,self.authorize)
+				t.daemon = True
+				t.start()
 			except Exception as e:
 				log("Error while authenticating with Spotify: " + repr(e))
+
+	def handle_json_result_error(self,result):
+		result = result.get('tracks') or result.get('albums') or result.get('artists')
+		if not result['items']:
+			return True

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "malojaserver"
-version = "3.2.0"
+version = "3.2.1"
 description = "Self-hosted music scrobble database"
 readme = "./README.md"
 requires-python = ">=3.10"