Merge branch 'feature-albums' into next_minor_version

maloja/database/__init__.py

@@ -1,6 +1,17 @@
 # server
 from bottle import request, response, FormsDict
 
+
+# we're running an auxiliary task that doesn't require all the random background
+# nonsense to be fired up
+# this is temporary
+# FIX YO DAMN ARCHITECTURE ALREADY
+AUX_MODE = False
+def set_aux_mode():
+	global AUX_MODE
+	AUX_MODE = True
+
+	
 # rest of the project
 from ..cleanup import CleanerAgent
 from .. import images
@@ -45,14 +56,7 @@ dbstatus = {
 }
 
 
-# we're running an auxiliary task that doesn't require all the random background
+
-# nonsense to be fired up
-# this is temporary
-# FIX YO DAMN ARCHITECTURE ALREADY
-AUX_MODE = False
-def set_aux_mode():
-	global AUX_MODE
-	AUX_MODE = True
 
 
 
@@ -110,7 +114,7 @@ def incoming_scrobble(rawscrobble,fix=True,client=None,api=None,dbconn=None):
 	proxy_scrobble_all(scrobbledict['track']['artists'],scrobbledict['track']['title'],scrobbledict['time'])
 
 	dbcache.invalidate_caches(scrobbledict['time'])
-	dbcache.invalidate_entity_cache() # because album info might have changed
+
 
 	#return {"status":"success","scrobble":scrobbledict}
 	return scrobbledict
@@ -151,7 +155,7 @@ def rawscrobble_to_scrobbledict(rawscrobble, fix=True, client=None):
 
 	# if we send [] as albumartists, it means various
 	# if we send nothing, the scrobbler just doesnt support it and we assume track artists
-	if 'album_artists' not in scrobbleinfo:
+	if ('album_title' in scrobbleinfo) and ('album_artists' not in scrobbleinfo):
 		scrobbleinfo['album_artists'] = scrobbleinfo.get('track_artists')
 
 	# New plan, do this further down
@@ -181,6 +185,9 @@ def rawscrobble_to_scrobbledict(rawscrobble, fix=True, client=None):
 		"rawscrobble":rawscrobble
 	}
 
+	if scrobbledict["track"]["album"]["albumtitle"] is None:
+		del scrobbledict["track"]["album"]
+
 	return scrobbledict
 
 
@@ -317,29 +324,29 @@ def get_albums_artist_appears_on(dbconn=None,**keys):
 
 
 @waitfordb
-def get_charts_artists(dbconn=None,**keys):
+def get_charts_artists(dbconn=None,resolve_ids=True,**keys):
 	(since,to) = keys.get('timerange').timestamps()
-	result = sqldb.count_scrobbles_by_artist(since=since,to=to,dbconn=dbconn)
+	result = sqldb.count_scrobbles_by_artist(since=since,to=to,resolve_ids=resolve_ids,dbconn=dbconn)
 	return result
 
 @waitfordb
-def get_charts_tracks(dbconn=None,**keys):
+def get_charts_tracks(dbconn=None,resolve_ids=True,**keys):
 	(since,to) = keys.get('timerange').timestamps()
 	if 'artist' in keys:
-		result = sqldb.count_scrobbles_by_track_of_artist(since=since,to=to,artist=keys['artist'],dbconn=dbconn)
+		result = sqldb.count_scrobbles_by_track_of_artist(since=since,to=to,artist=keys['artist'],resolve_ids=resolve_ids,dbconn=dbconn)
 	elif 'album' in keys:
-		result = sqldb.count_scrobbles_by_track_of_album(since=since,to=to,album=keys['album'],dbconn=dbconn)
+		result = sqldb.count_scrobbles_by_track_of_album(since=since,to=to,album=keys['album'],resolve_ids=resolve_ids,dbconn=dbconn)
 	else:
-		result = sqldb.count_scrobbles_by_track(since=since,to=to,dbconn=dbconn)
+		result = sqldb.count_scrobbles_by_track(since=since,to=to,resolve_ids=resolve_ids,dbconn=dbconn)
 	return result
 
 @waitfordb
-def get_charts_albums(dbconn=None,**keys):
+def get_charts_albums(dbconn=None,resolve_ids=True,**keys):
 	(since,to) = keys.get('timerange').timestamps()
 	if 'artist' in keys:
-		result = sqldb.count_scrobbles_by_album_of_artist(since=since,to=to,artist=keys['artist'],dbconn=dbconn)
+		result = sqldb.count_scrobbles_by_album_of_artist(since=since,to=to,artist=keys['artist'],resolve_ids=resolve_ids,dbconn=dbconn)
 	else:
-		result = sqldb.count_scrobbles_by_album(since=since,to=to,dbconn=dbconn)
+		result = sqldb.count_scrobbles_by_album(since=since,to=to,resolve_ids=resolve_ids,dbconn=dbconn)
 	return result
 
 @waitfordb
@@ -361,29 +368,32 @@ def get_performance(dbconn=None,**keys):
 
 	for rng in rngs:
 		if "track" in keys:
-			track = sqldb.get_track(sqldb.get_track_id(keys['track'],dbconn=dbconn),dbconn=dbconn)
+			track_id = sqldb.get_track_id(keys['track'],dbconn=dbconn)
-			charts = get_charts_tracks(timerange=rng,dbconn=dbconn)
+			#track = sqldb.get_track(track_id,dbconn=dbconn)
+			charts = get_charts_tracks(timerange=rng,resolve_ids=False,dbconn=dbconn)
 			rank = None
 			for c in charts:
-				if c["track"] == track:
+				if c["track_id"] == track_id:
 					rank = c["rank"]
 					break
 		elif "artist" in keys:
-			artist = sqldb.get_artist(sqldb.get_artist_id(keys['artist'],dbconn=dbconn),dbconn=dbconn)
+			artist_id = sqldb.get_artist_id(keys['artist'],dbconn=dbconn)
+			#artist = sqldb.get_artist(artist_id,dbconn=dbconn)
 			# ^this is the most useless line in programming history
 			# but I like consistency
-			charts = get_charts_artists(timerange=rng,dbconn=dbconn)
+			charts = get_charts_artists(timerange=rng,resolve_ids=False,dbconn=dbconn)
 			rank = None
 			for c in charts:
-				if c["artist"] == artist:
+				if c["artist_id"] == artist_id:
 					rank = c["rank"]
 					break
 		elif "album" in keys:
-			album = sqldb.get_album(sqldb.get_album_id(keys['album'],dbconn=dbconn),dbconn=dbconn)
+			album_id = sqldb.get_album_id(keys['album'],dbconn=dbconn)
-			charts = get_charts_albums(timerange=rng,dbconn=dbconn)
+			#album = sqldb.get_album(album_id,dbconn=dbconn)
+			charts = get_charts_albums(timerange=rng,resolve_ids=False,dbconn=dbconn)
 			rank = None
 			for c in charts:
-				if c["album"] == album:
+				if c["album_id"] == album_id:
 					rank = c["rank"]
 					break
 		else:
@@ -444,7 +454,9 @@ def artist_info(dbconn=None,**keys):
 	artist = keys.get('artist')
 	if artist is None: raise exceptions.MissingEntityParameter()
 
-	artist_id = sqldb.get_artist_id(artist,dbconn=dbconn)
+	artist_id = sqldb.get_artist_id(artist,create_new=False,dbconn=dbconn)
+	if not artist_id: raise exceptions.ArtistDoesNotExist(artist)
+
 	artist = sqldb.get_artist(artist_id,dbconn=dbconn)
 	alltimecharts = get_charts_artists(timerange=alltime(),dbconn=dbconn)
 	#we cant take the scrobble number from the charts because that includes all countas scrobbles
@@ -497,12 +509,14 @@ def track_info(dbconn=None,**keys):
 	track = keys.get('track')
 	if track is None: raise exceptions.MissingEntityParameter()
 
-	track_id = sqldb.get_track_id(track,dbconn=dbconn)
+	track_id = sqldb.get_track_id(track,create_new=False,dbconn=dbconn)
+	if not track_id: raise exceptions.TrackDoesNotExist(track['title'])
+
 	track = sqldb.get_track(track_id,dbconn=dbconn)
-	alltimecharts = get_charts_tracks(timerange=alltime(),dbconn=dbconn)
+	alltimecharts = get_charts_tracks(timerange=alltime(),resolve_ids=False,dbconn=dbconn)
 	#scrobbles = get_scrobbles_num(track=track,timerange=alltime())
 
-	c = [e for e in alltimecharts if e["track"] == track][0]
+	c = [e for e in alltimecharts if e["track_id"] == track_id][0]
 	scrobbles = c["scrobbles"]
 	position = c["rank"]
 	cert = None
@@ -533,9 +547,10 @@ def album_info(dbconn=None,**keys):
 	album = keys.get('album')
 	if album is None: raise exceptions.MissingEntityParameter()
 
-	album_id = sqldb.get_album_id(album,dbconn=dbconn)
+	album_id = sqldb.get_album_id(album,create_new=False,dbconn=dbconn)
-	album = sqldb.get_album(album_id,dbconn=dbconn)
+	if not album_id: raise exceptions.AlbumDoesNotExist(album['albumtitle'])
 
+	album = sqldb.get_album(album_id,dbconn=dbconn)
 	alltimecharts = get_charts_albums(timerange=alltime(),dbconn=dbconn)
 
 	#scrobbles = get_scrobbles_num(track=track,timerange=alltime())

maloja/database/dbcache.py

@@ -23,11 +23,13 @@ if malojaconfig['USE_GLOBAL_CACHE']:
 	@runhourly
 	def maintenance():
 		from . import AUX_MODE
-		if not AUX_MODE:
+		if AUX_MODE: return
-			print_stats()
+		print_stats()
-			trim_cache()
+		trim_cache()
 
 	def print_stats():
+		from . import AUX_MODE
+		if AUX_MODE: return
 		for name,c in (('Cache',cache),('Entity Cache',entitycache)):
 			hits, misses = c.get_stats()
 			log(f"{name}: Size: {len(c)} | Hits: {hits}/{hits+misses} | Estimated Memory: {human_readable_size(c)}")
@@ -35,6 +37,8 @@ if malojaconfig['USE_GLOBAL_CACHE']:
 
 
 	def cached_wrapper(inner_func):
+		from . import AUX_MODE
+		if AUX_MODE: return inner_func
 
 		def outer_func(*args,**kwargs):
 
@@ -59,6 +63,8 @@ if malojaconfig['USE_GLOBAL_CACHE']:
 	# we don't want a new cache entry for every single combination, but keep a common
 	# cache that's aware of what we're calling
 	def cached_wrapper_individual(inner_func):
+		from . import AUX_MODE
+		if AUX_MODE: return
 
 		def outer_func(set_arg,**kwargs):
 			if 'dbconn' in kwargs:
@@ -83,6 +89,9 @@ if malojaconfig['USE_GLOBAL_CACHE']:
 		return outer_func
 
 	def invalidate_caches(scrobbletime=None):
+		from . import AUX_MODE
+		if AUX_MODE: return
+
 		cleared, kept = 0, 0
 		for k in cache.keys():
 			# VERY BIG TODO: differentiate between None as in 'unlimited timerange' and None as in 'time doesnt matter here'!
@@ -95,10 +104,14 @@ if malojaconfig['USE_GLOBAL_CACHE']:
 
 
 	def invalidate_entity_cache():
+		from . import AUX_MODE
+		if AUX_MODE: return
 		entitycache.clear()
 
 
 	def trim_cache():
+		from . import AUX_MODE
+		if AUX_MODE: return
 		ramprct = psutil.virtual_memory().percent
 		if ramprct > malojaconfig["DB_MAX_MEMORY"]:
 			log(f"{ramprct}% RAM usage, clearing cache!")

maloja/database/exceptions.py

@@ -27,3 +27,19 @@ class MissingScrobbleParameters(Exception):
 
 class MissingEntityParameter(Exception):
 	pass
+
+class EntityDoesNotExist(HTTPError):
+	entitytype = 'Entity'
+	def __init__(self,name):
+		self.entityname = name
+		super().__init__(
+			status=404,
+			body=f"The {self.entitytype} '{self.entityname}' does not exist in the database."
+		)
+
+class ArtistDoesNotExist(EntityDoesNotExist):
+	entitytype = 'Artist'
+class AlbumDoesNotExist(EntityDoesNotExist):
+	entitytype = 'Album'
+class TrackDoesNotExist(EntityDoesNotExist):
+	entitytype = 'Track'

maloja/database/sqldb.py

@@ -6,7 +6,7 @@ from datetime import datetime
 from threading import Lock
 
 from ..pkg_global.conf import data_dir
-from .dbcache import cached_wrapper, cached_wrapper_individual
+from .dbcache import cached_wrapper, cached_wrapper_individual, invalidate_caches, invalidate_entity_cache
 from . import exceptions as exc
 
 from doreah.logging import log
@@ -352,6 +352,11 @@ def add_track_to_album(track_id,album_id,replace=False,dbconn=None):
 	)
 
 	result = dbconn.execute(op)
+
+	invalidate_entity_cache() # because album info has changed
+	invalidate_caches() # changing album info of tracks will change album charts
+
+
 	return True
 
 @connection_provider
@@ -391,17 +396,21 @@ def get_track_id(trackdict,create_new=True,update_album=False,dbconn=None):
 		#print("required artists",artist_ids,"this match",match_artist_ids)
 		if set(artist_ids) == set(match_artist_ids):
 			#print("ID for",trackdict['title'],"was",row[0])
-			if trackdict.get('album'):
+			if trackdict.get('album') and create_new:
+				# if we don't supply create_new, it means we just want to get info about a track
+				# which means no need to write album info, even if it was new
 				add_track_to_album(row.id,get_album_id(trackdict['album'],dbconn=dbconn),replace=update_album,dbconn=dbconn)
 			return row.id
 
 	if not create_new: return None
 
+	print("Creating new track")
 	op = DB['tracks'].insert().values(
 		**track_dict_to_db(trackdict,dbconn=dbconn)
 	)
 	result = dbconn.execute(op)
 	track_id = result.inserted_primary_key[0]
+	print(track_id)
 
 	for artist_id in artist_ids:
 		op = DB['trackartists'].insert().values(
@@ -866,7 +875,6 @@ def count_scrobbles_by_artist(since,to,resolve_ids=True,dbconn=None):
 	result = dbconn.execute(op).all()
 
 	if resolve_ids:
-		counts = [row.count for row in result]
 		artists = get_artists_map([row.artist_id for row in result],dbconn=dbconn)
 		result = [{'scrobbles':row.count,'artist':artists[row.artist_id]} for row in result]
 	else:
@@ -889,7 +897,6 @@ def count_scrobbles_by_track(since,to,resolve_ids=True,dbconn=None):
 	result = dbconn.execute(op).all()
 
 	if resolve_ids:
-		counts = [row.count for row in result]
 		tracks = get_tracks_map([row.track_id for row in result],dbconn=dbconn)
 		result = [{'scrobbles':row.count,'track':tracks[row.track_id]} for row in result]
 	else:
@@ -918,7 +925,6 @@ def count_scrobbles_by_album(since,to,resolve_ids=True,dbconn=None):
 	result = dbconn.execute(op).all()
 
 	if resolve_ids:
-		counts = [row.count for row in result]
 		albums = get_albums_map([row.album_id for row in result],dbconn=dbconn)
 		result = [{'scrobbles':row.count,'album':albums[row.album_id]} for row in result]
 	else:
@@ -956,7 +962,6 @@ def count_scrobbles_by_album_of_artist(since,to,artist,resolve_ids=True,dbconn=N
 	result = dbconn.execute(op).all()
 
 	if resolve_ids:
-		counts = [row.count for row in result]
 		albums = get_albums_map([row.album_id for row in result],dbconn=dbconn)
 		result = [{'scrobbles':row.count,'album':albums[row.album_id]} for row in result]
 	else:
@@ -994,7 +999,6 @@ def count_scrobbles_of_artist_by_album(since,to,artist,resolve_ids=True,dbconn=N
 	result = dbconn.execute(op).all()
 
 	if resolve_ids:
-		counts = [row.count for row in result]
 		albums = get_albums_map([row.album_id for row in result],dbconn=dbconn)
 		result = [{'scrobbles':row.count,'album':albums[row.album_id]} for row in result]
 	else:
@@ -1005,7 +1009,7 @@ def count_scrobbles_of_artist_by_album(since,to,artist,resolve_ids=True,dbconn=N
 
 @cached_wrapper
 @connection_provider
-def count_scrobbles_by_track_of_artist(since,to,artist,dbconn=None):
+def count_scrobbles_by_track_of_artist(since,to,artist,resolve_ids=True,dbconn=None):
 
 	artist_id = get_artist_id(artist,dbconn=dbconn)
 
@@ -1026,16 +1030,18 @@ def count_scrobbles_by_track_of_artist(since,to,artist,dbconn=None):
 	result = dbconn.execute(op).all()
 
 
-	counts = [row.count for row in result]
+	if resolve_ids:
-	tracks = get_tracks_map([row.track_id for row in result],dbconn=dbconn)
+		tracks = get_tracks_map([row.track_id for row in result],dbconn=dbconn)
-	result = [{'scrobbles':row.count,'track':tracks[row.track_id]} for row in result]
+		result = [{'scrobbles':row.count,'track':tracks[row.track_id]} for row in result]
+	else:
+		result = [{'scrobbles':row.count,'track_id':row.track_id} for row in result]
 	result = rank(result,key='scrobbles')
 	return result
 
 
 @cached_wrapper
 @connection_provider
-def count_scrobbles_by_track_of_album(since,to,album,dbconn=None):
+def count_scrobbles_by_track_of_album(since,to,album,resolve_ids=True,dbconn=None):
 
 	album_id = get_album_id(album,dbconn=dbconn)
 
@@ -1056,9 +1062,11 @@ def count_scrobbles_by_track_of_album(since,to,album,dbconn=None):
 	result = dbconn.execute(op).all()
 
 
-	counts = [row.count for row in result]
+	if resolve_ids:
-	tracks = get_tracks_map([row.track_id for row in result],dbconn=dbconn)
+		tracks = get_tracks_map([row.track_id for row in result],dbconn=dbconn)
-	result = [{'scrobbles':row.count,'track':tracks[row.track_id]} for row in result]
+		result = [{'scrobbles':row.count,'track':tracks[row.track_id]} for row in result]
+	else:
+		result = [{'scrobbles':row.count,'track_id':row.track_id} for row in result]
 	result = rank(result,key='scrobbles')
 	return result
 
@@ -1356,40 +1364,40 @@ def search_album(searchterm,dbconn=None):
 def clean_db(dbconn=None):
 
 	from . import AUX_MODE
+	if AUX_MODE: return
 
-	if not AUX_MODE:
+	with SCROBBLE_LOCK:
-		with SCROBBLE_LOCK:
+		log(f"Database Cleanup...")
-			log(f"Database Cleanup...")
 
-			to_delete = [
+		to_delete = [
-				# tracks with no scrobbles (trackartist entries first)
+			# tracks with no scrobbles (trackartist entries first)
-				"from trackartists where track_id in (select id from tracks where id not in (select track_id from scrobbles))",
+			"from trackartists where track_id in (select id from tracks where id not in (select track_id from scrobbles))",
-				"from tracks where id not in (select track_id from scrobbles)",
+			"from tracks where id not in (select track_id from scrobbles)",
-				# artists with no tracks AND no albums
+			# artists with no tracks AND no albums
-				"from artists where id not in (select artist_id from trackartists) \
+			"from artists where id not in (select artist_id from trackartists) \
-					and id not in (select target_artist from associated_artists) \
+				and id not in (select target_artist from associated_artists) \
-					and id not in (select artist_id from albumartists)",
+				and id not in (select artist_id from albumartists)",
-				# tracks with no artists (scrobbles first)
+			# tracks with no artists (scrobbles first)
-				"from scrobbles where track_id in (select id from tracks where id not in (select track_id from trackartists))",
+			"from scrobbles where track_id in (select id from tracks where id not in (select track_id from trackartists))",
-				"from tracks where id not in (select track_id from trackartists)",
+			"from tracks where id not in (select track_id from trackartists)",
-				# albums with no tracks (albumartist entries first)
+			# albums with no tracks (albumartist entries first)
-				"from albumartists where album_id in (select id from albums where id not in (select album_id from tracks where album_id is not null))",
+			"from albumartists where album_id in (select id from albums where id not in (select album_id from tracks where album_id is not null))",
-				"from albums where id not in (select album_id from tracks where album_id is not null)",
+			"from albums where id not in (select album_id from tracks where album_id is not null)",
-				# albumartist entries that are missing a reference
+			# albumartist entries that are missing a reference
-				"from albumartists where album_id not in (select album_id from tracks where album_id is not null)",
+			"from albumartists where album_id not in (select album_id from tracks where album_id is not null)",
-				"from albumartists where artist_id not in (select id from artists)",
+			"from albumartists where artist_id not in (select id from artists)",
-				# trackartist entries that mare missing a reference
+			# trackartist entries that mare missing a reference
-				"from trackartists where track_id not in (select id from tracks)",
+			"from trackartists where track_id not in (select id from tracks)",
-				"from trackartists where artist_id not in (select id from artists)"
+			"from trackartists where artist_id not in (select id from artists)"
-			]
+		]
 
-			for d in to_delete:
+		for d in to_delete:
-				selection = dbconn.execute(sql.text(f"select * {d}"))
+			selection = dbconn.execute(sql.text(f"select * {d}"))
-				for row in selection.all():
+			for row in selection.all():
-					log(f"Deleting {row}")
+				log(f"Deleting {row}")
-				deletion = dbconn.execute(sql.text(f"delete {d}"))
+			deletion = dbconn.execute(sql.text(f"delete {d}"))
 
-			log("Database Cleanup complete!")
+		log("Database Cleanup complete!")
 
 
 
@@ -1403,6 +1411,10 @@ def clean_db(dbconn=None):
 
 @runmonthly
 def renormalize_names():
+
+	from . import AUX_MODE
+	if AUX_MODE: return
+
 	with SCROBBLE_LOCK:
 		with engine.begin() as conn:
 			rows = conn.execute(DB['artists'].select()).all()
@@ -1505,7 +1517,7 @@ def guess_albums(track_ids=None,replace=False,dbconn=None):
 
 	# get all scrobbles of the respective tracks that have some info
 	conditions = [
-		DB['scrobbles'].c.extra.isnot(None)
+		DB['scrobbles'].c.extra.isnot(None) | DB['scrobbles'].c.rawscrobble.isnot(None)
 	]
 	if track_ids is not None:
 		# only do these tracks
@@ -1529,10 +1541,13 @@ def guess_albums(track_ids=None,replace=False,dbconn=None):
 	# for each track, count what album info appears how often
 	possible_albums = {}
 	for row in result:
-		extrainfo = json.loads(row.extra)
+		albumtitle, albumartists = None, None
-		albumtitle = extrainfo.get("album_name") or extrainfo.get("album_title")
+		if row.extra:
-		albumartists = extrainfo.get("album_artists",[])
+			extrainfo = json.loads(row.extra)
+			albumtitle = extrainfo.get("album_name") or extrainfo.get("album_title")
+			albumartists = extrainfo.get("album_artists",[])
 		if not albumtitle:
+			# either we didn't have info in the exta col, or there was no albumtitle
 			# try the raw scrobble
 			extrainfo = json.loads(row.rawscrobble)
 			albumtitle = extrainfo.get("album_name") or extrainfo.get("album_title")

maloja/images.py

@@ -111,20 +111,20 @@ def dl_image(url):
 ### even if we have already cached it, we will handle that on request
 def get_track_image(track=None,track_id=None):
 	if track_id is None:
-		track_id = database.sqldb.get_track_id(track)
+		track_id = database.sqldb.get_track_id(track,create_new=False)
 
 	return f"/image?type=track&id={track_id}"
 
 
 def get_artist_image(artist=None,artist_id=None):
 	if artist_id is None:
-		artist_id = database.sqldb.get_artist_id(artist)
+		artist_id = database.sqldb.get_artist_id(artist,create_new=False)
 
 	return f"/image?type=artist&id={artist_id}"
 
 def get_album_image(album=None,album_id=None):
 	if album_id is None:
-		album_id = database.sqldb.get_album_id(album)
+		album_id = database.sqldb.get_album_id(album,create_new=False)
 
 	return f"/image?type=album&id={album_id}"
 

maloja/web/jinja/partials/awards_album.jinja

@@ -28,7 +28,7 @@
 
 {% macro topweeks(info) %}
 
-{% set encodedtrack = mlj_uri.uriencode({'album':info.album}) %}
+{% set encodedalbum = mlj_uri.uriencode({'album':info.album}) %}
 
 <!-- TOPWEEKS -->
 <span>