Moved Last.FM image caching to doreah

.doreah

@@ -1,2 +1,3 @@
 logging.logfolder = logs
 settings.files = [ "settings/default.ini" , "settings/settings.ini" ]
+caching.folder = "images/cache/"

database.py

@@ -785,7 +785,7 @@ def build_db():
 	db_rulestate = consistentRulestate("scrobbles",cla.checksums)
 
 	# load cached images
-	loadCache()
+	#loadCache()
 
 	log("Database fully built!")
 
@@ -826,7 +826,7 @@ def sync():
 	log("Database saved to disk.")
 
 	# save cached images
-	saveCache()
+	#saveCache()
 
 
 

rules/predefined/krateng_kpopgirlgroups.tsv

@@ -29,13 +29,13 @@ replaceartist	BLΛƆKPIИK				BLACKPINK
 replaceartist	Dal Shabet				Dal★Shabet
 replaceartist	Dal shabet				Dal★Shabet
 replaceartist	Dalshabet				Dal★Shabet
-replaceartist	Dalshabet(달샤벳)				Dal★Shabet
+replaceartist	Dalshabet(달샤벳)			Dal★Shabet
 replaceartist	달샤벳(Dal★shabet) (Dalshabet)		Dal★Shabet
 replacetitle	JOKER (Inst.)				Joker (Instrumental)
 replacetitle	JOKER (Instrumental)			Joker (Instrumental)
 replacetitle	Joker (inst.)				Joker (Instrumental)
 replacetitle	Someone like U				Someone Like U
-replacetitle	Be Ambitious(내 다리를 봐)			Be Ambitious
+replacetitle	Be Ambitious(내 다리를 봐)		Be Ambitious
 
 # f(x)
 replacetitle	When I’m Alone				When I'm Alone
@@ -97,7 +97,7 @@ replacetitle	PYONG PYONG (Shooting Love)		PYONG PYONG
 replacetitle	PYONG PYONG (Shooting Love) (inst)	PYONG PYONG (instrumental)
 
 # Bambino
-replaceartist	밤비노 (Bambino)				Bambino
+replaceartist	밤비노 (Bambino)			Bambino
 
 # Laysha
 replaceartist	LAYSHA					Laysha
@@ -108,3 +108,7 @@ replaceartist	여자친구 GFriend			GFriend
 
 # Girl's Generation
 replaceartist	소녀시대				Girls' Generation
+
+# Apink
+replaceartist	A Pink					Apink
+replaceartist	A pink					Apink

utilities.py

@@ -6,6 +6,7 @@ import pickle
 import urllib
 import datetime
 from doreah import settings
+from doreah import caching
 from doreah.logging import log
 
 
@@ -262,114 +263,138 @@ def apirequest(artists=None,artist=None,title=None):
 
 		return None
 
+
+
+
+### Caches
+
+cacheage = settings.get_settings("CACHE_EXPIRE_POSITIVE") * 24 * 3600
+cacheage_neg = settings.get_settings("CACHE_EXPIRE_NEGATIVE") * 24 * 3600
+
+artist_cache = caching.Cache.create(name="artist_cache",maxage=cacheage,maxage_negative=cacheage_neg)
+track_cache = caching.Cache.create(name="track_cache",maxage=cacheage,maxage_negative=cacheage_neg)
+
+
+
 # I think I've only just understood modules
-cachedTracks = {}
+#cachedTracks = {}
-cachedArtists = {}
+#cachedArtists = {}
+#
+#cachedTracksDays = {}
+#cachedArtistsDays = {}
+#
+#def cache_track(artists,title,result):
+#	cachedTracks[(frozenset(artists),title)] = result
+#	day = datetime.date.today().toordinal()
+#	cachedTracksDays[(frozenset(artists),title)] = day
+#def cache_artist(artist,result):
+#	if result is None: log("Caching None for " + artist,module="debug")
+#	cachedArtists[artist] = result
+#	day = datetime.date.today().toordinal()
+#	cachedArtistsDays[artist] = day
 
-cachedTracksDays = {}
+#def track_from_cache(artists,title):
-cachedArtistsDays = {}
+#	try:
+#		res = cachedTracks[(frozenset(artists),title)]
+#	except:
+#		# no entry there, let the calling function know
+#		raise KeyError()
+#
+#	if res is None:
+#		retain = settings.get_settings("CACHE_EXPIRE_NEGATIVE")
+#	else:
+#		retain = settings.get_settings("CACHE_EXPIRE_POSITIVE")
+#
+#	# if the settings say caches never expire, just return
+#	if retain is None: return res
+#
+#	# look if entry is too old
+#	nowday = datetime.date.today().toordinal()
+#	cacheday = cachedTracksDays[(frozenset(artists),title)]
+#
+#	if (nowday - cacheday) > retain:
+#		# fetch the new image in the background, but still return the old one for one last time
+#		log("Expired cache for " + "/".join(artists) + " - " + title)
+#		del cachedTracks[(frozenset(artists),title)]
+#		t = Thread(target=getTrackImage,args=(artists,title,))
+#		t.start()
+#	return res
 
-def cache_track(artists,title,result):
+#def artist_from_cache(artist):
-	cachedTracks[(frozenset(artists),title)] = result
+#	try:
-	day = datetime.date.today().toordinal()
+#		res = cachedArtists[artist]
-	cachedTracksDays[(frozenset(artists),title)] = day
+#	except:
-def cache_artist(artist,result):
+#		# no entry there, let the calling function know
-	if result is None: log("Caching None for " + artist,module="debug")
+#		raise KeyError()
-	cachedArtists[artist] = result
+#
-	day = datetime.date.today().toordinal()
+#	if res is None:
-	cachedArtistsDays[artist] = day
+#		retain = settings.get_settings("CACHE_EXPIRE_NEGATIVE")
+#	else:
+#		retain = settings.get_settings("CACHE_EXPIRE_POSITIVE")
+#
+#	# if the settings say caches never expire, just return
+#	if retain is None: return res
+#
+#	# look if entry is too old
+#	nowday = datetime.date.today().toordinal()
+#	cacheday = cachedArtistsDays[artist]
+#
+#	if (nowday - cacheday) > retain:
+#		# fetch the new image in the background, but still return the old one for one last time
+#		log("Expired cache for " + artist)
+#		del cachedArtists[artist]
+#		t = Thread(target=getArtistImage,args=(artist,))
+#		t.start()
+#	return res
+#
 
-def track_from_cache(artists,title):
+#def saveCache():
-	try:
+#	fl = open("images/cache","wb")
-		res = cachedTracks[(frozenset(artists),title)]
+#	stream = pickle.dumps({"tracks":cachedTracks,"artists":cachedArtists,"tracks_days":cachedTracksDays,"artists_days":cachedArtistsDays})
-	except:
+#	fl.write(stream)
-		# no entry there, let the calling function know
+#	fl.close()
-		raise KeyError()
 
-	if res is None:
+#	pass
-		retain = settings.get_settings("CACHE_EXPIRE_NEGATIVE")
+#	persistence.save(artist_cache,"artist_cache")
-	else:
+#	persistence.save(track_cache,"track_cache")
-		retain = settings.get_settings("CACHE_EXPIRE_POSITIVE")
 
-	# if the settings say caches never expire, just return
+#def loadCache():
-	if retain is None: return res
+#	global artist_cache, track_cache
+#	artist_cache_tmp = persistence.load("artist_cache")
+#	track_cache_tmp = persistence.load("track_cache")
+#	if artist_cache_tmp is not None: artist_cache = artist_cache_tmp
+#	if track_cache_tmp is not None: track_cache = track_cache_tmp
+#	pass
 
-	# look if entry is too old
+#	try:
-	nowday = datetime.date.today().toordinal()
+#		fl = open("images/cache","rb")
-	cacheday = cachedTracksDays[(frozenset(artists),title)]
+#	except:
-
+#		return
-	if (nowday - cacheday) > retain:
-		# fetch the new image in the background, but still return the old one for one last time
-		log("Expired cache for " + "/".join(artists) + " - " + title)
-		del cachedTracks[(frozenset(artists),title)]
-		t = Thread(target=getTrackImage,args=(artists,title,))
-		t.start()
-	return res
-
-def artist_from_cache(artist):
-	try:
-		res = cachedArtists[artist]
-	except:
-		# no entry there, let the calling function know
-		raise KeyError()
-
-	if res is None:
-		retain = settings.get_settings("CACHE_EXPIRE_NEGATIVE")
-	else:
-		retain = settings.get_settings("CACHE_EXPIRE_POSITIVE")
-
-	# if the settings say caches never expire, just return
-	if retain is None: return res
-
-	# look if entry is too old
-	nowday = datetime.date.today().toordinal()
-	cacheday = cachedArtistsDays[artist]
-
-	if (nowday - cacheday) > retain:
-		# fetch the new image in the background, but still return the old one for one last time
-		log("Expired cache for " + artist)
-		del cachedArtists[artist]
-		t = Thread(target=getArtistImage,args=(artist,))
-		t.start()
-	return res
-
-
-def saveCache():
-	fl = open("images/cache","wb")
-	stream = pickle.dumps({"tracks":cachedTracks,"artists":cachedArtists,"tracks_days":cachedTracksDays,"artists_days":cachedArtistsDays})
-	fl.write(stream)
-	fl.close()
-
-def loadCache():
-	try:
-		fl = open("images/cache","rb")
-	except:
-		return
-
-	try:
-		ob = pickle.loads(fl.read())
-		global cachedTracks, cachedArtists, cachedTracksDays, cachedArtistsDays
-		cachedTracks, cachedArtists, cachedTracksDays, cachedArtistsDays = ob["tracks"],ob["artists"],ob["tracks_days"],ob["artists_days"]
-		#(cachedTracks, cachedArtists) = ob
-	finally:
-		fl.close()
 
+#	try:
+#		ob = pickle.loads(fl.read())
+#		global cachedTracks, cachedArtists, cachedTracksDays, cachedArtistsDays
+#		cachedTracks, cachedArtists, cachedTracksDays, cachedArtistsDays = ob["tracks"],ob["artists"],ob["tracks_days"],ob["artists_days"]
+#		#(cachedTracks, cachedArtists) = ob
+#	finally:
+#		fl.close()
+#
 	# remove corrupt caching from previous versions
-	toremove = []
+#	toremove = []
-	for k in cachedTracks:
+#	for k in cachedTracks:
-		if cachedTracks[k] == "":
+#		if cachedTracks[k] == "":
-			toremove.append(k)
+#			toremove.append(k)
-	for k in toremove:
+#	for k in toremove:
-		del cachedTracks[k]
+#		del cachedTracks[k]
-		log("Removed invalid cache key: " + str(k))
+#		log("Removed invalid cache key: " + str(k))
 
-	toremove = []
+#	toremove = []
-	for k in cachedArtists:
+#	for k in cachedArtists:
-		if cachedArtists[k] == "":
+#		if cachedArtists[k] == "":
-			toremove.append(k)
+#			toremove.append(k)
-	for k in toremove:
+#	for k in toremove:
-		del cachedArtists[k]
+#		del cachedArtists[k]
-		log("Removed invalid cache key: " + str(k))
+#		log("Removed invalid cache key: " + str(k))
 
 def getTrackImage(artists,title,fast=False):
 
@@ -398,7 +423,7 @@ def getTrackImage(artists,title,fast=False):
 		# if we have cached the nonexistence of that image, we immediately return the redirect to the artist and let the resolver handle it
 		# (even if we're not in a fast lookup right now)
 		#result = cachedTracks[(frozenset(artists),title)]
-		result = track_from_cache(artists,title)
+		result = track_cache.get((frozenset(artists),title)) #track_from_cache(artists,title)
 		if result is not None: return result
 		else:
 			for a in artists:
@@ -421,7 +446,7 @@ def getTrackImage(artists,title,fast=False):
 
 	# cache results (even negative ones)
 	#cachedTracks[(frozenset(artists),title)] = result
-	cache_track(artists,title,result)
+	track_cache.add((frozenset(artists),title),result) #cache_track(artists,title,result)
 
 	# return either result or redirect to artist
 	if result is not None: return result
@@ -458,7 +483,7 @@ def getArtistImage(artist,fast=False):
 
 	try:
 		#result = cachedArtists[artist]
-		result = artist_from_cache(artist)
+		result = artist_cache.get(artist) #artist_from_cache(artist)
 		if result is not None: return result
 		else: return ""
 	except:
@@ -480,7 +505,7 @@ def getArtistImage(artist,fast=False):
 
 	# cache results (even negative ones)
 	#cachedArtists[artist] = result
-	cache_artist(artist,result)
+	artist_cache.add(artist,result) #cache_artist(artist,result)
 
 	if result is not None: return result
 	else: return ""