Replaced directory changing with fully generated file paths

maloja/__init__.py

@@ -33,23 +33,4 @@ commands = {
 	"maloja":"controller:main"
 }
 
-### DOREAH CONFIGURATION
+from . import globalconf
-
-from doreah import config
-config(
-	logging={
-		"logfolder": "logs"
-	},
-	settings={
-		"files":[
-			"settings/default.ini",
-			"settings/settings.ini"
-		]
-	},
-	caching={
-		"folder": "cache/"
-	},
-	regular={
-		"autostart": False
-	}
-)

maloja/backup.py

@@ -2,6 +2,7 @@ import tarfile
 from datetime import datetime
 import glob
 import os
+from globalconf import datadir
 
 
 user_files = {
@@ -22,7 +23,7 @@ def backup(folder,level="full"):
 	selected_files = user_files["minimal"] if level == "minimal" else user_files["minimal"] + user_files["full"]
 	real_files = []
 	for g in selected_files:
-		real_files += glob.glob(g)
+		real_files += glob.glob(datadir(g))
 
 	now = datetime.utcnow()
 	timestr = now.strftime("%Y_%m_%d_%H_%M_%S")

maloja/cleanup.py

@@ -1,6 +1,8 @@
 import re
 from . import utilities
 from doreah import tsv, settings
+from .globalconf import datadir
+import pkg_resources
 
 # need to do this as a class so it can retain loaded settings from file
 # apparently this is not true
@@ -11,7 +13,7 @@ class CleanerAgent:
 		self.updateRules()
 
 	def updateRules(self):
-		raw = tsv.parse_all("rules","string","string","string","string")
+		raw = tsv.parse_all(datadir("rules"),"string","string","string","string")
 		self.rules_belongtogether = [b for [a,b,c,d] in raw if a=="belongtogether"]
 		self.rules_notanartist = [b for [a,b,c,d] in raw if a=="notanartist"]
 		self.rules_replacetitle = {b.lower():c for [a,b,c,d] in raw if a=="replacetitle"}
@@ -160,7 +162,7 @@ class CollectorAgent:
 	# rules_include			dict: credited artist -> all real artists
 
 	def updateRules(self):
-		raw = tsv.parse_all("rules","string","string","string")
+		raw = tsv.parse_all(datadir("rules"),"string","string","string")
 		self.rules_countas = {b:c for [a,b,c] in raw if a=="countas"}
 		self.rules_countas_id = {}
 		self.rules_include = {} #Twice the memory, double the performance!

maloja/controller.py

@@ -12,18 +12,17 @@ import pkg_resources
 from doreah.control import mainfunction
 from doreah.io import col
 
-from .globalconf import DATA_DIR
+from .globalconf import datadir
 from .backup import backup
 
 
 
 origpath = os.getcwd()
-os.chdir(DATA_DIR)
 
 def copy_initial_local_files():
 	folder = pkg_resources.resource_filename(__name__,"data_files")
 	#shutil.copy(folder,DATA_DIR)
-	dir_util.copy_tree(folder,DATA_DIR,update=False)
+	dir_util.copy_tree(folder,datadir(),update=False)
 
 
 def setup():
@@ -49,13 +48,13 @@ def setup():
 			print("\t" + "Please enter your " + apikeys[k] + ". If you do not want to use one at this moment, simply leave this empty and press Enter.")
 			key = input()
 			if key == "": key = None
-			settings.update_settings("settings/settings.ini",{k:key},create_new=True)
+			settings.update_settings(datadir("settings/settings.ini"),{k:key},create_new=True)
 		else:
 			print("\t" + apikeys[k] + " found.")
 
 
 	# OWN API KEY
-	if os.path.exists("./clients/authenticated_machines.tsv"):
+	if os.path.exists(datadir("clients/authenticated_machines.tsv")):
 		pass
 	else:
 		print("Do you want to set up a key to enable scrobbling? Your scrobble extension needs that key so that only you can scrobble tracks to your database. [Y/n]")
@@ -66,7 +65,7 @@ def setup():
 			for i in range(64):
 				key += str(random.choice(list(range(10)) + list("abcdefghijklmnopqrstuvwxyz") + list("ABCDEFGHIJKLMNOPQRSTUVWXYZ")))
 			print("Your API Key: " + col["yellow"](key))
-			with open("./clients/authenticated_machines.tsv","w") as keyfile:
+			with open(datadir("clients/authenticated_machines.tsv"),"w") as keyfile:
 				keyfile.write(key + "\t" + "Default Generated Key")
 		elif answer.lower() in ["n","no","nay","0","negative","false"]:
 			pass
@@ -76,16 +75,15 @@ def setup():
 		print("Please enter your name. This will be displayed e.g. when comparing your charts to another user. Leave this empty if you would not like to specify a name right now.")
 		name = input()
 		if name == "": name = "Generic Maloja User"
-		settings.update_settings("settings/settings.ini",{"NAME":name},create_new=True)
+		settings.update_settings(datadir("settings/settings.ini"),{"NAME":name},create_new=True)
 
 	if settings.get_settings("SEND_STATS") is None:
 		print("I would like to know how many people use Maloja. Would it be okay to send a daily ping to my server (this contains no data that isn't accessible via your web interface already)? [Y/n]")
 		answer = input()
 		if answer.lower() in ["y","yes","yea","1","positive","true",""]:
-			settings.update_settings("settings/settings.ini",{"SEND_STATS":True},create_new=True)
+			settings.update_settings(datadir("settings/settings.ini"),{"SEND_STATS":True,"PUBLIC_URL":None},create_new=True)
-			settings.update_settings("settings/settings.ini",{"PUBLIC_URL":None},create_new=True)
 		else:
-			settings.update_settings("settings/settings.ini",{"SEND_STATS":False},create_new=True)
+			settings.update_settings(datadir("settings/settings.ini"),{"SEND_STATS":False},create_new=True)
 
 
 def getInstance():
@@ -107,7 +105,7 @@ def getInstanceSupervisor():
 def start():
 	setup()
 	try:
-		with open("logs/stderr.log","w") as logf:
+		with open(datadir("logs/stderr.log"),"w") as logf:
 			p = subprocess.Popen(["python3","-m","maloja.server"],stdout=subprocess.DEVNULL,stderr=logf,cwd=DATA_DIR)
 			sp = subprocess.Popen(["python3","-m","maloja.supervisor"],stdout=subprocess.DEVNULL,stderr=logf,cwd=DATA_DIR)
 		print(col["green"]("Maloja started!") + " PID: " + str(p.pid))
@@ -152,14 +150,14 @@ def loadlastfm(filename):
 		print("Please specify a file!")
 		return
 
-	if os.path.exists("./scrobbles/lastfmimport.tsv"):
+	if os.path.exists(datadir("scrobbles/lastfmimport.tsv")):
 		print("Already imported Last.FM data. Overwrite? [y/N]")
 		if input().lower() in ["y","yes","yea","1","positive","true"]:
 			pass
 		else:
 			return
 	print("Please wait...")
-	os.system("python3 -m maloja.lastfmconverter " + filename + " ./scrobbles/lastfmimport.tsv")
+	os.system("python3 -m maloja.lastfmconverter " + filename + " " + datadir("scrobbles/lastfmimport.tsv"))
 	print("Successfully imported your Last.FM scrobbles!")
 
 def direct():

maloja/database.py

@@ -8,6 +8,7 @@ from .urihandler import uri_to_internal, internal_to_uri, compose_querystring
 from . import compliant_api
 from .external import proxy_scrobble
 from .__init__ import version
+from .globalconf import datadir
 # doreah toolkit
 from doreah.logging import log
 from doreah import tsv
@@ -68,7 +69,7 @@ lastsync = 0
 db_rulestate = False
 
 try:
-	with open("known_servers.yml","r") as f:
+	with open(datadir("known_servers.yml"),"r") as f:
 		KNOWN_SERVERS = set(yaml.safe_load(f))
 except:
 	KNOWN_SERVERS = set()
@@ -76,7 +77,7 @@ except:
 
 def add_known_server(url):
 	KNOWN_SERVERS.add(url)
-	with open("known_servers.yml","w") as f:
+	with open(datadir("known_servers.yml"),"w") as f:
 		f.write(yaml.dump(list(KNOWN_SERVERS)))
 
 
@@ -84,9 +85,9 @@ def add_known_server(url):
 ### symmetric keys are fine for now since we hopefully use HTTPS
 def loadAPIkeys():
 	global clients
-	tsv.create("clients/authenticated_machines.tsv")
+	tsv.create(datadir("clients/authenticated_machines.tsv"))
 	#createTSV("clients/authenticated_machines.tsv")
-	clients = tsv.parse("clients/authenticated_machines.tsv","string","string")
+	clients = tsv.parse(datadir("clients/authenticated_machines.tsv"),"string","string")
 	#clients = parseTSV("clients/authenticated_machines.tsv","string","string")
 	log("Authenticated Machines: " + ", ".join([m[1] for m in clients]))
 
@@ -847,10 +848,10 @@ def import_rulemodule(**keys):
 
 		if remove:
 			log("Deactivating predefined rulefile " + filename)
-			os.remove("rules/" + filename + ".tsv")
+			os.remove(datadir("rules/" + filename + ".tsv"))
 		else:
 			log("Importing predefined rulefile " + filename)
-			os.symlink("predefined/" + filename + ".tsv","rules/" + filename + ".tsv")
+			os.symlink(datadir("predefined/" + filename + ".tsv"),datadir("rules/" + filename + ".tsv"))
 
 
 
@@ -951,7 +952,7 @@ def build_db():
 
 
 	# parse files
-	db = tsv.parse_all("scrobbles","int","string","string",comments=False)
+	db = tsv.parse_all(datadir("scrobbles"),"int","string","string",comments=False)
 	#db = parseAllTSV("scrobbles","int","string","string",escape=False)
 	for sc in db:
 		artists = sc[1].split("␟")
@@ -985,7 +986,7 @@ def build_db():
 	utilities.send_stats()
 
 	global db_rulestate
-	db_rulestate = utilities.consistentRulestate("scrobbles",cla.checksums)
+	db_rulestate = utilities.consistentRulestate(datadir("scrobbles"),cla.checksums)
 
 	log("Database fully built!")
 
@@ -1019,9 +1020,9 @@ def sync():
 	#log("Sorted into months",module="debug")
 
 	for e in entries:
-		tsv.add_entries("scrobbles/" + e + ".tsv",entries[e],comments=False)
+		tsv.add_entries(datadir("scrobbles/" + e + ".tsv"),entries[e],comments=False)
 		#addEntries("scrobbles/" + e + ".tsv",entries[e],escape=False)
-		utilities.combineChecksums("scrobbles/" + e + ".tsv",cla.checksums)
+		utilities.combineChecksums(datadir("scrobbles/" + e + ".tsv"),cla.checksums)
 
 	#log("Written files",module="debug")
 
@@ -1046,7 +1047,7 @@ import copy
 
 cache_query = {}
 if doreah.version >= (0,7,1) and settings.get_settings("EXPERIMENTAL_FEATURES"):
-	cache_query_permanent = DiskDict(name="dbquery",folder="cache",maxmemory=1024*1024*500,maxstorage=1024*1024*settings.get_settings("DB_CACHE_SIZE"))
+	cache_query_permanent = DiskDict(name="dbquery",folder=datadir("cache"),maxmemory=1024*1024*500,maxstorage=1024*1024*settings.get_settings("DB_CACHE_SIZE"))
 else:
 	cache_query_permanent = Cache(maxmemory=1024*1024*500)
 cacheday = (0,0,0)

maloja/fixexisting.py

@@ -1,7 +1,5 @@
 import os
-from .globalconf import DATA_DIR
+from .globalconf import datadir
-os.chdir(DATA_DIR)
-
 import re
 from .cleanup import CleanerAgent
 from doreah.logging import log
@@ -13,29 +11,28 @@ wendigo = CleanerAgent()
 
 exp = r"([0-9]*)(\t+)([^\t]+?)(\t+)([^\t]+)(\t*)([^\t]*)\n"
 
-pthj = os.path.join
 
 
 def fix():
 
-	backup(level="minimal",folder=pthj(DATA_DIR,"backups"))
+	backup(level="minimal",folder=datadir("backups"))
 
 	now = datetime.datetime.utcnow()
 	nowstr = now.strftime("%Y_%m_%d_%H_%M_%S")
 	datestr = now.strftime("%Y/%m/%d")
 	timestr = now.strftime("%H:%M:%S")
 
-	with open(pthj(DATA_DIR,"logs","dbfix",nowstr + ".log"),"a") as logfile:
+	with open(datadir("logs","dbfix",nowstr + ".log"),"a") as logfile:
 
 		logfile.write("Database fix initiated on " + datestr + " " + timestr + " UTC")
 		logfile.write("\n\n")
 
-		for filename in os.listdir(pthj(DATA_DIR,"scrobbles")):
+		for filename in os.listdir(datadir("scrobbles")):
 			if filename.endswith(".tsv"):
 				filename_new = filename + "_new"
 
-				with open(pthj(DATA_DIR,"scrobbles",filename_new),"w") as newfile:
+				with open(datadir("scrobbles",filename_new),"w") as newfile:
-					with open(pthj(DATA_DIR,"scrobbles",filename),"r") as oldfile:
+					with open(datadir("scrobbles",filename),"r") as oldfile:
 
 						for l in oldfile:
 
@@ -50,8 +47,8 @@ def fix():
 
 
 				#os.system("diff " + "scrobbles/" + fn + "_new" + " " + "scrobbles/" + fn)
-				with open(pthj(DATA_DIR,"scrobbles",filename_new),"r") as newfile:
+				with open(datadir("scrobbles",filename_new),"r") as newfile:
-					with open(pthj(DATA_DIR,"scrobbles",filename),"r") as oldfile:
+					with open(datadir("scrobbles",filename),"r") as oldfile:
 
 						diff = difflib.unified_diff(oldfile.read().split("\n"),newfile.read().split("\n"),lineterm="")
 						diff = list(diff)[2:]
@@ -61,7 +58,7 @@ def fix():
 						logfile.write(output)
 						logfile.write("\n")
 
-				os.rename(pthj(DATA_DIR,"scrobbles",filename_new),pthj(DATA_DIR,"scrobbles",filename))
+				os.rename(datadir("scrobbles",filename_new),datadir("scrobbles",filename))
 
-				with open(pthj(DATA_DIR,"scrobbles",filename + ".rulestate"),"w") as checkfile:
+				with open(datadir("scrobbles",filename + ".rulestate"),"w") as checkfile:
 					checkfile.write(wendigo.checksums)

maloja/globalconf.py

@@ -1,6 +1,51 @@
-from doreah.settings import get_settings
 import os
 
+
+# data folder
+# must be determined first because getting settings relies on it
+
+try:
+	DATA_DIR = os.environ["XDG_DATA_HOME"].split(":")[0]
+	assert os.path.exists(DATA_DIR)
+except:
+	DATA_DIR = os.path.join(os.environ["HOME"],".local/share/")
+
+DATA_DIR = os.path.join(DATA_DIR,"maloja")
+os.makedirs(DATA_DIR,exist_ok=True)
+
+def datadir(*args):
+	return os.path.join(DATA_DIR,*args)
+
+
+
+### DOREAH CONFIGURATION
+
+from doreah import config
+
+config(
+	logging={
+		"logfolder": datadir("logs")
+	},
+	settings={
+		"files":[
+			datadir("settings/default.ini"),
+			datadir("settings/settings.ini")
+		]
+	},
+	caching={
+		"folder": datadir("cache")
+	},
+	regular={
+		"autostart": False
+	}
+)
+
+
+
+from doreah.settings import get_settings
+
+# thumbor
+
 THUMBOR_SERVER, THUMBOR_SECRET = get_settings("THUMBOR_SERVER","THUMBOR_SECRET")
 try:
 	USE_THUMBOR = THUMBOR_SERVER is not None and THUMBOR_SECRET is not None
@@ -12,15 +57,3 @@ try:
 except:
 	USE_THUMBOR = False
 	log("Thumbor could not be initialized. Is libthumbor installed?")
-
-
-
-
-try:
-	DATA_DIR = os.environ["XDG_DATA_HOME"].split(":")[0]
-	assert os.path.exists(DATA_DIR)
-except:
-	DATA_DIR = os.path.join(os.environ["HOME"],".local/share/")
-
-DATA_DIR = os.path.join(DATA_DIR,"maloja")
-os.makedirs(DATA_DIR,exist_ok=True)

maloja/server.py

@@ -1,7 +1,6 @@
 #!/usr/bin/env python
 import os
-from .globalconf import DATA_DIR
+from .globalconf import datadir, DATA_DIR
-os.chdir(DATA_DIR)
 
 
 # server stuff
@@ -50,6 +49,7 @@ BaseRequest.MEMFILE_MAX = 15 * 1024 * 1024
 
 WEBFOLDER = pkg_resources.resource_filename(__name__,"web")
 STATICFOLDER = pkg_resources.resource_filename(__name__,"static")
+DATAFOLDER = DATADIR
 
 webserver = Bottle()
 
@@ -122,27 +122,27 @@ def dynamic_image():
 @webserver.route("/images/<pth:re:.*\\.gif>")
 def static_image(pth):
 	if globalconf.USE_THUMBOR:
-		return static_file("images/" + pth,root="")
+		return static_file(pthjoin("images",pth),root=DATAFOLDER)
 
 	type = pth.split(".")[-1]
 	small_pth = pth + "-small"
-	if os.path.exists("images/" + small_pth):
+	if os.path.exists(datadir("images",small_pth)):
-		response = static_file("images/" + small_pth,root="")
+		response = static_file(pthjoin("images",small_pth),root=DATAFOLDER)
 	else:
 		try:
 			from wand.image import Image
-			img = Image(filename="images/" + pth)
+			img = Image(filename=datadir("images",pth))
 			x,y = img.size[0], img.size[1]
 			smaller = min(x,y)
 			if smaller > 300:
 				ratio = 300/smaller
 				img.resize(int(ratio*x),int(ratio*y))
-				img.save(filename="images/" + small_pth)
+				img.save(filename=datadir("images",small_pth))
-				response = static_file("images/" + small_pth,root="")
+				response = static_file(pthjoin("images",small_pth),root=DATAFOLDER)
 			else:
-				response = static_file("images/" + pth,root="")
+				response = static_file(pthjoin("images",pth),root=DATAFOLDER)
 		except:
-			response = static_file("images/" + pth,root="")
+			response = static_file(pthjoin("images",pth),root=DATAFOLDER)
 
 	#response = static_file("images/" + pth,root="")
 	response.set_header("Cache-Control", "public, max-age=86400")

maloja/supervisor.py

@@ -1,7 +1,5 @@
 #!/usr/bin/env python3
 import os
-from .globalconf import DATA_DIR
-os.chdir(DATA_DIR)
 
 import subprocess
 import time

maloja/utilities.py

@@ -17,6 +17,7 @@ from doreah.regular import yearly, daily
 from .external import api_request_track, api_request_artist
 from .__init__ import version
 from . import globalconf
+from .globalconf import datadir
 
 
 
@@ -170,7 +171,7 @@ def get_all_possible_filenames(artist=None,artists=None,title=None):
 	else: return []
 
 
-	superfolder = "images/tracks/" if track else "images/artists/"
+	superfolder = datadir("images/tracks/") if track else datadir("images/artists/")
 
 	filenames = []
 

maloja/web/setup.py

@@ -1,4 +1,5 @@
 import os
+from ..globalconf import datadir
 
 def instructions(keys):
 
@@ -8,7 +9,7 @@ def instructions(keys):
 
 
     validchars = "-_abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
-    for f in os.listdir("rules/predefined"):
+    for f in os.listdir(datadir("rules/predefined")):
         if f.endswith(".tsv"):
 
             rawf = f.replace(".tsv","")
@@ -22,7 +23,7 @@ def instructions(keys):
             if not "_" in rawf: continue
 
             try:
-                with open("rules/predefined/" + f) as tsvfile:
+                with open(datadir("rules/predefined",f)) as tsvfile:
                     line1 = tsvfile.readline()
                     line2 = tsvfile.readline()
 
@@ -39,7 +40,7 @@ def instructions(keys):
 
             html += "<tr>"
 
-            if os.path.exists("rules/" + f):
+            if os.path.exists(datadir("rules",f)):
                 html += "<td class='interaction' onclick=deactivateRuleModule(this,'" + rawf + "')><a class='textlink'>Remove:</a></td>"
             else:
                 html += "<td class='interaction' onclick=activateRuleModule(this,'" + rawf + "')><a class='textlink'>Add:</a></td>"