From dbc23ca73ce866440737fa59752a324337118dae Mon Sep 17 00:00:00 2001 From: Krateng Date: Sun, 15 Dec 2019 15:18:33 +0100 Subject: [PATCH] Replaced directory changing with fully generated file paths --- maloja/__init__.py | 21 +-------------- maloja/backup.py | 3 ++- maloja/cleanup.py | 6 +++-- maloja/controller.py | 24 ++++++++---------- maloja/database.py | 23 +++++++++-------- maloja/fixexisting.py | 23 ++++++++--------- maloja/globalconf.py | 59 +++++++++++++++++++++++++++++++++---------- maloja/server.py | 20 +++++++-------- maloja/supervisor.py | 2 -- maloja/utilities.py | 3 ++- maloja/web/setup.py | 7 ++--- 11 files changed, 102 insertions(+), 89 deletions(-) diff --git a/maloja/__init__.py b/maloja/__init__.py index 6829b7f..758a375 100644 --- a/maloja/__init__.py +++ b/maloja/__init__.py @@ -33,23 +33,4 @@ commands = { "maloja":"controller:main" } -### DOREAH CONFIGURATION - -from doreah import config -config( - logging={ - "logfolder": "logs" - }, - settings={ - "files":[ - "settings/default.ini", - "settings/settings.ini" - ] - }, - caching={ - "folder": "cache/" - }, - regular={ - "autostart": False - } -) +from . import globalconf diff --git a/maloja/backup.py b/maloja/backup.py index 93ff2b7..b08bfe0 100644 --- a/maloja/backup.py +++ b/maloja/backup.py @@ -2,6 +2,7 @@ import tarfile from datetime import datetime import glob import os +from globalconf import datadir user_files = { @@ -22,7 +23,7 @@ def backup(folder,level="full"): selected_files = user_files["minimal"] if level == "minimal" else user_files["minimal"] + user_files["full"] real_files = [] for g in selected_files: - real_files += glob.glob(g) + real_files += glob.glob(datadir(g)) now = datetime.utcnow() timestr = now.strftime("%Y_%m_%d_%H_%M_%S") diff --git a/maloja/cleanup.py b/maloja/cleanup.py index 3bc7cf3..d9a427f 100644 --- a/maloja/cleanup.py +++ b/maloja/cleanup.py @@ -1,6 +1,8 @@ import re from . import utilities from doreah import tsv, settings +from .globalconf import datadir +import pkg_resources # need to do this as a class so it can retain loaded settings from file # apparently this is not true @@ -11,7 +13,7 @@ class CleanerAgent: self.updateRules() def updateRules(self): - raw = tsv.parse_all("rules","string","string","string","string") + raw = tsv.parse_all(datadir("rules"),"string","string","string","string") self.rules_belongtogether = [b for [a,b,c,d] in raw if a=="belongtogether"] self.rules_notanartist = [b for [a,b,c,d] in raw if a=="notanartist"] self.rules_replacetitle = {b.lower():c for [a,b,c,d] in raw if a=="replacetitle"} @@ -160,7 +162,7 @@ class CollectorAgent: # rules_include dict: credited artist -> all real artists def updateRules(self): - raw = tsv.parse_all("rules","string","string","string") + raw = tsv.parse_all(datadir("rules"),"string","string","string") self.rules_countas = {b:c for [a,b,c] in raw if a=="countas"} self.rules_countas_id = {} self.rules_include = {} #Twice the memory, double the performance! diff --git a/maloja/controller.py b/maloja/controller.py index 98b8f82..da88d6b 100755 --- a/maloja/controller.py +++ b/maloja/controller.py @@ -12,18 +12,17 @@ import pkg_resources from doreah.control import mainfunction from doreah.io import col -from .globalconf import DATA_DIR +from .globalconf import datadir from .backup import backup origpath = os.getcwd() -os.chdir(DATA_DIR) def copy_initial_local_files(): folder = pkg_resources.resource_filename(__name__,"data_files") #shutil.copy(folder,DATA_DIR) - dir_util.copy_tree(folder,DATA_DIR,update=False) + dir_util.copy_tree(folder,datadir(),update=False) def setup(): @@ -49,13 +48,13 @@ def setup(): print("\t" + "Please enter your " + apikeys[k] + ". If you do not want to use one at this moment, simply leave this empty and press Enter.") key = input() if key == "": key = None - settings.update_settings("settings/settings.ini",{k:key},create_new=True) + settings.update_settings(datadir("settings/settings.ini"),{k:key},create_new=True) else: print("\t" + apikeys[k] + " found.") # OWN API KEY - if os.path.exists("./clients/authenticated_machines.tsv"): + if os.path.exists(datadir("clients/authenticated_machines.tsv")): pass else: print("Do you want to set up a key to enable scrobbling? Your scrobble extension needs that key so that only you can scrobble tracks to your database. [Y/n]") @@ -66,7 +65,7 @@ def setup(): for i in range(64): key += str(random.choice(list(range(10)) + list("abcdefghijklmnopqrstuvwxyz") + list("ABCDEFGHIJKLMNOPQRSTUVWXYZ"))) print("Your API Key: " + col["yellow"](key)) - with open("./clients/authenticated_machines.tsv","w") as keyfile: + with open(datadir("clients/authenticated_machines.tsv"),"w") as keyfile: keyfile.write(key + "\t" + "Default Generated Key") elif answer.lower() in ["n","no","nay","0","negative","false"]: pass @@ -76,16 +75,15 @@ def setup(): print("Please enter your name. This will be displayed e.g. when comparing your charts to another user. Leave this empty if you would not like to specify a name right now.") name = input() if name == "": name = "Generic Maloja User" - settings.update_settings("settings/settings.ini",{"NAME":name},create_new=True) + settings.update_settings(datadir("settings/settings.ini"),{"NAME":name},create_new=True) if settings.get_settings("SEND_STATS") is None: print("I would like to know how many people use Maloja. Would it be okay to send a daily ping to my server (this contains no data that isn't accessible via your web interface already)? [Y/n]") answer = input() if answer.lower() in ["y","yes","yea","1","positive","true",""]: - settings.update_settings("settings/settings.ini",{"SEND_STATS":True},create_new=True) - settings.update_settings("settings/settings.ini",{"PUBLIC_URL":None},create_new=True) + settings.update_settings(datadir("settings/settings.ini"),{"SEND_STATS":True,"PUBLIC_URL":None},create_new=True) else: - settings.update_settings("settings/settings.ini",{"SEND_STATS":False},create_new=True) + settings.update_settings(datadir("settings/settings.ini"),{"SEND_STATS":False},create_new=True) def getInstance(): @@ -107,7 +105,7 @@ def getInstanceSupervisor(): def start(): setup() try: - with open("logs/stderr.log","w") as logf: + with open(datadir("logs/stderr.log"),"w") as logf: p = subprocess.Popen(["python3","-m","maloja.server"],stdout=subprocess.DEVNULL,stderr=logf,cwd=DATA_DIR) sp = subprocess.Popen(["python3","-m","maloja.supervisor"],stdout=subprocess.DEVNULL,stderr=logf,cwd=DATA_DIR) print(col["green"]("Maloja started!") + " PID: " + str(p.pid)) @@ -152,14 +150,14 @@ def loadlastfm(filename): print("Please specify a file!") return - if os.path.exists("./scrobbles/lastfmimport.tsv"): + if os.path.exists(datadir("scrobbles/lastfmimport.tsv")): print("Already imported Last.FM data. Overwrite? [y/N]") if input().lower() in ["y","yes","yea","1","positive","true"]: pass else: return print("Please wait...") - os.system("python3 -m maloja.lastfmconverter " + filename + " ./scrobbles/lastfmimport.tsv") + os.system("python3 -m maloja.lastfmconverter " + filename + " " + datadir("scrobbles/lastfmimport.tsv")) print("Successfully imported your Last.FM scrobbles!") def direct(): diff --git a/maloja/database.py b/maloja/database.py index 99682e0..0448cde 100644 --- a/maloja/database.py +++ b/maloja/database.py @@ -8,6 +8,7 @@ from .urihandler import uri_to_internal, internal_to_uri, compose_querystring from . import compliant_api from .external import proxy_scrobble from .__init__ import version +from .globalconf import datadir # doreah toolkit from doreah.logging import log from doreah import tsv @@ -68,7 +69,7 @@ lastsync = 0 db_rulestate = False try: - with open("known_servers.yml","r") as f: + with open(datadir("known_servers.yml"),"r") as f: KNOWN_SERVERS = set(yaml.safe_load(f)) except: KNOWN_SERVERS = set() @@ -76,7 +77,7 @@ except: def add_known_server(url): KNOWN_SERVERS.add(url) - with open("known_servers.yml","w") as f: + with open(datadir("known_servers.yml"),"w") as f: f.write(yaml.dump(list(KNOWN_SERVERS))) @@ -84,9 +85,9 @@ def add_known_server(url): ### symmetric keys are fine for now since we hopefully use HTTPS def loadAPIkeys(): global clients - tsv.create("clients/authenticated_machines.tsv") + tsv.create(datadir("clients/authenticated_machines.tsv")) #createTSV("clients/authenticated_machines.tsv") - clients = tsv.parse("clients/authenticated_machines.tsv","string","string") + clients = tsv.parse(datadir("clients/authenticated_machines.tsv"),"string","string") #clients = parseTSV("clients/authenticated_machines.tsv","string","string") log("Authenticated Machines: " + ", ".join([m[1] for m in clients])) @@ -847,10 +848,10 @@ def import_rulemodule(**keys): if remove: log("Deactivating predefined rulefile " + filename) - os.remove("rules/" + filename + ".tsv") + os.remove(datadir("rules/" + filename + ".tsv")) else: log("Importing predefined rulefile " + filename) - os.symlink("predefined/" + filename + ".tsv","rules/" + filename + ".tsv") + os.symlink(datadir("predefined/" + filename + ".tsv"),datadir("rules/" + filename + ".tsv")) @@ -951,7 +952,7 @@ def build_db(): # parse files - db = tsv.parse_all("scrobbles","int","string","string",comments=False) + db = tsv.parse_all(datadir("scrobbles"),"int","string","string",comments=False) #db = parseAllTSV("scrobbles","int","string","string",escape=False) for sc in db: artists = sc[1].split("␟") @@ -985,7 +986,7 @@ def build_db(): utilities.send_stats() global db_rulestate - db_rulestate = utilities.consistentRulestate("scrobbles",cla.checksums) + db_rulestate = utilities.consistentRulestate(datadir("scrobbles"),cla.checksums) log("Database fully built!") @@ -1019,9 +1020,9 @@ def sync(): #log("Sorted into months",module="debug") for e in entries: - tsv.add_entries("scrobbles/" + e + ".tsv",entries[e],comments=False) + tsv.add_entries(datadir("scrobbles/" + e + ".tsv"),entries[e],comments=False) #addEntries("scrobbles/" + e + ".tsv",entries[e],escape=False) - utilities.combineChecksums("scrobbles/" + e + ".tsv",cla.checksums) + utilities.combineChecksums(datadir("scrobbles/" + e + ".tsv"),cla.checksums) #log("Written files",module="debug") @@ -1046,7 +1047,7 @@ import copy cache_query = {} if doreah.version >= (0,7,1) and settings.get_settings("EXPERIMENTAL_FEATURES"): - cache_query_permanent = DiskDict(name="dbquery",folder="cache",maxmemory=1024*1024*500,maxstorage=1024*1024*settings.get_settings("DB_CACHE_SIZE")) + cache_query_permanent = DiskDict(name="dbquery",folder=datadir("cache"),maxmemory=1024*1024*500,maxstorage=1024*1024*settings.get_settings("DB_CACHE_SIZE")) else: cache_query_permanent = Cache(maxmemory=1024*1024*500) cacheday = (0,0,0) diff --git a/maloja/fixexisting.py b/maloja/fixexisting.py index f376c08..2dba2b6 100644 --- a/maloja/fixexisting.py +++ b/maloja/fixexisting.py @@ -1,7 +1,5 @@ import os -from .globalconf import DATA_DIR -os.chdir(DATA_DIR) - +from .globalconf import datadir import re from .cleanup import CleanerAgent from doreah.logging import log @@ -13,29 +11,28 @@ wendigo = CleanerAgent() exp = r"([0-9]*)(\t+)([^\t]+?)(\t+)([^\t]+)(\t*)([^\t]*)\n" -pthj = os.path.join def fix(): - backup(level="minimal",folder=pthj(DATA_DIR,"backups")) + backup(level="minimal",folder=datadir("backups")) now = datetime.datetime.utcnow() nowstr = now.strftime("%Y_%m_%d_%H_%M_%S") datestr = now.strftime("%Y/%m/%d") timestr = now.strftime("%H:%M:%S") - with open(pthj(DATA_DIR,"logs","dbfix",nowstr + ".log"),"a") as logfile: + with open(datadir("logs","dbfix",nowstr + ".log"),"a") as logfile: logfile.write("Database fix initiated on " + datestr + " " + timestr + " UTC") logfile.write("\n\n") - for filename in os.listdir(pthj(DATA_DIR,"scrobbles")): + for filename in os.listdir(datadir("scrobbles")): if filename.endswith(".tsv"): filename_new = filename + "_new" - with open(pthj(DATA_DIR,"scrobbles",filename_new),"w") as newfile: - with open(pthj(DATA_DIR,"scrobbles",filename),"r") as oldfile: + with open(datadir("scrobbles",filename_new),"w") as newfile: + with open(datadir("scrobbles",filename),"r") as oldfile: for l in oldfile: @@ -50,8 +47,8 @@ def fix(): #os.system("diff " + "scrobbles/" + fn + "_new" + " " + "scrobbles/" + fn) - with open(pthj(DATA_DIR,"scrobbles",filename_new),"r") as newfile: - with open(pthj(DATA_DIR,"scrobbles",filename),"r") as oldfile: + with open(datadir("scrobbles",filename_new),"r") as newfile: + with open(datadir("scrobbles",filename),"r") as oldfile: diff = difflib.unified_diff(oldfile.read().split("\n"),newfile.read().split("\n"),lineterm="") diff = list(diff)[2:] @@ -61,7 +58,7 @@ def fix(): logfile.write(output) logfile.write("\n") - os.rename(pthj(DATA_DIR,"scrobbles",filename_new),pthj(DATA_DIR,"scrobbles",filename)) + os.rename(datadir("scrobbles",filename_new),datadir("scrobbles",filename)) - with open(pthj(DATA_DIR,"scrobbles",filename + ".rulestate"),"w") as checkfile: + with open(datadir("scrobbles",filename + ".rulestate"),"w") as checkfile: checkfile.write(wendigo.checksums) diff --git a/maloja/globalconf.py b/maloja/globalconf.py index 1f2965b..8281f62 100644 --- a/maloja/globalconf.py +++ b/maloja/globalconf.py @@ -1,6 +1,51 @@ -from doreah.settings import get_settings import os + +# data folder +# must be determined first because getting settings relies on it + +try: + DATA_DIR = os.environ["XDG_DATA_HOME"].split(":")[0] + assert os.path.exists(DATA_DIR) +except: + DATA_DIR = os.path.join(os.environ["HOME"],".local/share/") + +DATA_DIR = os.path.join(DATA_DIR,"maloja") +os.makedirs(DATA_DIR,exist_ok=True) + +def datadir(*args): + return os.path.join(DATA_DIR,*args) + + + +### DOREAH CONFIGURATION + +from doreah import config + +config( + logging={ + "logfolder": datadir("logs") + }, + settings={ + "files":[ + datadir("settings/default.ini"), + datadir("settings/settings.ini") + ] + }, + caching={ + "folder": datadir("cache") + }, + regular={ + "autostart": False + } +) + + + +from doreah.settings import get_settings + +# thumbor + THUMBOR_SERVER, THUMBOR_SECRET = get_settings("THUMBOR_SERVER","THUMBOR_SECRET") try: USE_THUMBOR = THUMBOR_SERVER is not None and THUMBOR_SECRET is not None @@ -12,15 +57,3 @@ try: except: USE_THUMBOR = False log("Thumbor could not be initialized. Is libthumbor installed?") - - - - -try: - DATA_DIR = os.environ["XDG_DATA_HOME"].split(":")[0] - assert os.path.exists(DATA_DIR) -except: - DATA_DIR = os.path.join(os.environ["HOME"],".local/share/") - -DATA_DIR = os.path.join(DATA_DIR,"maloja") -os.makedirs(DATA_DIR,exist_ok=True) diff --git a/maloja/server.py b/maloja/server.py index fe281a4..fd634ef 100755 --- a/maloja/server.py +++ b/maloja/server.py @@ -1,7 +1,6 @@ #!/usr/bin/env python import os -from .globalconf import DATA_DIR -os.chdir(DATA_DIR) +from .globalconf import datadir, DATA_DIR # server stuff @@ -50,6 +49,7 @@ BaseRequest.MEMFILE_MAX = 15 * 1024 * 1024 WEBFOLDER = pkg_resources.resource_filename(__name__,"web") STATICFOLDER = pkg_resources.resource_filename(__name__,"static") +DATAFOLDER = DATADIR webserver = Bottle() @@ -122,27 +122,27 @@ def dynamic_image(): @webserver.route("/images/") def static_image(pth): if globalconf.USE_THUMBOR: - return static_file("images/" + pth,root="") + return static_file(pthjoin("images",pth),root=DATAFOLDER) type = pth.split(".")[-1] small_pth = pth + "-small" - if os.path.exists("images/" + small_pth): - response = static_file("images/" + small_pth,root="") + if os.path.exists(datadir("images",small_pth)): + response = static_file(pthjoin("images",small_pth),root=DATAFOLDER) else: try: from wand.image import Image - img = Image(filename="images/" + pth) + img = Image(filename=datadir("images",pth)) x,y = img.size[0], img.size[1] smaller = min(x,y) if smaller > 300: ratio = 300/smaller img.resize(int(ratio*x),int(ratio*y)) - img.save(filename="images/" + small_pth) - response = static_file("images/" + small_pth,root="") + img.save(filename=datadir("images",small_pth)) + response = static_file(pthjoin("images",small_pth),root=DATAFOLDER) else: - response = static_file("images/" + pth,root="") + response = static_file(pthjoin("images",pth),root=DATAFOLDER) except: - response = static_file("images/" + pth,root="") + response = static_file(pthjoin("images",pth),root=DATAFOLDER) #response = static_file("images/" + pth,root="") response.set_header("Cache-Control", "public, max-age=86400") diff --git a/maloja/supervisor.py b/maloja/supervisor.py index 5021e87..a84066f 100644 --- a/maloja/supervisor.py +++ b/maloja/supervisor.py @@ -1,7 +1,5 @@ #!/usr/bin/env python3 import os -from .globalconf import DATA_DIR -os.chdir(DATA_DIR) import subprocess import time diff --git a/maloja/utilities.py b/maloja/utilities.py index bee18a0..96452f0 100644 --- a/maloja/utilities.py +++ b/maloja/utilities.py @@ -17,6 +17,7 @@ from doreah.regular import yearly, daily from .external import api_request_track, api_request_artist from .__init__ import version from . import globalconf +from .globalconf import datadir @@ -170,7 +171,7 @@ def get_all_possible_filenames(artist=None,artists=None,title=None): else: return [] - superfolder = "images/tracks/" if track else "images/artists/" + superfolder = datadir("images/tracks/") if track else datadir("images/artists/") filenames = [] diff --git a/maloja/web/setup.py b/maloja/web/setup.py index e34faf3..596a3c1 100644 --- a/maloja/web/setup.py +++ b/maloja/web/setup.py @@ -1,4 +1,5 @@ import os +from ..globalconf import datadir def instructions(keys): @@ -8,7 +9,7 @@ def instructions(keys): validchars = "-_abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" - for f in os.listdir("rules/predefined"): + for f in os.listdir(datadir("rules/predefined")): if f.endswith(".tsv"): rawf = f.replace(".tsv","") @@ -22,7 +23,7 @@ def instructions(keys): if not "_" in rawf: continue try: - with open("rules/predefined/" + f) as tsvfile: + with open(datadir("rules/predefined",f)) as tsvfile: line1 = tsvfile.readline() line2 = tsvfile.readline() @@ -39,7 +40,7 @@ def instructions(keys): html += "" - if os.path.exists("rules/" + f): + if os.path.exists(datadir("rules",f)): html += "Remove:" else: html += "Add:"