Merge branch 'processcontrolrework'

maloja/__pkginfo__.py

@@ -33,5 +33,5 @@ resources = [
 ]
 
 commands = {
-	"maloja":"controller:main"
+	"maloja":"proccontrol.control:main"
 }

maloja/controller.py

@@ -1,202 +0,0 @@
-#!/usr/bin/env python3
-
-import subprocess
-import sys
-import signal
-import os
-import shutil
-from distutils import dir_util
-import stat
-import pathlib
-import pkg_resources
-from doreah.control import mainfunction
-from doreah.io import col, ask, prompt
-
-from .globalconf import datadir
-from .backup import backup
-
-
-
-
-def copy_initial_local_files():
-	folder = pkg_resources.resource_filename(__name__,"data_files")
-	#shutil.copy(folder,DATA_DIR)
-	dir_util.copy_tree(folder,datadir(),update=False)
-
-
-def setup():
-
-	copy_initial_local_files()
-
-	from doreah import settings
-
-	# EXTERNAL API KEYS
-	apikeys = {
-		"LASTFM_API_KEY":"Last.fm API Key",
-		"FANARTTV_API_KEY":"Fanart.tv API Key",
-		"SPOTIFY_API_ID":"Spotify Client ID",
-		"SPOTIFY_API_SECRET":"Spotify Client Secret"
-	}
-
-	SKIP = settings.get_settings("SKIP_SETUP")
-
-	print("Various external services can be used to display images. If not enough of them are set up, only local images will be used.")
-	for k in apikeys:
-		key = settings.get_settings(k)
-		if key is None:
-			print("\t" + "Currently not using a " + apikeys[k] + " for image display.")
-		elif key == "ASK":
-			print("\t" + "Please enter your " + apikeys[k] + ". If you do not want to use one at this moment, simply leave this empty and press Enter.")
-			key = prompt("",types=(str,),default=None,skip=SKIP)
-			settings.update_settings(datadir("settings/settings.ini"),{k:key},create_new=True)
-		else:
-			print("\t" + apikeys[k] + " found.")
-
-
-	# OWN API KEY
-	if os.path.exists(datadir("clients/authenticated_machines.tsv")):
-		pass
-	else:
-		answer = ask("Do you want to set up a key to enable scrobbling? Your scrobble extension needs that key so that only you can scrobble tracks to your database.",default=True,skip=SKIP)
-		if answer:
-			import random
-			key = ""
-			for i in range(64):
-				key += str(random.choice(list(range(10)) + list("abcdefghijklmnopqrstuvwxyz") + list("ABCDEFGHIJKLMNOPQRSTUVWXYZ")))
-			print("Your API Key: " + col["yellow"](key))
-			with open(datadir("clients/authenticated_machines.tsv"),"w") as keyfile:
-				keyfile.write(key + "\t" + "Default Generated Key")
-		else:
-			pass
-
-
-	if settings.get_settings("NAME") is None:
-		name = prompt("Please enter your name. This will be displayed e.g. when comparing your charts to another user. Leave this empty if you would not like to specify a name right now.",default="Generic Maloja User",skip=SKIP)
-		settings.update_settings(datadir("settings/settings.ini"),{"NAME":name},create_new=True)
-
-	if settings.get_settings("SEND_STATS") is None:
-		answer = ask("I would like to know how many people use Maloja. Would it be okay to send a daily ping to my server (this contains no data that isn't accessible via your web interface already)?",default=True,skip=SKIP)
-		if answer:
-			settings.update_settings(datadir("settings/settings.ini"),{"SEND_STATS":True,"PUBLIC_URL":None},create_new=True)
-		else:
-			settings.update_settings(datadir("settings/settings.ini"),{"SEND_STATS":False},create_new=True)
-
-
-def getInstance():
-	try:
-		output = subprocess.check_output(["pidof","Maloja"])
-		pid = int(output)
-		return pid
-	except:
-		return None
-
-def getInstanceSupervisor():
-	try:
-		output = subprocess.check_output(["pidof","maloja_supervisor"])
-		pid = int(output)
-		return pid
-	except:
-		return None
-
-def start():
-	setup()
-	try:
-		#p = subprocess.Popen(["python3","-m","maloja.server"],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
-		sp = subprocess.Popen(["python3","-m","maloja.supervisor"],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
-		print(col["green"]("Maloja started!"))
-
-		from doreah import settings
-		port = settings.get_settings("WEB_PORT")
-
-		print("Visit your server address (Port " + str(port) + ") to see your web interface. Visit /setup to get started.")
-		print("If you're installing this on your local machine, these links should get you there:")
-		print("\t" + col["blue"]("http://localhost:" + str(port)))
-		print("\t" + col["blue"]("http://localhost:" + str(port) + "/setup"))
-		return True
-	except:
-		print("Error while starting Maloja.")
-		return False
-
-def restart():
-	wasrunning = stop()
-	start()
-	return wasrunning
-
-def stop():
-	pid_sv = getInstanceSupervisor()
-	if pid_sv is not None:
-		os.kill(pid_sv,signal.SIGTERM)
-#		return True
-
-#	else:
-#		print("Server is not running")
-#		return False
-
-
-	pid = getInstance()
-	if pid is not None:
-#		print("Server is not running")
-#		return False
-#		pass
-#	else:
-		os.kill(pid,signal.SIGTERM)
-#		print("Maloja stopped! PID: " + str(pid))
-	if pid is not None or pid_sv is not None:
-		return True
-	else:
-		return False
-
-
-def loadlastfm(filename):
-
-	if not os.path.exists(filename):
-		print("File could not be found.")
-		return
-
-	if os.path.exists(datadir("scrobbles/lastfmimport.tsv")):
-		print("Already imported Last.FM data. Overwrite? [y/N]")
-		if input().lower() in ["y","yes","yea","1","positive","true"]:
-			pass
-		else:
-			return
-	print("Please wait...")
-	from .lastfmconverter import convert
-	convert(filename,datadir("scrobbles/lastfmimport.tsv"))
-	#os.system("python3 -m maloja.lastfmconverter " + filename + " " + datadir("scrobbles/lastfmimport.tsv"))
-	print("Successfully imported your Last.FM scrobbles!")
-
-def direct():
-	setup()
-	from . import server
-
-def backuphere():
-	backup(folder=os.getcwd())
-
-def update():
-	os.system("pip3 install malojaserver --upgrade --no-cache-dir")
-	restart()
-
-def fixdb():
-	from .fixexisting import fix
-	fix()
-
-@mainfunction({"l":"level"},shield=True)
-def main(action,*args,**kwargs):
-	actions = {
-		"start":restart,
-		"restart":restart,
-		"stop":stop,
-		"import":loadlastfm,
-		"debug":direct,
-		"backup":backuphere,
-		"update":update,
-		"fix":fixdb,
-		"run":direct
-	}
-	if action in actions: actions[action](*args,**kwargs)
-	else: print("Valid commands: " + " ".join(a for a in actions))
-
-	return True
-
-#if __name__ == "__main__":
-#	main()

maloja/data_files/settings/default.ini

@@ -59,7 +59,7 @@ SCROBBLES_GOLD = 250
 SCROBBLES_PLATINUM = 500
 SCROBBLES_DIAMOND = 1000
 # name for comparisons
-NAME = "A Maloja User"
+NAME = None
 
 [Misc]
 
@@ -67,7 +67,5 @@ EXPERIMENTAL_FEATURES = no
 USE_PYHP = no #not recommended at the moment
 USE_JINJA = no #overwrites pyhp preference
 FEDERATION = yes #does nothing yet
-UPDATE_AFTER_CRASH = no #update when server is automatically restarted
-DAILY_RESTART = 2 # hour of day. no / none means no daily restarts
 SKIP_SETUP = no
 LOGGING = true

maloja/database.py

@@ -854,7 +854,7 @@ def rebuild(**keys):
 		global db_rulestate
 		db_rulestate = False
 		sync()
-		from .fixexisting import fix
+		from .proccontrol.tasks.fixexisting import fix
 		fix()
 		global cla, coa
 		cla = CleanerAgent()

maloja/proccontrol/control.py

@@ -0,0 +1,94 @@
+import subprocess
+from doreah import settings
+from doreah.control import mainfunction
+from doreah.io import col
+import os
+import signal
+
+from .setup import setup
+from . import tasks
+
+def getInstance():
+	try:
+		output = subprocess.check_output(["pidof","Maloja"])
+		pid = int(output)
+		return pid
+	except:
+		return None
+
+def getInstanceSupervisor():
+	try:
+		output = subprocess.check_output(["pidof","maloja_supervisor"])
+		pid = int(output)
+		return pid
+	except:
+		return None
+
+def restart():
+	stop()
+	start()
+
+def start():
+	if getInstanceSupervisor() is not None:
+		print("Maloja is already running.")
+	else:
+		setup()
+		try:
+			#p = subprocess.Popen(["python3","-m","maloja.server"],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
+			sp = subprocess.Popen(["python3","-m","maloja.proccontrol.supervisor"],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
+			print(col["green"]("Maloja started!"))
+
+			port = settings.get_settings("WEB_PORT")
+
+			print("Visit your server address (Port " + str(port) + ") to see your web interface. Visit /setup to get started.")
+			print("If you're installing this on your local machine, these links should get you there:")
+			print("\t" + col["blue"]("http://localhost:" + str(port)))
+			print("\t" + col["blue"]("http://localhost:" + str(port) + "/setup"))
+			return True
+		except:
+			print("Error while starting Maloja.")
+			return False
+
+
+def stop():
+
+	pid_sv = getInstanceSupervisor()
+	if pid_sv is not None:
+		os.kill(pid_sv,signal.SIGTERM)
+
+	pid = getInstance()
+	if pid is not None:
+		os.kill(pid,signal.SIGTERM)
+
+	if pid is not None or pid_sv is not None:
+		print("Maloja stopped!")
+		return True
+	else:
+		return False
+
+
+
+def direct():
+	setup()
+	from .. import server
+
+
+
+@mainfunction({"l":"level"},shield=True)
+def main(action,*args,**kwargs):
+	actions = {
+		"start":start,
+		"restart":restart,
+		"stop":stop,
+		"run":direct,
+		"debug":direct,
+
+		"import":tasks.loadlastfm,
+		"backup":tasks.backuphere,
+	#	"update":update,
+		"fix":tasks.fixdb
+	}
+	if action in actions: actions[action](*args,**kwargs)
+	else: print("Valid commands: " + " ".join(a for a in actions))
+
+	return True

maloja/proccontrol/setup.py

@@ -0,0 +1,71 @@
+import pkg_resources
+from distutils import dir_util
+from doreah import settings
+from doreah.io import col, ask, prompt
+import os
+
+from ..globalconf import datadir
+
+
+# EXTERNAL API KEYS
+apikeys = {
+	"LASTFM_API_KEY":"Last.fm API Key",
+	"FANARTTV_API_KEY":"Fanart.tv API Key",
+	"SPOTIFY_API_ID":"Spotify Client ID",
+	"SPOTIFY_API_SECRET":"Spotify Client Secret"
+}
+
+
+
+def copy_initial_local_files():
+	folder = pkg_resources.resource_filename("maloja","data_files")
+	#shutil.copy(folder,DATA_DIR)
+	dir_util.copy_tree(folder,datadir(),update=False)
+
+
+
+def setup():
+
+	copy_initial_local_files()
+	SKIP = settings.get_settings("SKIP_SETUP")
+
+	print("Various external services can be used to display images. If not enough of them are set up, only local images will be used.")
+	for k in apikeys:
+		key = settings.get_settings(k)
+		if key is None:
+			print("\t" + "Currently not using a " + apikeys[k] + " for image display.")
+		elif key == "ASK":
+			print("\t" + "Please enter your " + apikeys[k] + ". If you do not want to use one at this moment, simply leave this empty and press Enter.")
+			key = prompt("",types=(str,),default=None,skip=SKIP)
+			settings.update_settings(datadir("settings/settings.ini"),{k:key},create_new=True)
+		else:
+			print("\t" + apikeys[k] + " found.")
+
+
+	# OWN API KEY
+	if os.path.exists(datadir("clients/authenticated_machines.tsv")):
+		pass
+	else:
+		answer = ask("Do you want to set up a key to enable scrobbling? Your scrobble extension needs that key so that only you can scrobble tracks to your database.",default=True,skip=SKIP)
+		if answer:
+			import random
+			key = ""
+			for i in range(64):
+				key += str(random.choice(list(range(10)) + list("abcdefghijklmnopqrstuvwxyz") + list("ABCDEFGHIJKLMNOPQRSTUVWXYZ")))
+			print("Your API Key: " + col["yellow"](key))
+			with open(datadir("clients/authenticated_machines.tsv"),"w") as keyfile:
+				keyfile.write(key + "\t" + "Default Generated Key")
+		else:
+			pass
+
+
+	if settings.get_settings("NAME") is None:
+		name = prompt("Please enter your name. This will be displayed e.g. when comparing your charts to another user. Leave this empty if you would not like to specify a name right now.",default="Generic Maloja User",skip=SKIP)
+		settings.update_settings(datadir("settings/settings.ini"),{"NAME":name},create_new=True)
+
+	if settings.get_settings("SEND_STATS") is None:
+		answer = ask("I would like to know how many people use Maloja. Would it be okay to send a daily ping to my server (this contains no data that isn't accessible via your web interface already)?",default=True,skip=SKIP)
+		if answer:
+			settings.update_settings(datadir("settings/settings.ini"),{"SEND_STATS":True,"PUBLIC_URL":None},create_new=True)
+		else:
+			settings.update_settings(datadir("settings/settings.ini"),{"SEND_STATS":False},create_new=True)

maloja/proccontrol/supervisor.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python3
+import os
+
+import subprocess
+import setproctitle
+import signal
+from doreah.logging import log
+from doreah.settings import get_settings
+
+from .control import getInstance
+
+
+setproctitle.setproctitle("maloja_supervisor")
+
+def update():
+	log("Updating...",module="supervisor")
+	try:
+		os.system("pip3 install maloja --upgrade --no-cache-dir")
+	except:
+		log("Could not update.",module="supervisor")
+
+def start():
+	try:
+		p = subprocess.Popen(["python3","-m","maloja.server"],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
+		return p
+	except e:
+		log("Error starting Maloja: " + str(e),module="supervisor")
+
+
+
+while True:
+	log("Maloja is not running, starting...",module="supervisor")
+	if get_settings("UPDATE_AFTER_CRASH"):
+		update()
+	process = start()
+
+	process.wait()

maloja/proccontrol/tasks/__init__.py

@@ -0,0 +1,34 @@
+import os
+from .lastfmconverter import convert
+from .backup import backup
+from .fixexisting import fix
+
+from ...globalconf import datadir
+from ..control import restart
+from doreah.io import ask
+
+def loadlastfm(filename):
+
+	if not os.path.exists(filename):
+		print("File could not be found.")
+		return
+
+	if os.path.exists(datadir("scrobbles/lastfmimport.tsv")):
+		overwrite = ask("Already imported Last.FM data. Overwrite?",default=False)
+		if not overwrite: return
+	print("Please wait...")
+
+	convert(filename,datadir("scrobbles/lastfmimport.tsv"))
+	#os.system("python3 -m maloja.lastfmconverter " + filename + " " + datadir("scrobbles/lastfmimport.tsv"))
+	print("Successfully imported your Last.FM scrobbles!")
+
+
+def backuphere():
+	backup(folder=os.getcwd())
+
+def update():
+	os.system("pip3 install malojaserver --upgrade --no-cache-dir")
+	restart()
+
+def fixdb():
+	fix()

maloja/proccontrol/tasks/backup.py

@@ -2,7 +2,10 @@ import tarfile
 from datetime import datetime
 import glob
 import os
-from .globalconf import datadir
+from ...globalconf import datadir
+from pathlib import PurePath
+
+from doreah.logging import log
 
 
 user_files = {
@@ -25,6 +28,8 @@ def backup(folder,level="full"):
 	for g in selected_files:
 		real_files += glob.glob(datadir(g))
 
+	log("Creating backup of " + str(len(real_files)) + " files...")
+
 	now = datetime.utcnow()
 	timestr = now.strftime("%Y_%m_%d_%H_%M_%S")
 	filename = "maloja_backup_" + timestr + ".tar.gz"
@@ -32,4 +37,7 @@ def backup(folder,level="full"):
 	assert not os.path.exists(archivefile)
 	with tarfile.open(name=archivefile,mode="x:gz") as archive:
 		for f in real_files:
-			archive.add(f)
+			p = PurePath(f)
+			r = p.relative_to(datadir())
+			archive.add(f,arcname=r)
+	log("Backup created!")

maloja/proccontrol/tasks/fixexisting.py

@@ -1,7 +1,7 @@
 import os
-from .globalconf import datadir
+from ...globalconf import datadir
 import re
-from .cleanup import CleanerAgent
+from ...cleanup import CleanerAgent
 from doreah.logging import log
 import difflib
 import datetime
@@ -35,9 +35,10 @@ def fix():
 
 	#with open(datadir("logs","dbfix",nowstr + ".log"),"a") as logfile:
 
-
+	log("Fixing database...")
 	for filename in os.listdir(datadir("scrobbles")):
 		if filename.endswith(".tsv"):
+			log("Fix file " + filename)
 			filename_new = filename + "_new"
 
 			with open(datadir("scrobbles",filename_new),"w") as newfile:
@@ -68,3 +69,5 @@ def fix():
 
 			with open(datadir("scrobbles",filename + ".rulestate"),"w") as checkfile:
 				checkfile.write(wendigo.checksums)
+
+	log("Database fixed!")

maloja/proccontrol/tasks/lastfmconverter.py

@@ -1,6 +1,6 @@
 import os, datetime, re
-from .cleanup import *
+from ...cleanup import *
-from .utilities import *
+from ...utilities import *
 
 
 

maloja/supervisor.py

@@ -1,67 +0,0 @@
-#!/usr/bin/env python3
-import os
-
-import subprocess
-import time
-import setproctitle
-import signal
-from datetime import datetime
-from doreah.logging import log
-from doreah.settings import get_settings
-
-
-setproctitle.setproctitle("maloja_supervisor")
-
-lastrestart = ()
-
-def get_pid():
-	try:
-		output = subprocess.check_output(["pidof","Maloja"])
-		return int(output)
-	except:
-		return None
-
-def update():
-	log("Updating...",module="supervisor")
-	try:
-		os.system("pip3 install maloja --upgrade --no-cache-dir")
-	except:
-		log("Could not update.",module="supervisor")
-
-def start():
-	try:
-		p = subprocess.Popen(["python3","-m","maloja.server"],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
-
-	except e:
-		log("Error starting Maloja: " + str(e),module="supervisor")
-
-
-
-while True:
-
-
-	now = datetime.now()
-	today = now.year, now.month, now.day
-
-	pid = get_pid()
-
-	if pid:
-
-		restart = get_settings("DAILY_RESTART")
-		if restart not in [None,False]:
-			if today != lastrestart:
-				if now.hour == restart:
-					log("Daily restart...",module="supervisor")
-					os.kill(pid,signal.SIGTERM)
-					start()
-					lastrestart = today
-
-	else:
-		log("Maloja is not running, starting...",module="supervisor")
-		if get_settings("UPDATE_AFTER_CRASH"):
-			update()
-		start()
-		lastrestart = today
-
-
-	time.sleep(60)