moved utils to sync/utils

This commit is contained in:
Sam Schott 2019-08-19 17:51:28 +01:00
parent 43b47e946b
commit 263be803a1
11 changed files with 268 additions and 251 deletions

View File

@ -17,210 +17,19 @@ import os
# external packages
import click
import Pyro4
import Pyro4.naming
import Pyro4.errors
Pyro4.config.SERIALIZER = "pickle"
Pyro4.config.SERIALIZERS_ACCEPTED.add('pickle')
URI = "PYRO:maestral.{0}@{1}"
OK = click.style("[OK]", fg="green")
FAILED = click.style("[FAILED]", fg="red")
KILLED = click.style("[KILLED]", fg="red")
# maestral modules
from maestral.sync.daemon import write_pid, delete_pid, get_maestral_process_info
from maestral.sync.daemon import (start_maestral_daemon, start_daemon_subprocess,
stop_maestral_daemon, MaestralProxy)
# ========================================================================================
# Maestral daemon
# ========================================================================================
def write_pid(config_name, socket_address="gui"):
"""
Write the PID of the current process to the appropriate file for the given
config name. If a socket_address is given, it will be appended after a '|'.
"""
from maestral.config.base import get_conf_path
pid_file = get_conf_path("maestral", config_name + ".pid")
with open(pid_file, "w") as f:
f.write(str(os.getpid()) + "|" + socket_address)
def read_pid(config_name):
"""
Reads the PID of the current process to the appropriate file for the given
config name.
"""
from maestral.config.base import get_conf_path
pid_file = get_conf_path("maestral", config_name + ".pid")
with open(pid_file, "r") as f:
pid, socket = f.read().split("|")
pid = int(pid)
return pid, socket
def delete_pid(config_name):
"""
Reads the PID of the current process to the appropriate file for the given
config name.
"""
from maestral.config.base import get_conf_path
pid_file = get_conf_path("maestral", config_name + ".pid")
os.unlink(pid_file)
def start_maestral_daemon(config_name):
"""
Wraps :class:`maestral.main.Maestral` as Pyro daemon object, creates a new instance
and start Pyro's event loop to listen for requests on 'localhost'. This call will
block until the event loop shuts down.
This command will create a new daemon on each run. Take care not to sync the same
directory with multiple instances of Meastral! You can use `get_maestral_process_info`
to check if either a Meastral gui or daemon is already running for the given
`config_name`.
:param str config_name: The name of maestral configuration to use.
"""
os.environ["MAESTRAL_CONFIG"] = config_name
from maestral.sync.main import Maestral
daemon = Pyro4.Daemon()
write_pid(config_name, daemon.locationStr) # write PID to file
try:
# we wrap this in a try-except block to make sure that the PID file is always
# removed, even when Maestral crashes for some reason
ExposedMaestral = Pyro4.expose(Maestral)
m = ExposedMaestral()
daemon.register(m, "maestral.{}".format(config_name))
daemon.requestLoop(loopCondition=m._shutdown_requested)
daemon.close()
except Exception:
import traceback
traceback.print_exc()
finally:
delete_pid(config_name) # remove PID file
def start_daemon_subprocess(config_name):
"""Starts the Maestral daemon as a subprocess (by calling `start_maestral_daemon`).
This command will create a new daemon on each run. Take care not to sync the same
directory with multiple instances of Meastral! You can use `get_maestral_process_info`
to check if either a Meastral gui or daemon is already running for the given
`config_name`.
:param str config_name: The name of maestral configuration to use.
:returns: Popen object instance.
"""
import subprocess
from maestral.sync.main import Maestral
if Maestral.pending_link() or Maestral.pending_dropbox_folder():
# run onboarding
m = Maestral(run=False)
m.create_dropbox_directory()
m.select_excluded_folders()
click.echo("Starting Maestral...", nl=False)
proc = subprocess.Popen("maestral sync -c {}".format(config_name),
shell=True, stdin=subprocess.DEVNULL,
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
# check if the subprocess is still running after 1 sec
try:
proc.wait(timeout=1)
click.echo("\rStarting Maestral... " + FAILED)
except subprocess.TimeoutExpired:
click.echo("\rStarting Maestral... " + OK)
return proc
def stop_maestral_daemon(config_name="maestral"):
"""stops maestral by finding its PID and shutting it down"""
import signal
import time
pid, socket, p_type = get_maestral_process_info(config_name)
if p_type == "daemon":
try:
# try to shut down gracefully
click.echo("Stopping Maestral...", nl=False)
with MaestralProxy(config_name) as m:
m.stop_sync()
m.shutdown_daemon()
except Pyro4.errors.CommunicationError:
try:
# send SIGTERM if failed
os.kill(pid, signal.SIGTERM)
except ProcessLookupError:
delete_pid(pid)
finally:
t0 = time.time()
while any(get_maestral_process_info(config_name)):
time.sleep(0.2)
if time.time() - t0 > 5:
# send SIGKILL if still running
os.kill(pid, signal.SIGKILL)
click.echo("\rStopping Maestral... " + KILLED)
return
click.echo("\rStopping Maestral... " + OK)
else:
click.echo("Maestral daemon is not running.")
def get_maestral_daemon_proxy(config_name="maestral", fallback=False):
"""
Returns a proxy of the running Maestral daemon. If fallback == True,
a new instance of Maestral will be returned when the daemon cannot be reached. This
can be dangerous if the GUI is running at the same time.
"""
pid, location, p_type = get_maestral_process_info(config_name)
if p_type == "daemon":
maestral_daemon = Pyro4.Proxy(URI.format(config_name, location))
try:
maestral_daemon._pyroBind()
return maestral_daemon
except Pyro4.errors.CommunicationError:
maestral_daemon._pyroRelease()
if fallback:
from maestral.sync.main import Maestral
m = Maestral(run=False)
return m
else:
raise Pyro4.errors.CommunicationError
class MaestralProxy(object):
"""A context manager to open and close a Proxy to the Maestral daemon."""
def __init__(self, config_name="maestral", fallback=False):
self.m = get_maestral_daemon_proxy(config_name, fallback)
def __enter__(self):
return self.m
def __exit__(self, exc_type, exc_value, traceback):
if hasattr(self.m, "_pyroRelease"):
self.m._pyroRelease()
def is_maestral_linked(config_name):
"""
This does not create a Maestral instance and is therefore safe to call from anywhere
@ -235,52 +44,6 @@ def is_maestral_linked(config_name):
return True
def get_maestral_process_info(config_name):
"""
Returns Maestral's PID, the socket location, and the type of instance as (pid,
socket, running) if Maestral is running or (``None``, ``None``, ``False``) otherwise.
Possible values for ``running`` are "gui", "daemon" or ``False``. Possible values for
``socket`` are "gui", "<network_address>:<port>" or "None".
If ``running == False`` but the PID and socket values are set, this means that
Maestral is running but is unresponsive. This function will attempt to kill it by
sending SIGKILL.
"""
import signal
pid = None
socket = None
running = False
try:
pid, socket = read_pid(config_name)
except Exception:
return pid, socket, running
try:
# test if the daemon process receives signals
os.kill(pid, 0)
except ProcessLookupError:
# if the process does not exist, delete pid file
try:
delete_pid(config_name)
except Exception:
pass
return pid, socket, running
except OSError:
# if the process does not respond, try to kill it
os.kill(pid, signal.SIGKILL)
try:
delete_pid(config_name)
except Exception:
pass
return pid, socket, running
else:
# everything ok, return process info
running = "gui" if socket == "gui" else "daemon"
return pid, socket, running
# ========================================================================================
# Command groups
# ========================================================================================
@ -593,7 +356,7 @@ def account_info(config_name: str, running):
@with_config_opt
def show(config_name: str, running):
"""Shows Maestral's log file."""
from maestral.utils.app_dirs import get_log_path
from maestral.sync.utils.app_dirs import get_log_path
log_file = get_log_path("maestral", config_name + ".log")
@ -612,7 +375,7 @@ def show(config_name: str, running):
@with_config_opt
def clear(config_name: str, running):
"""Clears Maestral's log file."""
from maestral.utils.app_dirs import get_log_path
from maestral.sync.utils.app_dirs import get_log_path
log_file = get_log_path("maestral", config_name + ".log")

View File

@ -10,8 +10,8 @@ import platform
import sys
import os
from maestral.utils.app_dirs import get_autostart_path
from maestral.utils import is_macos_bundle
from maestral.sync.utils.app_dirs import get_autostart_path
from maestral.sync.utils import is_macos_bundle
from maestral.sync.main import __version__
_root = os.path.abspath(os.path.dirname(__file__))

View File

@ -20,7 +20,7 @@ from PyQt5.QtGui import QBrush, QImage, QPainter, QPixmap, QWindow
# maestral modules
from maestral.gui.resources import APP_ICON_PATH, rgb_to_luminance
from maestral.utils import is_macos_bundle
from maestral.sync.utils import is_macos_bundle
THEME_DARK = "dark"
THEME_LIGHT = "light"

254
maestral/sync/daemon.py Normal file
View File

@ -0,0 +1,254 @@
# -*- coding: utf-8 -*-
"""
@author: Sam Schott (ss2151@cam.ac.uk)
(c) Sam Schott; This work is licensed under a Creative Commons
Attribution-NonCommercial-NoDerivs 2.0 UK: England & Wales License.
"""
# system imports
import os
# external packages
import Pyro4
import click
URI = "PYRO:maestral.{0}@{1}"
OK = click.style("[OK]", fg="green")
FAILED = click.style("[FAILED]", fg="red")
KILLED = click.style("[KILLED]", fg="red")
Pyro4.config.SERIALIZER = "pickle"
Pyro4.config.SERIALIZERS_ACCEPTED.add('pickle')
def write_pid(config_name, socket_address="gui"):
"""
Write the PID of the current process to the appropriate file for the given
config name. If a socket_address is given, it will be appended after a '|'.
"""
from maestral.config.base import get_conf_path
pid_file = get_conf_path("maestral", config_name + ".pid")
with open(pid_file, "w") as f:
f.write(str(os.getpid()) + "|" + socket_address)
def read_pid(config_name):
"""
Reads the PID of the current process to the appropriate file for the given
config name.
"""
from maestral.config.base import get_conf_path
pid_file = get_conf_path("maestral", config_name + ".pid")
with open(pid_file, "r") as f:
pid, socket = f.read().split("|")
pid = int(pid)
return pid, socket
def delete_pid(config_name):
"""
Reads the PID of the current process to the appropriate file for the given
config name.
"""
from maestral.config.base import get_conf_path
pid_file = get_conf_path("maestral", config_name + ".pid")
os.unlink(pid_file)
def start_maestral_daemon(config_name):
"""
Wraps :class:`maestral.main.Maestral` as Pyro daemon object, creates a new instance
and start Pyro's event loop to listen for requests on 'localhost'. This call will
block until the event loop shuts down.
This command will create a new daemon on each run. Take care not to sync the same
directory with multiple instances of Meastral! You can use `get_maestral_process_info`
to check if either a Meastral gui or daemon is already running for the given
`config_name`.
:param str config_name: The name of maestral configuration to use.
"""
os.environ["MAESTRAL_CONFIG"] = config_name
from maestral.sync.main import Maestral
daemon = Pyro4.Daemon()
write_pid(config_name, daemon.locationStr) # write PID to file
try:
# we wrap this in a try-except block to make sure that the PID file is always
# removed, even when Maestral crashes for some reason
ExposedMaestral = Pyro4.expose(Maestral)
m = ExposedMaestral()
daemon.register(m, "maestral.{}".format(config_name))
daemon.requestLoop(loopCondition=m._shutdown_requested)
daemon.close()
except Exception:
import traceback
traceback.print_exc()
finally:
delete_pid(config_name) # remove PID file
def start_daemon_subprocess(config_name):
"""Starts the Maestral daemon as a subprocess (by calling `start_maestral_daemon`).
This command will create a new daemon on each run. Take care not to sync the same
directory with multiple instances of Meastral! You can use `get_maestral_process_info`
to check if either a Meastral gui or daemon is already running for the given
`config_name`.
:param str config_name: The name of maestral configuration to use.
:returns: Popen object instance.
"""
import subprocess
from maestral.sync.main import Maestral
if Maestral.pending_link() or Maestral.pending_dropbox_folder():
# run onboarding
m = Maestral(run=False)
m.create_dropbox_directory()
m.select_excluded_folders()
click.echo("Starting Maestral...", nl=False)
proc = subprocess.Popen("maestral sync -c {}".format(config_name),
shell=True, stdin=subprocess.DEVNULL,
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
# check if the subprocess is still running after 1 sec
try:
proc.wait(timeout=1)
click.echo("\rStarting Maestral... " + FAILED)
except subprocess.TimeoutExpired:
click.echo("\rStarting Maestral... " + OK)
return proc
def stop_maestral_daemon(config_name="maestral"):
"""stops maestral by finding its PID and shutting it down"""
import signal
import time
pid, socket, p_type = get_maestral_process_info(config_name)
if p_type == "daemon":
try:
# try to shut down gracefully
click.echo("Stopping Maestral...", nl=False)
with MaestralProxy(config_name) as m:
m.stop_sync()
m.shutdown_daemon()
except Pyro4.errors.CommunicationError:
try:
# send SIGTERM if failed
os.kill(pid, signal.SIGTERM)
except ProcessLookupError:
delete_pid(pid)
finally:
t0 = time.time()
while any(get_maestral_process_info(config_name)):
time.sleep(0.2)
if time.time() - t0 > 5:
# send SIGKILL if still running
os.kill(pid, signal.SIGKILL)
click.echo("\rStopping Maestral... " + KILLED)
return
click.echo("\rStopping Maestral... " + OK)
else:
click.echo("Maestral daemon is not running.")
def get_maestral_daemon_proxy(config_name="maestral", fallback=False):
"""
Returns a proxy of the running Maestral daemon. If fallback == True,
a new instance of Maestral will be returned when the daemon cannot be reached. This
can be dangerous if the GUI is running at the same time.
"""
pid, location, p_type = get_maestral_process_info(config_name)
if p_type == "daemon":
maestral_daemon = Pyro4.Proxy(URI.format(config_name, location))
try:
maestral_daemon._pyroBind()
return maestral_daemon
except Pyro4.errors.CommunicationError:
maestral_daemon._pyroRelease()
if fallback:
from maestral.sync.main import Maestral
m = Maestral(run=False)
return m
else:
raise Pyro4.errors.CommunicationError
class MaestralProxy(object):
"""A context manager to open and close a Proxy to the Maestral daemon."""
def __init__(self, config_name="maestral", fallback=False):
self.m = get_maestral_daemon_proxy(config_name, fallback)
def __enter__(self):
return self.m
def __exit__(self, exc_type, exc_value, traceback):
if hasattr(self.m, "_pyroRelease"):
self.m._pyroRelease()
def get_maestral_process_info(config_name):
"""
Returns Maestral's PID, the socket location, and the type of instance as (pid,
socket, running) if Maestral is running or (``None``, ``None``, ``False``) otherwise.
Possible values for ``running`` are "gui", "daemon" or ``False``. Possible values for
``socket`` are "gui", "<network_address>:<port>" or "None".
If ``running == False`` but the PID and socket values are set, this means that
Maestral is running but is unresponsive. This function will attempt to kill it by
sending SIGKILL.
"""
import signal
pid = None
socket = None
running = False
try:
pid, socket = read_pid(config_name)
except Exception:
return pid, socket, running
try:
# test if the daemon process receives signals
os.kill(pid, 0)
except ProcessLookupError:
# if the process does not exist, delete pid file
try:
delete_pid(config_name)
except Exception:
pass
return pid, socket, running
except OSError:
# if the process does not respond, try to kill it
os.kill(pid, signal.SIGKILL)
try:
delete_pid(config_name)
except Exception:
pass
return pid, socket, running
else:
# everything ok, return process info
running = "gui" if socket == "gui" else "daemon"
return pid, socket, running

View File

@ -34,7 +34,7 @@ from maestral.sync.monitor import (MaestralMonitor, IDLE, DISCONNECTED,
path_exists_case_insensitive)
from maestral.config.main import CONF
from maestral.config.base import get_home_dir
from maestral.utils.app_dirs import get_log_path, get_cache_path
from maestral.sync.utils.app_dirs import get_log_path, get_cache_path
config_name = os.getenv("MAESTRAL_CONFIG", "maestral")

View File

@ -33,8 +33,8 @@ from watchdog.utils.dirsnapshot import DirectorySnapshot
# maestral modules
from maestral.config.main import CONF
from maestral.utils.content_hasher import DropboxContentHasher
from maestral.utils.notify import Notipy
from maestral.sync.utils.content_hasher import DropboxContentHasher
from maestral.sync.utils import Notipy
from maestral.sync.errors import (CONNECTION_ERRORS, MaestralApiError, CursorResetError,
RevFileError, DropboxDeletedError, DropboxAuthError)

View File

@ -16,7 +16,7 @@ import keyring
from keyring.errors import KeyringLocked
# maestral modules
from maestral.utils import is_macos_bundle, is_linux_bundle
from maestral.sync.utils import is_macos_bundle, is_linux_bundle
from maestral.config.main import CONF, SUBFOLDER
from maestral.config.base import get_conf_path
from maestral.sync.oauth_implicit import DropboxOAuth2FlowImplicit

View File

@ -8,7 +8,7 @@ Created on Wed Oct 31 16:23:13 2018
import os
from enum import Enum
from maestral.config.main import CONF
from maestral.utils import is_macos_bundle
from maestral.sync.utils import is_macos_bundle
if is_macos_bundle:
import Foundation