ok
Mini Shell
import datetime
import ftplib
import functools
import inspect
import os
import re
import shutil
import socket
import warnings
from itertools import chain
from dateutil import parser
def from_env(**param_to_env):
def decorator(f):
spec = inspect.getfullargspec(f)
assert all(
param in chain(spec.args, spec.kwonlyargs)
for param in param_to_env
)
f.from_env = param_to_env
return f
return decorator
class DateTime(datetime.datetime):
def __new__(cls, s):
return parser.parse(s, ignoretz=True)
fromtimestamp = datetime.datetime.fromtimestamp
now = datetime.datetime.now
utcfromtimestamp = datetime.datetime.utcfromtimestamp
def reconnect_ftp(func):
@functools.wraps(func)
def wrapper(ftp, *args, **kwargs):
try:
return func(ftp, *args, **kwargs)
except ftplib.error_temp:
ftp.connect()
return func(ftp, *args, **kwargs)
return wrapper
class FtpError(Exception):
pass
def _handle_ftp_error(func):
@functools.wraps(func)
def wrapper(ftp, *args, **kwargs):
try:
return func(ftp, *args, **kwargs)
except (ftplib.Error, socket.error) as e:
raise FtpError(*e.args)
return wrapper
@functools.lru_cache(maxsize=None)
class Ftp:
ftp = None
def __init__(self, host, login, password, use_ftps, passive_mode=True,
port=21, timeout=10, **_):
self.host = host
self.login = login
self.password = password
self.passive_mode = passive_mode
self.use_ftps = use_ftps
self.port = port
self.timeout = timeout
@property
def _address(self):
return self.login + '@' + self.host + ':' + str(self.port)
def __str__(self):
protocol = 'ftps' if self.use_ftps else 'ftp'
return protocol + '://' + self._address
@_handle_ftp_error
def connect(self):
if self.use_ftps:
ftp = ftplib.FTP_TLS()
else:
ftp = ftplib.FTP()
ftp.connect(self.host, self.port, self.timeout)
ftp.login(self.login, self.password)
if self.use_ftps:
ftp.prot_p()
ftp.set_pasv(self.passive_mode)
self.ftp = ftp
@_handle_ftp_error
@reconnect_ftp
def listdir(self, *args):
return self.ftp.nlst(*args)
@_handle_ftp_error
@reconnect_ftp
def mlistdir(self, *args, **kwargs):
return self.ftp.mlsd(*args, **kwargs)
@_handle_ftp_error
@reconnect_ftp
def retrieve(self, path, destination):
"""
:raises FtpError:
:raises IsNotDirError:
:raises DirNotEmptyError:
"""
if not os.path.isabs(destination):
destination = os.path.abspath(destination)
destination = os.path.join(destination, self._address)
target_name = os.path.join(destination, path.lstrip(os.path.sep))
target_dir = os.path.dirname(target_name)
mkdir(target_dir)
free_space = shutil.disk_usage(target_dir).free
try:
target_size = self.size(path)
except FtpError:
# Skip this step if SIZE command is not supported on the server
pass
else:
if target_size > free_space:
raise FtpError('Disk is full', free_space, target_size)
with open(target_name, 'wb') as w:
self.ftp.retrbinary('RETR ' + path, w.write)
return target_name
@_handle_ftp_error
@reconnect_ftp
def size(self, dir_path):
""" Get the size of the file on the server """
try:
self.ftp.sendcmd('TYPE I')
return self.ftp.size(dir_path)
finally:
# Fallback to the default ASCII mode
self.ftp.sendcmd('TYPE A')
class DirNotEmptyError(FileExistsError):
def __init__(self, message=None):
message = message or 'destination directory exists and it is not empty'
super().__init__(message)
class IsNotDirError(FileExistsError):
def __init__(self, message=None):
message = message or 'destination exists but it is not a directory'
super().__init__(message)
class ActionError(Exception):
def __init__(self, message=None, code=1):
self.message = message
self.code = code
def mkdir(path, check_empty=False):
if os.path.isdir(path):
if check_empty and os.listdir(path):
raise DirNotEmptyError()
elif os.path.exists(path):
raise IsNotDirError()
else:
os.makedirs(path)
def read(fileobj, size=2 ** 20):
for chunk in iter(lambda: fileobj.read(size), b''):
yield chunk
optional_arg_match = re.compile(r'--(?P<key>\w+)(=(?P<value>\S+))?')
def parse_extra_args(arg_list):
# FIXME: move this logic into a special ArgumentParser argument and
# get rid of `extra_args` magic
args = []
kwargs = {}
for arg in arg_list:
match = optional_arg_match.match(arg)
if match:
kw = match.groupdict()
value = kw['value']
if value is None or value.lower() in ('1', 'true', 'yes', 'y'):
value = True
elif value.lower() in ('0', 'false', 'no', 'n'):
value = False
kwargs[kw['key']] = value
else:
args.append(arg)
return args, kwargs
def _fill_out_positional_args(parsed_args, env_vars, positional_parameters):
if len(parsed_args) < len(positional_parameters):
provided_args_it = iter(parsed_args)
result = []
for param in positional_parameters:
if param in env_vars:
result.append(env_vars[param])
else:
try:
result.append(next(provided_args_it))
except StopIteration:
continue
result.extend(provided_args_it)
return result
return parsed_args
def _fill_out_keyword_args(parsed_kwargs, env_vars, keyword_parameters):
return {
**{
arg: env_vars[arg] for arg in keyword_parameters if arg in env_vars
},
**parsed_kwargs,
}
def fill_args_from_env(spec, parsed_args, parsed_kwargs, env_vars):
defaults_len = len(spec.defaults) if spec.defaults else 0
required = spec.args[: -defaults_len]
optional = spec.args[-defaults_len:]
return (
_fill_out_positional_args(parsed_args, env_vars, required),
_fill_out_keyword_args(parsed_kwargs, env_vars, optional),
)
def validate_params(spec, args, kwargs):
"""
Return tuple of lists with missing and unknown arguments of a func
:param spec: func spec
:param args: list of args to validate
:param kwargs: list of args to validate
:return: tuple of missing and unknown lists of arguments
"""
defaults_len = -len(spec.defaults) if spec.defaults else None
required = spec.args[:defaults_len]
required_len = len(required)
args_len = len(args)
varargs = spec.varargs
if not varargs and required_len != args_len or required_len > args_len:
missing = required
else:
missing = []
optional = spec.args[defaults_len:]
unknown = list(set(kwargs) - set(optional))
return missing, unknown
def _formatwarning(message, *_, **__):
return 'Warning: %s%s' % (message, os.linesep)
warnings.formatwarning = _formatwarning
def warning(message, once=False):
if once:
warnings.warn(message)
else:
with warnings.catch_warnings():
warnings.simplefilter('always')
warnings.warn(message)
Zerion Mini Shell 1.0