ok
Mini Shell
#!/opt/cloudlinux/venv/bin/python3 -sbb
# coding=utf-8
#
# Copyright © Cloud Linux GmbH & Cloud Linux Software, Inc 2010-2020 All Rights Reserved
#
# Licensed under CLOUD LINUX LICENSE AGREEMENT
# http://cloudlinux.com/docs/LICENCE.TXT
#
import os
import contextlib
import sqlite3
from datetime import datetime, timedelta
from sqlalchemy import (
Column,
Boolean,
DateTime,
Integer,
String,
create_engine,
event, func
)
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.engine.reflection import Inspector
from sqlalchemy.orm import Session
SSA_DB = '/var/lve/ssa.db'
RETENTION_TIME_DAYS = 1
Base = declarative_base()
class RequestResult(Base):
"""
Describes processed request stored in database file.
E.g.
{
"timestamp": "1650008727",
"url": "http://mydomain.com/index.php",
"duration": 162077,
"hitting_limits": false,
"throttled_time": 0,
"io_throttled_time": 0,
"wordpress": true
}
Note: created_at, updated_at is saved in local TZ format
"""
__tablename__ = 'scrape_result'
id = Column(Integer, primary_key=True)
domain = Column(String, index=True, nullable=False)
path = Column(String, index=True, nullable=False)
timestamp = Column(Integer, nullable=False)
duration = Column(Integer, nullable=False)
is_slow_request = Column(Boolean, nullable=False)
hitting_limits = Column(Boolean, nullable=False)
throttled_time = Column(Integer, nullable=False)
io_throttled_time = Column(Integer, nullable=False)
wordpress = Column(Boolean, nullable=False)
created_at = Column(DateTime(timezone=True), server_default=func.now())
updated_at = Column(DateTime(timezone=True), onupdate=func.now(), server_default=func.now())
def cleanup_old_data(engine):
"""
Removes outdated records from database, saving disk space.
"""
n_days_ago = datetime.today() - timedelta(days=RETENTION_TIME_DAYS)
with session_scope(engine) as session:
session.query(RequestResult)\
.filter(RequestResult.created_at < n_days_ago)\
.delete()
def create_db_if_not_exist(engine):
if not is_db_present(engine):
Base.metadata.create_all(engine)
def is_db_present(engine):
if not os.path.isfile(SSA_DB):
return False
database_inspection = Inspector.from_engine(engine)
tables = [table for table in database_inspection.get_table_names()]
return len(tables) > 0
def setup_wal_mode(dbapi_con, con_record):
dbapi_con.execute('PRAGMA journal_mode = WAL')
def _setup_database(readonly):
connection_string = f'file:{SSA_DB}'
if readonly:
connection_string = f'{connection_string}?mode=ro'
creator = lambda: sqlite3.connect(connection_string, uri=True)
engine = create_engine(
'sqlite:////', creator=creator, echo=False,
)
event.listen(engine, 'connect', setup_wal_mode)
create_db_if_not_exist(engine)
return engine
def setup_database(readonly=False):
return _setup_database(readonly)
@contextlib.contextmanager
def session_scope(engine) -> Session:
"""
Provide a transactional scope around a series of operations.
"""
session = Session(bind=engine)
try:
yield session
session.commit()
except:
session.rollback()
raise
finally:
session.close()
Zerion Mini Shell 1.0