ok
Mini Shell
"""
This program is free software: you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License,
or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
See the GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
Copyright © 2019 Cloud Linux Software Inc.
This software is also available under ImunifyAV commercial license,
see <https://www.imunify360.com/legal/eula>
Utilities to help upload a malicious file."""
import asyncio
import json
import logging
import os
from dataclasses import dataclass
from typing import Iterable, List
from urllib.parse import quote_from_bytes, urljoin
import urllib
from defence360agent import utils
from defence360agent.contracts.config import Core, Malware as Config
from defence360agent.contracts.license import LicenseCLN, LicenseError
from defence360agent.internals.iaid import (
IAIDTokenError,
IndependentAgentIDAPI,
)
from imav.contracts.config import MalwareTune
logger = logging.getLogger(__name__)
if utils.OsReleaseInfo.id_like() & utils.OsReleaseInfo.DEBIAN:
_CURL = "/opt/alt/curlssl/usr/bin/curl"
else:
_CURL = "/opt/alt/curlssl11/usr/bin/curl"
_API_BASE_URL = os.environ.get("I360_MRS_API_BASE_URL", Core.API_BASE_URL)
_ENDPOINT_UPLOAD = os.environ.get("I360_MRS_ENDPOINT_UPLOAD", "api/v1/upload")
_ENDPOINT_CHECK = os.environ.get(
"I360_MRS_ENDPOINT_CHECK", "api/v1/check-known-hashes"
)
_POST_FILE_TIMEOUT = int(
os.environ.get("IMUNIFY360_POST_FILE_TIMEOUT", 60 * 60) # hour
)
FALSE_NEGATIVE = "false_negative"
FALSE_POSITIVE = "false_positive"
UNKNOWN_REASON = "unknown"
class ClientError(OSError):
"""HTTP client error.
It is used to hide what specific http client is used by upload_file().
"""
class FileTooLargeError(OSError):
pass
def _token_to_headers():
token = LicenseCLN.get_token()
headers = {
"I360-Id": token["id"],
"I360-Limit": token["limit"],
"I360-Status": token["status"],
"I360-Token-Expire-Utc": token["token_expire_utc"],
"I360-Token-Created-Utc": token["token_created_utc"],
"I360-Sign": token["sign"],
}
headers = {key: str(value) for key, value in headers.items()}
return headers
async def _post_file(filename, url, headers=None, timeout=None):
"""
Post *filename* as multipart/form-data to *url* with given HTTP *headers*.
Return server response as bytes (http body).
Raise TimeoutError on timeout.
Raise ConnectionError if failed to connect to host.
Raise ClientError on error.
"""
if headers is None:
headers = {}
headers_args = [
b"-H%s: %s" % (header.encode("ascii"), value.encode("latin-1"))
for header, value in headers.items()
]
quoted_full_path = quote_from_bytes(os.fsencode(filename), safe="").encode(
"ascii"
)
cmd = (
[os.fsencode(_CURL)]
+ headers_args
+ [b"--max-time", str(timeout).encode("ascii")] * (timeout is not None)
+ [
b"--form",
# https://curl.haxx.se/docs/knownbugs.html#multipart_formposts_file_name_en
b'file=@"%s";filename="%s"'
% (
# escape backslash, double quotes
os.fsencode(filename)
.replace(b"\\", b"\\\\")
.replace(b'"', b'\\"'),
quoted_full_path,
),
b"--fail",
# disable progress meter
b"--silent",
b"--show-error",
url.encode("ascii"),
]
)
rc, out, err = await utils.run(cmd)
if rc != 0:
Error = (
ConnectionError
if rc == 7
else TimeoutError
if rc == 28
else ClientError
)
raise Error(
(
"Failed to post {filename} to {url}:"
" curl: cmd={cmd}, rc={rc}, out={out}, err={err}"
).format(**vars())
)
return out
async def upload_file(file: str, upload_reason=UNKNOWN_REASON):
"""
Upload a file to Malware Response Service.
:param file: path to file
:param upload_reason: one of 'unknown', 'false_positive', 'false_negative'
:return: dict representing json response
:raises LicenseError:
"""
if not LicenseCLN.is_valid():
raise LicenseError(
"File uploading to Malware Responce Serivce "
"requires a valid license"
)
file_size = os.path.getsize(file)
if file_size > Config.MAX_MRS_UPLOAD_FILE:
raise FileTooLargeError(
"File {} is {} bytes, files larger than {} bytes "
"are not allowed.".format(
file, file_size, Config.MAX_MRS_UPLOAD_FILE
)
)
url = urljoin(_API_BASE_URL, _ENDPOINT_UPLOAD)
headers = {
**_token_to_headers(),
"I360-Upload-Reason": upload_reason,
}
response_body = await _post_file(
file, url, headers, timeout=_POST_FILE_TIMEOUT
)
result = json.loads(response_body.decode())
logger.info(
"Uploaded file %r to the Malware Response Service with reason: %s",
file,
upload_reason,
)
return result
async def upload_with_retries(file, upload_reason=UNKNOWN_REASON):
"""
:raises LicenseError,
ClientError,
TimeoutError,
ConnectionError,
"""
# exponential backoff: total delay ~6 min (not including the upload time)
delays = [0.5, 2.5, 6, 15, 40, 100, 200]
max_tries = len(delays) + 1
for i, pause in enumerate(delays, start=1):
error = await _try_upload(
file, raise_errors=False, upload_reason=upload_reason
)
if not error:
break
if isinstance(error, FileTooLargeError):
logger.warning("File %s is too big. Stop retrying to upload", file)
break
logger.warning(
"Attempt %d/%d: failed uploading file %s, reason: %s. Retrying in"
" %s seconds",
i,
max_tries,
file,
error,
pause,
)
await asyncio.sleep(pause)
else: # exhausted retries, one last attempt, raise error if it fails
await _try_upload(file, raise_errors=True, upload_reason=upload_reason)
async def _try_upload(file, raise_errors, *, upload_reason=UNKNOWN_REASON):
"""Return error instead of raising it unless *raise_errors* is true.
:raises LicenseError:
:raises ClientError,
TimeoutError,
ConnectionError,
FileTooLargeError: if raise_errors is True
"""
try:
await upload_file(file, upload_reason=upload_reason)
except (
ClientError,
ConnectionError,
FileTooLargeError,
TimeoutError,
) as e:
if raise_errors:
raise e
return e
else:
return None
@dataclass
class HitInfo:
file: str
hash: str
async def check_known_hashes(
loop, hashes: Iterable[str], upload_reason=UNKNOWN_REASON
) -> List[str]:
hashes = list(hashes)
if MalwareTune.NO_CHECK_KNOWN_HASHES:
return hashes
try:
token = await IndependentAgentIDAPI.get_token()
except IAIDTokenError:
return hashes
url = urljoin(_API_BASE_URL, _ENDPOINT_CHECK)
headers = {
"X-Auth": token,
"I360-Upload-Reason": upload_reason,
"Content-Type": "application/json",
}
request = {"hashes": hashes}
try:
result = await loop.run_in_executor(
None,
_do_request,
urllib.request.Request(
url,
data=json.dumps(request).encode(),
headers=headers,
method="POST",
),
)
except Exception as e:
logger.warning("Failed to check known hashes: %s", e)
return hashes
return result["unknown_hashes"]
def _do_request(request: urllib.request.Request) -> None:
with urllib.request.urlopen(
request, timeout=Core.DEFAULT_SOCKET_TIMEOUT
) as response:
if response.status != 200:
raise Exception("status code is {}".format(response.status))
return json.loads(response.read().decode())
Zerion Mini Shell 1.0