FM / agent

/opt/cloudlinux/venv/lib/python3.11/site-packages/xray/agent/

/opt/cloudlinux/venv/lib/python3.11/site-packages/xray/agent

Name	Size	Mode	Actions
__pycache__/	-	0755	rm
daemon.py	33808	0644	edit dl rm
executor.py	2195	0644	edit dl rm
fault_detector.py	8807	0644	edit dl rm
__init__.py	231	0644	edit dl rm

Edit: /opt/cloudlinux/venv/lib/python3.11/site-packages/xray/agent/fault_detector.py (8807B)

# -*- coding: utf-8 -*-

# Copyright © Cloud Linux GmbH & Cloud Linux Software, Inc 2010-2021 All Rights Reserved
#
# Licensed under CLOUD LINUX LICENSE AGREEMENT
# http://cloudlinux.com/docs/LICENSE.TXT

"""
This module contains FaultDetector class, aimed to track throttling
for incoming requests
"""
import collections
import logging
import time
from datetime import datetime, timedelta
from threading import RLock
from typing import Optional, Tuple, Any

from xray.internal.constants import drop_after, check_period, throttling_threshold

class FaultDetector:
    """
    Fault Detector class
    """

_MAX_ENTRIES = 10000
    # Per-UID sub-limit on how many entries a single SO_PEERCRED uid may occupy
    # in the shared throttling_stat dict. The request-init path is reachable by
    # any local uid (the agent socket is world-rw by design), so a per-UID quota
    # prevents one uid from flooding distinct PIDs and starving other tenants'
    # entries within the global _MAX_ENTRIES cap. 1000 is ~10x above realistic
    # concurrent PHP worker counts per user (PHP-FPM pm.max_children) yet only
    # 1/10 of the global cap, so a single uid's flood can never block another.
    _MAX_ENTRIES_PER_UID = 1000

def __init__(self):
        self.logger = logging.getLogger('fault_detector')
        self.throttling_stat = dict()
        # per-UID bookkeeping: number of live entries owned by each uid, and the
        # owning uid for each key (kept in lockstep with throttling_stat so that
        # expiry/consumption can decrement the right uid's count).
        self._uid_entry_count = collections.defaultdict(int)
        self._entry_uid = dict()
        self.lock = RLock()
        self.time_marker_file = '/usr/share/alt-php-xray/flush.latest'

def __call__(self, t_key: Any, cpu_value: int) -> Tuple[bool, int]:
        """
        Perform fault detection
        """
        with self.lock:
            try:
                saved_cpu, _ = self.throttling_stat[t_key]
            except KeyError:
                hitting_limits, throttled_time = False, 0
            else:
                hitting_limits, throttled_time = self._was_throttled_since(
                    saved_cpu, cpu_value)
                self._flush_entry(t_key)
        return hitting_limits, throttled_time

@property
    def timestamp(self) -> int:
        """
        Current Unix timestamp
        """
        return int(time.time())

@property
    def _drop_after(self) -> timedelta:
        """
        Drop after value as datetime.timedelta
        """
        return self.timedelta_in_minutes(drop_after)

@property
    def _check_period(self) -> timedelta:
        """
        Check period value as datetime.timedelta
        """
        return self.timedelta_in_minutes(check_period)

@property
    def should_flush(self) -> bool:
        """
        If throttling_stat should be flushed right now
        regarding check period value
        """
        try:
            with open(self.time_marker_file) as latest_info:
                latest_flush_time = int(latest_info.read().strip())
            return self.current_delta(latest_flush_time) > self._check_period
        except (OSError, ValueError):
            with self.lock:
                self._update_latest_flush()
            return False

@staticmethod
    def timedelta_in_minutes(mins: int) -> timedelta:
        """
        Get timedelta object for given number of minutes
        """
        return timedelta(minutes=mins)

def current_delta(self, time_stamp: int) -> timedelta:
        """
        Calculate timedelta for given time_stamp from current timestamp
        """

def _cast(ts: int) -> datetime:
            """
            Make a datetime object from ts (int)
            """
            return datetime.fromtimestamp(ts)

return _cast(self.timestamp) - _cast(time_stamp)

def save(self, key: Any, cpu: int, uid: Optional[int] = None) -> None:
        """
        Save throttling entry for given key.

:param uid: SO_PEERCRED uid of the connecting peer (when known). A
                    per-UID quota is enforced so one uid's request-init flood
                    cannot saturate the shared cap and starve other tenants.
        """
        stamp = self.timestamp
        with self.lock:
            is_new = key not in self.throttling_stat
            # Per-UID quota: only new keys count against a uid's quota; updating
            # an existing entry never costs a new slot. Dropping a NEW entry for
            # an over-quota uid leaves room in the global cap for other uids.
            if is_new and uid is not None \
                    and self._uid_entry_count.get(uid, 0) >= self._MAX_ENTRIES_PER_UID:
                self.logger.warning('throttling_stat per-uid quota (%d) reached for '
                                    'uid=%s, dropping entry', self._MAX_ENTRIES_PER_UID, uid)
                return
            if is_new and len(self.throttling_stat) >= self._MAX_ENTRIES:
                self.logger.warning('throttling_stat at capacity (%d), dropping entry',
                                    len(self.throttling_stat))
                return
            self.logger.debug('Adding new throttling entry %s: (%i, %i)',
                              key, cpu, stamp)
            self.throttling_stat[key] = (cpu, stamp)
            if is_new:
                # Track ownership so the uid's count can be decremented when the
                # entry is consumed (request-end) or expired (flush).
                self._entry_uid[key] = uid
                if uid is not None:
                    self._uid_entry_count[uid] += 1
            self.logger.debug('Throttling entries total %i',
                              len(self.throttling_stat))

def _flush_entry(self, key: Any) -> None:
        """
        Remove throttling entry by given key
        """
        with self.lock:
            self.logger.debug('Abandon throttling entry %s', key)
            existed = key in self.throttling_stat
            self.throttling_stat.pop(key, 0)
            if existed:
                # release the per-UID slot held by this entry
                uid = self._entry_uid.pop(key, None)
                if uid is not None and self._uid_entry_count.get(uid, 0) > 0:
                    self._uid_entry_count[uid] -= 1
                    if self._uid_entry_count[uid] <= 0:
                        del self._uid_entry_count[uid]
            self.logger.debug('Throttling entries total %i',
                              len(self.throttling_stat))

def _was_throttled_since(self,
                             initial_value: int,
                             current_value: int) -> Tuple[bool, int]:
        """
        Check current throttling time of given lve_id vs given initial_value
        Return: tuple(fact of throttling, throttling time in ms)
        """
        # initial values of throttling time are in nanoseconds!
        spent_time = (current_value - initial_value) // 1000000  # cast to ms
        throttling_fact = spent_time > throttling_threshold
        self.logger.debug('Throttling checked with %i: %s, %i',
                          throttling_threshold,
                          throttling_fact, spent_time)
        return throttling_fact, spent_time if spent_time > 0 else 0

def flush(self) -> None:
        """
        Clear throttling stat dict
        """
        if self.should_flush:
            with self.lock:
                self.logger.debug('Flush started, total entries %i',
                                  len(self.throttling_stat))
                for key in self._expired_entries():
                    self._flush_entry(key)
                self._update_latest_flush()
                self.logger.debug('Flush finished, total entries %i',
                                  len(self.throttling_stat))

def _update_latest_flush(self) -> None:
        """
        Write current timestamp into time marker file
        """
        try:
            with open(self.time_marker_file, 'w') as latest_info:
                latest_info.write(str(self.timestamp))
        except OSError:
            self.logger.error('Failed to save timestamp of latest flush')

def _expired_entries(self) -> list:
        """
        Collect expired throttling stat entries keys
        regarding the drop after value
        """
        expired = list()
        for key, value in self.throttling_stat.items():
            _, timestamp = value
            _delta = self.current_delta(timestamp)
            if _delta > self._drop_after:
                self.logger.debug('Expired entry detected %i: %s elapsed',
                                  key, _delta)
                expired.append(key)
        return expired

cmd: