FM / common

/opt/cloudlinux/venv/lib/python3.11/site-packages/wmt/common/

/opt/cloudlinux/venv/lib/python3.11/site-packages/wmt/common

Name	Size	Mode	Actions
__pycache__/	-	0755	rm
config.py	9430	0644	edit dl rm
const.py	1081	0644	edit dl rm
exceptions.py	942	0644	edit dl rm
notification.py	11297	0644	edit dl rm
report.py	6441	0644	edit dl rm
service.py	807	0644	edit dl rm
url_parser.py	2107	0644	edit dl rm
utils.py	7239	0644	edit dl rm
__init__.py	270	0644	edit dl rm

Edit: /opt/cloudlinux/venv/lib/python3.11/site-packages/wmt/common/url_parser.py (2107B)

from urllib.parse import ParseResult, urlparse, urlsplit

def parse(domain: str, scheme: str = 'http') -> str:
    """
    Convert domain name to 'http://www.domain.com' format
    """
    # https://stackoverflow.com/questions/21659044/how-can-i-prepend-http-to-a-url-if-it-doesnt-begin-with-http
    p = urlparse(domain, 'http')
    netloc = p.netloc or p.path
    path = p.path if p.netloc else ''
    p = ParseResult(p.scheme, netloc, path, *p[3:])
    return p.geturl()

def hostname_only(value: str) -> str:
    """
    Reduce a domain or a URL to its bare, lowercased hostname.

The canonical host-comparison helper: report.py matches
    ScrapeResult.website against the domain list with it, and
    ConfigManager.is_domain_ignored() normalises with it before applying
    ignore_list patterns. Both receive a mix of shapes -- get_domains() runs
    every domain through parse(), so it yields 'http://host', while
    ScrapeResult.website holds the post-redirect URL (scheme + trailing slash
    + path, sometimes a port).

Implemented on urlsplit() of a '//'-prefixed value rather than
    urlparse(value, 'http'): for a scheme-less 'host:port' the RFC 3986
    scheme grammar permits dots, so urlparse reads 'sub.example.com' as the
    scheme and '8080' as the path. Forcing netloc parsing also gets port,
    credential and IPv6-bracket stripping from .hostname for free.

Never raises. Returns '' for input with no host, and for input urlsplit()
    rejects -- notably a bracketed fnmatch character class such as
    '*.test[0-9].example.com', which it reads as a malformed IPv6 literal.
    ignore_list entries may legitimately contain those, and a caller treats ''
    as "not a hostname", falling back to verbatim matching. An exception here
    would instead escape is_domain_ignored() and abort the whole scanner
    iteration plus every report path.
    """
    value = (value or '').strip()
    if '://' not in value and not value.startswith('//'):
        value = f'//{value}'
    try:
        return urlsplit(value).hostname or ''
    except ValueError:
        return ''

from urllib.parse import ParseResult, urlparse, urlsplit def parse(domain: str, scheme: str = 'http') -> str: """ Convert domain name to 'http://www.domain.com' format """ # https://stackoverflow.com/questions/21659044/how-can-i-prepend-http-to-a-url-if-it-doesnt-begin-with-http p = urlparse(domain, 'http') netloc = p.netloc or p.path path = p.path if p.netloc else '' p = ParseResult(p.scheme, netloc, path, *p[3:]) return p.geturl() def hostname_only(value: str) -> str: """ Reduce a domain or a URL to its bare, lowercased hostname. The canonical host-comparison helper: report.py matches ScrapeResult.website against the domain list with it, and ConfigManager.is_domain_ignored() normalises with it before applying ignore_list patterns. Both receive a mix of shapes -- get_domains() runs every domain through parse(), so it yields 'http://host', while ScrapeResult.website holds the post-redirect URL (scheme + trailing slash + path, sometimes a port). Implemented on urlsplit() of a '//'-prefixed value rather than urlparse(value, 'http'): for a scheme-less 'host:port' the RFC 3986 scheme grammar permits dots, so urlparse reads 'sub.example.com' as the scheme and '8080' as the path. Forcing netloc parsing also gets port, credential and IPv6-bracket stripping from .hostname for free. Never raises. Returns '' for input with no host, and for input urlsplit() rejects -- notably a bracketed fnmatch character class such as '*.test[0-9].example.com', which it reads as a malformed IPv6 literal. ignore_list entries may legitimately contain those, and a caller treats '' as "not a hostname", falling back to verbatim matching. An exception here would instead escape is_domain_ignored() and abort the whole scanner iteration plus every report path. """ value = (value or '').strip() if '://' not in value and not value.startswith('//'): value = f'//{value}' try: return urlsplit(value).hostname or '' except ValueError: return ''