async def as_completed(coros):
    """
    Async generator that yields completed Tasks as they are completed.

    Args:
        coros (iterable): An iterable of coroutine objects or asyncio Tasks.

    Yields:
        asyncio.Task: A Task object that has completed its execution.

    Examples:
        >>> async def main():
        ...     async for task in as_completed([coro1(), coro2(), coro3()]):
        ...         result = task.result()
        ...         print(f'Task completed with result: {result}')

        >>> asyncio.run(main())
    """
    tasks = {coro if isinstance(coro, asyncio.Task) else asyncio.create_task(coro): coro for coro in coros}
    while tasks:
        done, _ = await asyncio.wait(tasks.keys(), return_when=asyncio.FIRST_COMPLETED)
        for task in done:
            tasks.pop(task)
            yield task

def backup_file(filename, max_backups=10):
    """
    Renames a file by appending an iteration number as a backup. Recursively renames
    files up to a specified maximum number of backups.

    Args:
        filename (str or pathlib.Path): The file to backup.
        max_backups (int, optional): The maximum number of backups to keep. Defaults to 10.

    Returns:
        pathlib.Path: The new backup filepath.

    Examples:
        >>> backup_file("/tmp/test.txt")
        PosixPath("/tmp/test.0.txt")
        >>> backup_file("/tmp/test.0.txt")
        PosixPath("/tmp/test.1.txt")
        >>> backup_file("/tmp/test.1.txt")
        PosixPath("/tmp/test.2.txt")
    """
    filename = Path(filename).resolve()
    suffixes = [s.strip(".") for s in filename.suffixes]
    iteration = 1
    with suppress(Exception):
        iteration = min(max_backups - 1, max(0, int(suffixes[0]))) + 1
        suffixes = suffixes[1:]
    stem = filename.stem.split(".")[0]
    destination = filename.parent / f"{stem}.{iteration}.{'.'.join(suffixes)}"
    if destination.exists() and iteration < max_backups:
        backup_file(destination)
    if filename.exists():
        filename.rename(destination)
    return destination

def best_http_status(code1, code2):
    """
    Determine the better HTTP status code between two given codes.

    The 'better' status code is considered based on typical usage and priority in HTTP communication.
    Lower codes are generally better than higher codes. Within the same class (e.g., 2xx), a lower code is better.
    Between different classes, the order of preference is 2xx > 3xx > 1xx > 4xx > 5xx.

    Args:
        code1 (int): The first HTTP status code.
        code2 (int): The second HTTP status code.

    Returns:
        int: The better HTTP status code between the two provided codes.

    Examples:
        >>> better_http_status(200, 404)
        200
        >>> better_http_status(500, 400)
        400
        >>> better_http_status(301, 302)
        301
    """

    # Classify the codes into their respective categories (1xx, 2xx, 3xx, 4xx, 5xx)
    def classify_code(code):
        return int(code) // 100

    class1 = classify_code(code1)
    class2 = classify_code(code2)

    # Priority order for classes
    priority_order = {2: 1, 3: 2, 1: 3, 4: 4, 5: 5}

    # Compare based on class priority
    p1 = priority_order.get(class1, 10)
    p2 = priority_order.get(class2, 10)
    if p1 != p2:
        return code1 if p1 < p2 else code2

    # If in the same class, the lower code is better
    return min(code1, code2)

def bytes_to_human(_bytes):
    """Convert a bytes size to a human-readable string.

    This function converts a numeric bytes value into a human-readable string format, complete
    with the appropriate unit symbol (B, KB, MB, GB, etc.).

    Args:
        _bytes (int): The number of bytes to convert.

    Returns:
        str: A string representing the number of bytes in a more readable format, rounded to two
             decimal places.

    Examples:
        >>> bytes_to_human(1234129384)
        '1.15GB'
    """
    sizes = ["B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB"]
    units = {}
    for count, size in enumerate(sizes):
        units[size] = pow(1024, count)
    for size in sizes:
        if abs(_bytes) < 1024.0:
            if size == sizes[0]:
                _bytes = str(int(_bytes))
            else:
                _bytes = f"{_bytes:.2f}"
            return f"{_bytes}{size}"
        _bytes /= 1024
    raise ValueError(f'Unable to convert "{_bytes}" to human filesize')

def calculate_entropy(data):
    """Calculate the Shannon entropy of a byte sequence"""
    if not data:
        return 0
    frequency = {}
    for byte in data:
        if byte in frequency:
            frequency[byte] += 1
        else:
            frequency[byte] = 1
    data_len = len(data)
    entropy = -sum((count / data_len) * math.log2(count / data_len) for count in frequency.values())
    return entropy

def can_sudo_without_password():
    """Check if the current user has passwordless sudo access.

    This function checks whether the current user can use sudo without entering a password.
    It runs a command with sudo and checks the return code to determine this.

    Returns:
        bool: True if the current user can use sudo without a password, False otherwise.

    Examples:
        >>> can_sudo_without_password()
        True
    """
    if os.geteuid() != 0:
        env = dict(os.environ)
        env["SUDO_ASKPASS"] = "/bin/false"
        try:
            sp.run(["sudo", "-K"], stderr=sp.DEVNULL, stdout=sp.DEVNULL, check=True, env=env)
            sp.run(["sudo", "-An", "/bin/true"], stderr=sp.DEVNULL, stdout=sp.DEVNULL, check=True, env=env)
        except sp.CalledProcessError:
            return False
    return True

async def cancel_tasks(tasks, ignore_errors=True):
    """
    Asynchronously cancels a list of asyncio tasks.

    Args:
        tasks (list[Task]): A list of asyncio Task objects to cancel.
        ignore_errors (bool, optional): Whether to ignore errors other than asyncio.CancelledError. Defaults to True.

    Examples:
        >>> async def main():
        ...     task1 = asyncio.create_task(async_function1())
        ...     task2 = asyncio.create_task(async_function2())
        ...     await cancel_tasks([task1, task2])
        ...
        >>> asyncio.run(main())

    Note:
        This function will not cancel the current task that it is called from.
    """
    current_task = asyncio.current_task()
    tasks = [t for t in tasks if t != current_task]
    for task in tasks:
        # log.debug(f"Cancelling task: {task}")
        task.cancel()
    if ignore_errors:
        for task in tasks:
            try:
                await task
            except BaseException as e:
                if not isinstance(e, asyncio.CancelledError):
                    import traceback

                    log.trace(traceback.format_exc())

def cancel_tasks_sync(tasks):
    """
    Synchronously cancels a list of asyncio tasks.

    Args:
        tasks (list[Task]): A list of asyncio Task objects to cancel.

    Examples:
        >>> loop = asyncio.get_event_loop()
        >>> task1 = loop.create_task(some_async_function1())
        >>> task2 = loop.create_task(some_async_function2())
        >>> cancel_tasks_sync([task1, task2])

    Note:
        This function will not cancel the current task from which it is called.
    """
    current_task = asyncio.current_task()
    for task in tasks:
        if task != current_task:
            # log.debug(f"Cancelling task: {task}")
            task.cancel()

def chain_lists(
    l,
    try_files=False,
    msg=None,
    remove_blank=True,
    validate=False,
    validate_chars='<>:"/\\|?*)',
):
    """Chains together list elements, allowing for entries separated by commas.

    This function takes a list `l` and flattens it by splitting its entries on commas.
    It also allows you to optionally open entries as files and add their contents to the list.

    The order of entries is preserved, and deduplication is performed automatically.

    Args:
        l (list): The list of strings to chain together.
        try_files (bool, optional): Whether to try to open entries as files. Defaults to False.
        msg (str, optional): An optional message to log when reading from a file. Defaults to None.
        remove_blank (bool, optional): Whether to remove blank entries from the list. Defaults to True.
        validate (bool, optional): Whether to perform validation for undesirable characters. Defaults to False.
        validate_chars (str, optional): When performing validation, what additional set of characters to block (blocks non-printable ascii automatically). Defaults to '<>:"/\\|?*)'

    Returns:
        list: The list of chained elements.

    Raises:
        ValueError: If the input string contains invalid characters, when enabled (off by default).

    Examples:
        >>> chain_lists(["a", "b,c,d"])
        ['a', 'b', 'c', 'd']

        >>> chain_lists(["a,file.txt", "c,d"], try_files=True)
        ['a', 'f_line1', 'f_line2', 'f_line3', 'c', 'd']
    """
    if isinstance(l, str):
        l = [l]
    final_list = {}
    for entry in l:
        for s in split_regex.split(entry):
            f = s.strip()
            if validate:
                if any((c in validate_chars) or (ord(c) < 32 and c != " ") for c in f):
                    raise ValueError(f"Invalid character in string: {f}")
            f_path = Path(f).resolve()
            if try_files and f_path.is_file():
                if msg is not None:
                    new_msg = str(msg).format(filename=f_path)
                    log.info(new_msg)
                for line in str_or_file(f):
                    final_list[line] = None
            else:
                final_list[f] = None

    ret = list(final_list)
    if remove_blank:
        ret = [r for r in ret if r]
    return ret

def charset(p):
    """
    Determine the character set of the given string based on the types of characters it contains.

    Args:
        p (str): The string whose character set is to be determined.

    Returns:
        int: A bitmask representing the types of characters present in the string.
            - CHAR_LOWER = 1: Lowercase alphabets
            - CHAR_UPPER = 2: Uppercase alphabets
            - CHAR_DIGIT = 4: Digits
            - CHAR_SYMBOL = 8: Symbols/Special characters

    Examples:
        >>> charset('abc')
        1

        >>> charset('abcABC')
        3

        >>> charset('abc123')
        5

        >>> charset('!abc123')
        13
    """
    ret = 0
    for c in p:
        if c.islower():
            ret |= CHAR_LOWER
        elif c.isupper():
            ret |= CHAR_UPPER
        elif c.isnumeric():
            ret |= CHAR_DIGIT
        else:
            ret |= CHAR_SYMBOL
    return ret

def clean_dict(d, *key_names, fuzzy=False, exclude_keys=None, _prev_key=None):
    """
    Recursively clean unwanted keys from a dictionary.
    Useful for removing secrets from a config.

    Args:
        d (dict): The input dictionary.
        *key_names: Names of keys to remove.
        fuzzy (bool): Whether to perform fuzzy matching on keys.
        exclude_keys (list, None): List of keys to be excluded from removal.
        _prev_key (str, None): For internal recursive use; the previous key in the hierarchy.

    Returns:
        dict: A dictionary cleaned of the keys specified in key_names.

    """
    if exclude_keys is None:
        exclude_keys = []
    if isinstance(exclude_keys, str):
        exclude_keys = [exclude_keys]
    d = copy.deepcopy(d)
    if isinstance(d, dict):
        for key, val in list(d.items()):
            if key in key_names or (fuzzy and any(k in key for k in key_names)):
                if _prev_key not in exclude_keys:
                    d.pop(key)
                    continue
            d[key] = clean_dict(val, *key_names, fuzzy=fuzzy, _prev_key=key, exclude_keys=exclude_keys)
    return d

def clean_dns_record(record):
    """
    Cleans and formats a given DNS record for further processing.

    This static method converts the DNS record to text format if it's not already a string.
    It also removes any trailing dots and converts the record to lowercase.

    Args:
        record (str or dns.rdata.Rdata): The DNS record to clean.

    Returns:
        str: The cleaned and formatted DNS record.

    Examples:
        >>> clean_dns_record('www.evilcorp.com.')
        'www.evilcorp.com'

        >>> from dns.rrset import from_text
        >>> record = from_text('www.evilcorp.com', 3600, 'IN', 'A', '1.2.3.4')[0]
        >>> clean_dns_record(record)
        '1.2.3.4'
    """
    if not isinstance(record, str):
        record = str(record.to_text())
    return str(record).rstrip(".").lower()

def clean_old(d, keep=10, filter=lambda x: True, key=latest_mtime, reverse=True, raise_error=False):
    """Clean up old files and directories within a given directory based on various filtering and sorting options.

    This function removes the oldest files and directories in the provided directory 'd' that exceed a specified
    threshold ('keep'). The items to be deleted can be filtered using a lambda function 'filter', and they are
    sorted by a key function, defaulting to latest modification time.

    Args:
        d (str or Path): The directory path to clean up.
        keep (int): The number of items to keep. Ones beyond this count will be removed.
        filter (Callable): A lambda function for filtering which files or directories to consider.
                           Defaults to a lambda function that returns True for all.
        key (Callable): A function to sort the files and directories. Defaults to latest modification time.
        reverse (bool): Whether to reverse the order of sorted items before removing. Defaults to True.
        raise_error (bool): Whether to raise an error if directory deletion fails. Defaults to False.

    Examples:
        >>> clean_old("~/.bbot/scans", filter=lambda x: x.is_dir() and scan_name_regex.match(x.name))
    """
    d = Path(d)
    if not d.is_dir():
        return
    paths = [x for x in d.iterdir() if filter(x)]
    paths.sort(key=key, reverse=reverse)
    for path in paths[keep:]:
        try:
            log.debug(f"Removing {path}")
            rm_rf(path)
        except Exception as e:
            msg = f"Failed to delete directory: {path}, {e}"
            if raise_error:
                raise errors.DirectoryDeletionError()
            log.warning(msg)

def closest_match(s, choices, n=1, cutoff=0.0):
    """Finds the closest matching strings from a list of choices based on a given string.

    This function uses the difflib library to find the closest matches to a given string `s` from a list of `choices`.
    It can return either the single best match or a list of the top `n` best matches.

    Args:
        s (str): The string for which to find the closest match.
        choices (list): A list of strings to compare against.
        n (int, optional): The number of best matches to return. Defaults to 1.
        cutoff (float, optional): A float value that defines the similarity threshold. Strings with similarity below this value are not considered. Defaults to 0.0.

    Returns:
        str or list: Either the closest matching string or a list of the `n` closest matching strings.

    Examples:
        >>> closest_match("asdf", ["asd", "fds"])
        'asd'
        >>> closest_match("asdf", ["asd", "fds", "asdff"], n=3)
        ['asdff', 'asd', 'fds']
    """
    import difflib

    matches = difflib.get_close_matches(s, choices, n=n, cutoff=cutoff)
    if not choices or not matches:
        return
    if n == 1:
        return matches[0]
    return matches

def cloudcheck(ip):
    """
    Check whether an IP address belongs to a cloud provider and returns the provider name, type, and subnet.

    Args:
        ip (str): The IP address to check.

    Returns:
        tuple: A tuple containing provider name (str), provider type (str), and subnet (IPv4Network).

    Examples:
        >>> cloudcheck("168.62.20.37")
        ('Azure', 'cloud', IPv4Network('168.62.0.0/19'))
    """
    import cloudcheck as _cloudcheck

    return _cloudcheck.check(ip)

def cpu_architecture():
    """Return the CPU architecture of the current system.

    This function fetches and returns the architecture type of the CPU where the code is being executed.
    It maps common identifiers like "x86_64" to more general types like "amd64".

    Returns:
        str: A string representing the CPU architecture, such as "amd64", "armv7", or "arm64".

    Examples:
        >>> cpu_architecture()
        'amd64'
    """
    import platform

    uname = platform.uname()
    arch = uname.machine.lower()
    if arch.startswith("aarch"):
        return "arm64"
    elif arch == "x86_64":
        return "amd64"
    return arch

def delete_file(path):
    """Deletes a file at the given path.

    Args:
        path (str or Path): The path to the file to be deleted.

    Note:
        This function suppresses all exceptions to ensure that the program continues running even if the file could not be deleted.

    Examples:
        >>> delete_file("/tmp/test/file1.txt")
    """
    with suppress(Exception):
        Path(path).unlink(missing_ok=True)

def domain_parents(d, include_self=False):
    """
    Generate a list of parent domains for a given domain string.

    This function takes an input string `d` and generates a list of parent domains in decreasing order of specificity.
    If `include_self` is set to True, the list will also include the input domain if it is not a top-level domain.

    Args:
        d (str): The input string representing a domain or subdomain.
        include_self (bool, optional): Whether to include the input domain itself. Defaults to False.

    Yields:
        str: Parent domains of the input string in decreasing order of specificity.

    Examples:
        >>> list(domain_parents("test.www.evilcorp.co.uk"))
        ["www.evilcorp.co.uk", "evilcorp.co.uk"]

    Notes:
        - Port, if present in input, is preserved in the output.
    """

    parent = str(d)
    if include_self and not is_domain(parent):
        yield parent
    while 1:
        parent = parent_domain(parent)
        if is_subdomain(parent):
            yield parent
            continue
        elif is_domain(parent):
            yield parent
        break

def domain_stem(domain):
    """
    Returns an abbreviated representation of the hostname by removing the TLD (Top-Level Domain).

    Args:
        domain (str): The full domain name to be abbreviated.

    Returns:
        str: An abbreviated domain string without the TLD.

    Examples:
        >>> domain_stem("www.evilcorp.com")
        "www.evilcorp"

    Notes:
        - Utilizes the `tldextract` function for domain parsing.
    """
    parsed = tldextract(str(domain))
    return ".".join(parsed.subdomain.split(".") + parsed.domain.split(".")).strip(".")

async def execute_sync_or_async(callback, *args, **kwargs):
    """
    Execute a function or coroutine, handling either synchronous or asynchronous invocation.

    Args:
        callback (Union[Callable, Coroutine]): The function or coroutine to execute.
        *args: Variable-length argument list to pass to the callback.
        **kwargs: Arbitrary keyword arguments to pass to the callback.

    Returns:
        Any: The return value from the executed function or coroutine.

    Examples:
        >>> async def foo_async(x):
        ...     return x + 1
        >>> def foo_sync(x):
        ...     return x + 1

        >>> asyncio.run(execute_sync_or_async(foo_async, 1))
        2

        >>> asyncio.run(execute_sync_or_async(foo_sync, 1))
        2
    """
    if is_async_function(callback):
        return await callback(*args, **kwargs)
    else:
        return callback(*args, **kwargs)

def extract_emails(s):
    """
    Extract email addresses from a body of text

    This function takes in a string and yields all email addresses found in it.
    The emails are converted to lower case before yielding. It utilizes
    regular expressions for email pattern matching.

    Args:
        s (str): The input string from which to extract email addresses.

    Yields:
        str: Yields email addresses found in the input string, in lower case.

    Examples:
        >>> list(extract_emails("Contact us at info@evilcorp.com and support@evilcorp.com"))
        ['info@evilcorp.com', 'support@evilcorp.com']
    """
    for email in bbot_regexes.email_regex.findall(smart_decode(s)):
        yield email.lower()

def extract_host(s):
    """
    Attempts to find and extract the host portion of a string.

    Args:
        s (str): The string from which to extract the host.

    Returns:
        tuple: A tuple containing three strings:
               (hostname (None if not found), string_before_hostname, string_after_hostname).

    Examples:
        >>> extract_host("evilcorp.com:80")
        ("evilcorp.com", "", ":80")

        >>> extract_host("http://evilcorp.com:80/asdf.php?a=b")
        ("evilcorp.com", "http://", ":80/asdf.php?a=b")

        >>> extract_host("bob@evilcorp.com")
        ("evilcorp.com", "bob@", "")

        >>> extract_host("[dead::beef]:22")
        ("dead::beef", "[", "]:22")

        >>> extract_host("ftp://username:password@my-ftp.com/my-file.csv")
        (
            "my-ftp.com",
            "ftp://username:password@",
            "/my-file.csv",
        )
    """
    s = smart_decode(s)
    match = bbot_regexes.extract_host_regex.search(s)

    if match:
        hostname = match.group(1)
        before = s[: match.start(1)]
        after = s[match.end(1) :]
        host, port = split_host_port(hostname)
        netloc = make_netloc(host, port)
        if netloc != hostname:
            # invalid host / port
            return (None, s, "")
        if host is not None:
            if port is not None:
                after = f":{port}{after}"
            if is_ip(host, version=6) and hostname.startswith("["):
                before = f"{before}["
                after = f"]{after}"
            hostname = str(host)
        return (hostname, before, after)

    return (None, s, "")

def extract_params_json(json_data, compare_mode="getparam"):
    """
    Extracts key-value pairs from a JSON object and returns them as a set of tuples. Used by the `paramminer_headers` module.

    Args:
        json_data (str): JSON-formatted string containing key-value pairs.

    Returns:
        set: A set of tuples containing the keys and their corresponding values present in the JSON object.

    Raises:
        Returns an empty set if JSONDecodeError occurs.

    Examples:
        >>> extract_params_json('{"a": 1, "b": {"c": 2}}')
        {('a', 1), ('b', {'c': 2}), ('c', 2)}
    """
    try:
        data = json.loads(json_data)
    except json.JSONDecodeError:
        return set()

    key_value_pairs = set()
    stack = [(data, "")]

    while stack:
        current_data, path = stack.pop()
        if isinstance(current_data, dict):
            for key, value in current_data.items():
                full_key = f"{path}.{key}" if path else key
                if isinstance(value, dict):
                    stack.append((value, full_key))
                elif isinstance(value, list):
                    stack.append((value, full_key))
                else:
                    if validate_parameter(full_key, compare_mode):
                        key_value_pairs.add((full_key, value))
        elif isinstance(current_data, list):
            for item in current_data:
                if isinstance(item, (dict, list)):
                    stack.append((item, path))
    return key_value_pairs

def extract_params_xml(xml_data, compare_mode="getparam"):
    """
    Extracts tags and their text values from an XML object and returns them as a set of tuples.

    Args:
        xml_data (str): XML-formatted string containing elements.

    Returns:
        set: A set of tuples containing the tags and their corresponding sanitized text values present in the XML object.

    Raises:
        Returns an empty set if ParseError occurs.

    Examples:
        >>> extract_params_xml('<root><child1><child2>value</child2></child1></root>')
        {('root', None), ('child1', None), ('child2', 'value')}
    """
    import xml.etree.ElementTree as ET

    try:
        root = ET.fromstring(xml_data)
    except ET.ParseError:
        return set()

    tag_value_pairs = set()
    stack = [root]

    while stack:
        current_element = stack.pop()
        if validate_parameter(current_element.tag, compare_mode):
            # Sanitize the text value
            text_value = current_element.text.strip() if current_element.text else None
            sanitized_value = quote(text_value, safe="") if text_value else None
            tag_value_pairs.add((current_element.tag, sanitized_value))
        for child in current_element:
            stack.append(child)
    return tag_value_pairs

def extract_words(data, acronyms=True, wordninja=True, model=None, max_length=100, word_regexes=None):
    """Intelligently extracts words from given data.

    This function uses regular expressions and optionally wordninja to extract words
    from a given text string. Thanks to wordninja it can handle concatenated words intelligently.

    Args:
        data (str): The data from which words are to be extracted.
        acronyms (bool, optional): Whether to include acronyms. Defaults to True.
        wordninja (bool, optional): Whether to use the wordninja library to split concatenated words. Defaults to True.
        model (object, optional): A custom wordninja model for special types of data such as DNS names.
        max_length (int, optional): Maximum length for a word to be included. Defaults to 100.
        word_regexes (list, optional): A list of compiled regular expression objects for word extraction. Defaults to None.

    Returns:
        set: A set of extracted words.

    Examples:
        >>> extract_words('blacklanternsecurity')
        {'black', 'lantern', 'security', 'bls', 'blacklanternsecurity'}
    """
    import wordninja as _wordninja

    if word_regexes is None:
        word_regexes = bbot_regexes.word_regexes
    words = set()
    data = smart_decode(data)
    for r in word_regexes:
        for word in set(r.findall(data)):
            # blacklanternsecurity
            if len(word) <= max_length:
                words.add(word)

    # blacklanternsecurity --> ['black', 'lantern', 'security']
    # max_slice_length = 3
    for word in list(words):
        if wordninja:
            if model is None:
                model = _wordninja
            subwords = model.split(word)
            for subword in subwords:
                words.add(subword)
        # this section generates compound words
        # it is interesting but currently disabled the quality of its output doesn't quite justify its quantity
        # blacklanternsecurity --> ['black', 'lantern', 'security', 'blacklantern', 'lanternsecurity']
        # for s, e in combinations(range(len(subwords) + 1), 2):
        #    if e - s <= max_slice_length:
        #        subword_slice = "".join(subwords[s:e])
        #        words.add(subword_slice)
        # blacklanternsecurity --> bls
        if acronyms:
            if len(subwords) > 1:
                words.add("".join([c[0] for c in subwords if len(c) > 0]))

    return words

def filesize(f):
    """Get the file size of a given file.

    This function takes a file path as an argument and returns its size in bytes. If the path
    does not point to a file, the function returns 0.

    Args:
        f (str or Path): The file path for which to get the size.

    Returns:
        int: The size of the file in bytes, or 0 if the path does not point to a file.

    Examples:
        >>> filesize("/path/to/file.txt")
        1024
    """
    f = Path(f)
    if f.is_file():
        return f.stat().st_size
    return 0

def filter_dict(d, *key_names, fuzzy=False, exclude_keys=None, _prev_key=None):
    """
    Recursively filter a dictionary based on key names.

    Args:
        d (dict): The input dictionary.
        *key_names: Names of keys to filter for.
        fuzzy (bool): Whether to perform fuzzy matching on keys.
        exclude_keys (list, None): List of keys to be excluded from the final dict.
        _prev_key (str, None): For internal recursive use; the previous key in the hierarchy.

    Returns:
        dict: A dictionary containing only the keys specified in key_names.

    Examples:
        >>> filter_dict({"key1": "test", "key2": "asdf"}, "key2")
        {"key2": "asdf"}
        >>> filter_dict({"key1": "test", "key2": {"key3": "asdf"}}, "key1", "key3", exclude_keys="key2")
        {'key1': 'test'}
    """
    if exclude_keys is None:
        exclude_keys = []
    if isinstance(exclude_keys, str):
        exclude_keys = [exclude_keys]
    ret = {}
    if isinstance(d, dict):
        for key in d:
            if key in key_names or (fuzzy and any(k in key for k in key_names)):
                if not any(k in exclude_keys for k in [key, _prev_key]):
                    ret[key] = copy.deepcopy(d[key])
            elif isinstance(d[key], list) or isinstance(d[key], dict):
                child = filter_dict(d[key], *key_names, fuzzy=fuzzy, _prev_key=key, exclude_keys=exclude_keys)
                if child:
                    ret[key] = child
    return ret

def gen_numbers(n, padding=2):
    """Generates numbers with variable padding and returns them as a set of strings.

    Args:
        n (int): The upper limit of numbers to generate, exclusive.
        padding (int, optional): The maximum number of digits to pad the numbers with. Defaults to 2.

    Returns:
        set: A set of string representations of numbers with varying degrees of padding.

    Examples:
        >>> gen_numbers(5)
        {'0', '00', '01', '02', '03', '04', '1', '2', '3', '4'}

        >>> gen_numbers(3, padding=3)
        {'0', '00', '000', '001', '002', '01', '02', '1', '2'}

        >>> gen_numbers(5, padding=1)
        {'0', '1', '2', '3', '4'}
    """
    results = set()
    for i in range(n):
        for p in range(1, padding + 1):
            results.add(str(i).zfill(p))
    return results

def get_closest_match(s, choices, msg=None):
    """Finds the closest match from a list of choices for a given string.

    This function is particularly useful for CLI applications where you want to validate flags or modules.

    Args:
        s (str): The string for which to find the closest match.
        choices (list): A list of strings to compare against.
        msg (str, optional): Additional message to prepend in the warning message. Defaults to None.
        loglevel (str, optional): The log level to use for the warning message. Defaults to "HUGEWARNING".
        exitcode (int, optional): The exit code to use when exiting the program. Defaults to 2.

    Examples:
        >>> get_closest_match("some_module", ["some_mod", "some_other_mod"], msg="module")
        # Output: Could not find module "some_module". Did you mean "some_mod"?
    """
    if msg is None:
        msg = ""
    else:
        msg += " "
    closest = closest_match(s, choices)
    return f'Could not find {msg}"{s}". Did you mean "{closest}"?'

def get_exception_chain(e):
    """
    Retrieves the full chain of exceptions leading to the given exception.

    Args:
        e (BaseException): The exception for which to get the chain.

    Returns:
        list[BaseException]: List of exceptions in the chain, from the given exception back to the root cause.

    Examples:
        >>> try:
        ...     raise ValueError("This is a value error")
        ... except ValueError as e:
        ...     exc_chain = get_exception_chain(e)
        ...     for exc in exc_chain:
        ...         print(exc)
        This is a value error
    """
    exception_chain = []
    current_exception = e
    while current_exception is not None:
        exception_chain.append(current_exception)
        current_exception = getattr(current_exception, "__context__", None)
    return exception_chain

def get_file_extension(s):
    """
    Extracts the file extension from a given string representing a URL or file path.

    Args:
        s (str): The string from which to extract the file extension.

    Returns:
        str: The file extension, or an empty string if no extension is found.

    Examples:
        >>> get_file_extension("https://evilcorp.com/api/test.php")
        "php"
        >>> get_file_extension("/etc/test.conf")
        "conf"
        >>> get_file_extension("/etc/passwd")
        ""
    """
    s = str(s).lower().strip()
    rightmost_section = s.rsplit("/", 1)[-1]
    if "." in rightmost_section:
        extension = rightmost_section.rsplit(".", 1)[-1]
        return extension
    return ""

def get_get_params(url):
    """
    Extract the query parameters from the given URL as a dictionary.

    Args:
        url (Union[str, ParseResult]): The URL from which to extract query parameters.

    Returns:
        Dict[str, List[str]]: A dictionary containing the query parameters and their values.

    Examples:
        >>> get_get_params('https://www.evilcorp.com?foo=1&bar=2')
        {'foo': ['1'], 'bar': ['2']}

        >>> get_get_params('https://www.evilcorp.com?foo=1&foo=2')
        {'foo': ['1', '2']}
    """
    parsed = parse_url(url)
    return dict(parse_qs(parsed.query))

def get_keys_in_dot_syntax(config):
    """Retrieve all keys in an OmegaConf configuration in dot notation.

    This function converts an OmegaConf configuration into a list of keys
    represented in dot notation.

    Args:
        config (DictConfig): The OmegaConf configuration object.

    Returns:
        List[str]: A list of keys in dot notation.

    Examples:
        >>> config = OmegaConf.create({
        ...     "web": {
        ...         "test": True
        ...     },
        ...     "db": {
        ...         "host": "localhost",
        ...         "port": 5432
        ...     }
        ... })
        >>> get_keys_in_dot_syntax(config)
        ['web.test', 'db.host', 'db.port']
    """
    from omegaconf import OmegaConf

    container = OmegaConf.to_container(config, resolve=True)
    keys = []

    def recursive_keys(d, parent_key=""):
        for k, v in d.items():
            full_key = f"{parent_key}.{k}" if parent_key else k
            if isinstance(v, dict):
                recursive_keys(v, full_key)
            else:
                keys.append(full_key)

    recursive_keys(container)
    return keys

def get_size(obj, max_depth=5, seen=None):
    """
    Roughly estimate the memory footprint of a Python object using recursion.

    Parameters:
        obj (any): The object whose size is to be determined.
        max_depth (int, optional): Maximum depth to which nested objects will be inspected. Defaults to 5.
        seen (set, optional): Objects that have already been accounted for, to avoid loops.

    Returns:
        int: Approximate memory footprint of the object in bytes.

    Examples:
        >>> get_size(my_list)
        4200

        >>> get_size(my_dict, max_depth=3)
        8400
    """
    from collections.abc import Mapping

    # If seen is not provided, initialize an empty set
    if seen is None:
        seen = set()
    # Get the id of the object
    obj_id = id(obj)
    # Decrease the maximum depth for the next recursion
    new_max_depth = max_depth - 1
    # If the object has already been seen or we've reached the maximum recursion depth, return 0
    if obj_id in seen or new_max_depth <= 0:
        return 0
    # Get the size of the object
    size = sys.getsizeof(obj)
    # Add the object's id to the set of seen objects
    seen.add(obj_id)
    # If the object has a __dict__ attribute, we want to measure its size
    if hasattr(obj, "__dict__"):
        # Iterate over the Method Resolution Order (MRO) of the class of the object
        for cls in obj.__class__.__mro__:
            # If the class's __dict__ contains a __dict__ key
            if "__dict__" in cls.__dict__:
                for k, v in obj.__dict__.items():
                    size += get_size(k, new_max_depth, seen)
                    size += get_size(v, new_max_depth, seen)
                break
    # If the object is a mapping (like a dictionary), we want to measure the size of its items
    if isinstance(obj, Mapping):
        with suppress(StopIteration):
            k, v = next(iter(obj.items()))
            size += (get_size(k, new_max_depth, seen) + get_size(v, new_max_depth, seen)) * len(obj)
    # If the object is a container (like a list or tuple) but not a string or bytes-like object
    elif isinstance(obj, (list, tuple, set)):
        with suppress(StopIteration):
            size += get_size(next(iter(obj)), new_max_depth, seen) * len(obj)
    # If the object has __slots__, we want to measure the size of the attributes in __slots__
    if hasattr(obj, "__slots__"):
        size += sum(get_size(getattr(obj, s), new_max_depth, seen) for s in obj.__slots__ if hasattr(obj, s))
    return size

def get_traceback_details(e):
    """
    Retrieves detailed information from the traceback of an exception.

    Args:
        e (BaseException): The exception for which to get traceback details.

    Returns:
        tuple: A tuple containing filename (str), line number (int), and function name (str) where the exception was raised.

    Examples:
        >>> try:
        ...     raise ValueError("This is a value error")
        ... except ValueError as e:
        ...     filename, lineno, funcname = get_traceback_details(e)
        ...     print(f"File: {filename}, Line: {lineno}, Function: {funcname}")
        File: <stdin>, Line: 2, Function: <module>
    """
    import traceback

    tb = traceback.extract_tb(e.__traceback__)
    last_frame = tb[-1]  # Get the last frame in the traceback (the one where the exception was raised)
    filename = last_frame.filename
    lineno = last_frame.lineno
    funcname = last_frame.name
    return filename, lineno, funcname

def grouper(iterable, n):
    """
    Grouper groups an iterable into chunks of a given size.

    Args:
        iterable (iterable): The iterable to be chunked.
        n (int): The size of each chunk.

    Returns:
        iterator: An iterator that produces lists of elements from the original iterable, each of length `n` or less.

    Examples:
        >>> list(grouper('ABCDEFG', 3))
        [['A', 'B', 'C'], ['D', 'E', 'F'], ['G']]
    """
    from itertools import islice

    iterable = iter(iterable)
    return iter(lambda: list(islice(iterable, n)), [])

def hash_url(url):
    """
    Hashes a URL for the purpose of cleaning or collapsing similar URLs.

    Args:
        url (str): The URL to be hashed.

    Returns:
        int: The hash value of the cleaned URL.

    Examples:
        >>> hash_url('https://www.evilcorp.com')
        -7448777882396416944

        >>> hash_url('https://www.evilcorp.com/page/1')
        -8101275613229735915

        >>> hash_url('https://www.evilcorp.com/page/2')
        -8101275613229735915
    """
    parsed = parse_url(url)
    parsed = parsed._replace(fragment="", query="")
    to_hash = [parsed.netloc]
    for segment in parsed.path.split("/"):
        hash_segment = []
        hash_segment.append(charset(segment))
        hash_segment.append(param_type(segment))
        dot_split = segment.split(".")
        if len(dot_split) > 1:
            hash_segment.append(dot_split[-1])
        else:
            hash_segment.append("")
        to_hash.append(tuple(hash_segment))
    return hash(tuple(to_hash))

def human_timedelta(d):
    """Convert a TimeDelta object into a human-readable string.

    This function takes a datetime.timedelta object and converts it into a string format that
    is easier to read and understand.

    Args:
        d (datetime.timedelta): The TimeDelta object to convert.

    Returns:
        str: A string representation of the TimeDelta object in human-readable form.

    Examples:
        >>> from datetime import datetime
        >>>
        >>> start_time = datetime.now()
        >>> end_time = datetime.now()
        >>> elapsed_time = end_time - start_time
        >>> human_timedelta(elapsed_time)
        '2 hours, 30 minutes, 15 seconds'
    """
    hours, remainder = divmod(d.seconds, 3600)
    minutes, seconds = divmod(remainder, 60)
    result = []
    if hours:
        result.append(f"{hours:,} hour" + ("s" if hours > 1 else ""))
    if minutes:
        result.append(f"{minutes:,} minute" + ("s" if minutes > 1 else ""))
    if seconds:
        result.append(f"{seconds:,} second" + ("s" if seconds > 1 else ""))
    ret = ", ".join(result)
    if not ret:
        ret = "0 seconds"
    return ret

def human_to_bytes(filesize):
    """Convert a human-readable file size string to its bytes equivalent.

    This function takes a human-readable file size string, such as "2.5GB", and converts it
    to its equivalent number of bytes.

    Args:
        filesize (str or int): The human-readable file size string or integer bytes value to convert.

    Returns:
        int: The number of bytes equivalent to the input human-readable file size.

    Raises:
        ValueError: If the input string cannot be converted to bytes.

    Examples:
        >>> human_to_bytes("23.23gb")
        24943022571
    """
    if isinstance(filesize, int):
        return filesize
    sizes = ["B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB"]
    units = {}
    for count, size in enumerate(sizes):
        size_increment = pow(1024, count)
        units[size] = size_increment
        if len(size) == 2:
            units[size[0]] = size_increment
    match = filesize_regex.match(filesize)
    try:
        if match:
            num, size = match.groups()
            size = size.upper()
            size_increment = units[size]
            return int(float(num) * size_increment)
    except KeyError:
        pass
    raise ValueError(f'Unable to convert filesize "{filesize}" to bytes')

def in_exception_chain(e, exc_types):
    """
    Given an Exception and a list of Exception types, returns whether any of the specified types are contained anywhere in the Exception chain.

    Args:
        e (BaseException): The exception to check
        exc_types (list[Exception]): Exception types to consider intentional cancellations. Default is KeyboardInterrupt

    Returns:
        bool: Whether the error is the result of an intentional cancellaion

    Examples:
        >>> try:
        ...     raise ValueError("This is a value error")
        ... except Exception as e:
        ...     if not in_exception_chain(e, (KeyboardInterrupt, asyncio.CancelledError)):
        ...         raise
    """
    return any(isinstance(_, exc_types) for _ in get_exception_chain(e))

def integer_to_ordinal(n):
    """
    Convert an integer to its ordinal representation.

    Args:
        n (int): The integer to convert.

    Returns:
        str: The ordinal representation of the integer.

    Examples:
        >>> integer_to_ordinal(1)
        '1st'
        >>> integer_to_ordinal(2)
        '2nd'
        >>> integer_to_ordinal(3)
        '3rd'
        >>> integer_to_ordinal(11)
        '11th'
        >>> integer_to_ordinal(21)
        '21st'
        >>> integer_to_ordinal(101)
        '101st'
    """
    # Check the last digit
    last_digit = n % 10
    # Check the last two digits for special cases (11th, 12th, 13th)
    last_two_digits = n % 100

    if 10 <= last_two_digits <= 20:
        suffix = "th"
    else:
        if last_digit == 1:
            suffix = "st"
        elif last_digit == 2:
            suffix = "nd"
        elif last_digit == 3:
            suffix = "rd"
        else:
            suffix = "th"

    return f"{n}{suffix}"

def ip_network_parents(i, include_self=False):
    """
    Generates all parent IP networks for a given IP address or network, optionally including the network itself.

    Args:
        i (str or ipaddress.IPv4Network/ipaddress.IPv6Network): The IP address or network to find parents for.
        include_self (bool, optional): Whether to include the network itself in the result. Default is False.

    Yields:
        ipaddress.IPv4Network or ipaddress.IPv6Network: Parent IP networks in descending order of prefix length.

    Examples:
        >>> list(ip_network_parents("192.168.1.1"))
        [ipaddress.IPv4Network('192.168.1.0/31'), ipaddress.IPv4Network('192.168.1.0/30'), ... , ipaddress.IPv4Network('0.0.0.0/0')]

    Notes:
        - Utilizes Python's built-in `ipaddress` module for network operations.
    """
    net = ipaddress.ip_network(i, strict=False)
    for i in range(net.prefixlen - (0 if include_self else 1), -1, -1):
        yield ipaddress.ip_network(f"{net.network_address}/{i}", strict=False)

def is_async_function(f):
    """
    Check if a given function is an asynchronous function.

    Args:
        f (function): The function to check.

    Returns:
        bool: True if the function is asynchronous, False otherwise.

    Examples:
        >>> async def foo():
        ...     pass
        >>> is_async_function(foo)
        True
    """
    import inspect

    return inspect.iscoroutinefunction(f)

def is_dns_name(d):
    """
    Determines if the given string is a valid DNS name.

    Args:
        d (str): The string to be checked.

    Returns:
        bool: True if the string is a valid DNS name, False otherwise.

    Examples:
        >>> is_dns_name('www.example.com')
        True
        >>> is_dns_name('localhost')
        True
        >>> is_dns_name('192.168.1.1')
        False
    """
    if is_ip(d):
        return False
    d = smart_decode(d)
    if bbot_regexes.dns_name_validation_regex.match(d):
        return True
    return False

def is_domain(d):
    """
    Check if the given input represents a domain without subdomains.

    This function takes an input string `d` and returns True if it represents a domain without any subdomains.
    Otherwise, it returns False.

    Args:
        d (str): The input string containing the domain.

    Returns:
        bool: True if the input is a domain without subdomains, False otherwise.

    Examples:
        >>> is_domain("evilcorp.co.uk")
        True

        >>> is_domain("www.evilcorp.co.uk")
        False

    Notes:
        - Port, if present in input, is ignored.
    """
    d, _ = split_host_port(d)
    if is_ip(d):
        return False
    extracted = tldextract(d)
    if extracted.top_domain_under_public_suffix:
        if not extracted.subdomain:
            return True
    else:
        return d.count(".") == 1
    return False