FM / coverage

/opt/cloudlinux/venv/lib/python3.11/site-packages/coverage/

/opt/cloudlinux/venv/lib/python3.11/site-packages/coverage

Name	Size	Mode	Actions
fullcoverage/	-	0755	rm
htmlfiles/	-	0755	rm
__pycache__/	-	0755	rm
annotate.py	3758	0644	edit dl rm
bytecode.py	713	0644	edit dl rm
cmdline.py	34427	0644	edit dl rm
collector.py	20558	0644	edit dl rm
config.py	21989	0644	edit dl rm
context.py	2483	0644	edit dl rm
control.py	51705	0644	edit dl rm
data.py	7508	0644	edit dl rm
debug.py	17805	0644	edit dl rm
disposition.py	1916	0644	edit dl rm
env.py	6081	0644	edit dl rm
exceptions.py	1362	0644	edit dl rm
execfile.py	12133	0644	edit dl rm
files.py	19374	0644	edit dl rm
html.py	23165	0644	edit dl rm
inorout.py	23900	0644	edit dl rm
jsonreport.py	4754	0644	edit dl rm
lcovreport.py	4939	0644	edit dl rm
misc.py	12172	0644	edit dl rm
multiproc.py	3846	0644	edit dl rm
numbits.py	4669	0644	edit dl rm
parser.py	56720	0644	edit dl rm
phystokens.py	7805	0644	edit dl rm
plugin.py	19525	0644	edit dl rm
plugin_support.py	10351	0644	edit dl rm
py.typed	72	0644	edit dl rm
python.py	8066	0644	edit dl rm
pytracer.py	14420	0644	edit dl rm
report.py	10623	0644	edit dl rm
report_core.py	4068	0644	edit dl rm
results.py	13384	0644	edit dl rm
sqldata.py	51295	0644	edit dl rm
templite.py	10952	0644	edit dl rm
tomlconfig.py	7569	0644	edit dl rm
tracer.cpython-311-x86_64-linux-gnu.so	110352	0755	edit dl rm
types.py	5526	0644	edit dl rm
version.py	1431	0644	edit dl rm
xmlreport.py	9795	0644	edit dl rm
__init__.py	1284	0644	edit dl rm
__main__.py	257	0644	edit dl rm

Edit: /opt/cloudlinux/venv/lib/python3.11/site-packages/coverage/phystokens.py (7805B)

# Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0
# For details: https://github.com/nedbat/coveragepy/blob/master/NOTICE.txt

"""Better tokenizing for coverage.py."""

from __future__ import annotations

import ast
import io
import keyword
import re
import sys
import token
import tokenize

from typing import Iterable, List, Optional, Set, Tuple

from coverage import env
from coverage.types import TLineNo, TSourceTokenLines

TokenInfos = Iterable[tokenize.TokenInfo]

def _phys_tokens(toks: TokenInfos) -> TokenInfos:
    """Return all physical tokens, even line continuations.

tokenize.generate_tokens() doesn't return a token for the backslash that
    continues lines.  This wrapper provides those tokens so that we can
    re-create a faithful representation of the original source.

Returns the same values as generate_tokens()

"""
    last_line: Optional[str] = None
    last_lineno = -1
    last_ttext: str = ""
    for ttype, ttext, (slineno, scol), (elineno, ecol), ltext in toks:
        if last_lineno != elineno:
            if last_line and last_line.endswith("\\\n"):
                # We are at the beginning of a new line, and the last line
                # ended with a backslash.  We probably have to inject a
                # backslash token into the stream. Unfortunately, there's more
                # to figure out.  This code::
                #
                #   usage = """\
                #   HEY THERE
                #   """
                #
                # triggers this condition, but the token text is::
                #
                #   '"""\\\nHEY THERE\n"""'
                #
                # so we need to figure out if the backslash is already in the
                # string token or not.
                inject_backslash = True
                if last_ttext.endswith("\\"):
                    inject_backslash = False
                elif ttype == token.STRING:
                    if "\n" in ttext and ttext.split("\n", 1)[0][-1] == "\\":
                        # It's a multi-line string and the first line ends with
                        # a backslash, so we don't need to inject another.
                        inject_backslash = False
                if inject_backslash:
                    # Figure out what column the backslash is in.
                    ccol = len(last_line.split("\n")[-2]) - 1
                    # Yield the token, with a fake token type.
                    yield tokenize.TokenInfo(
                        99999, "\\\n",
                        (slineno, ccol), (slineno, ccol+2),
                        last_line
                    )
            last_line = ltext
        if ttype not in (tokenize.NEWLINE, tokenize.NL):
            last_ttext = ttext
        yield tokenize.TokenInfo(ttype, ttext, (slineno, scol), (elineno, ecol), ltext)
        last_lineno = elineno

class MatchCaseFinder(ast.NodeVisitor):
    """Helper for finding match/case lines."""
    def __init__(self, source: str) -> None:
        # This will be the set of line numbers that start match or case statements.
        self.match_case_lines: Set[TLineNo] = set()
        self.visit(ast.parse(source))

if sys.version_info >= (3, 10):
        def visit_Match(self, node: ast.Match) -> None:
            """Invoked by ast.NodeVisitor.visit"""
            self.match_case_lines.add(node.lineno)
            for case in node.cases:
                self.match_case_lines.add(case.pattern.lineno)
            self.generic_visit(node)

def source_token_lines(source: str) -> TSourceTokenLines:
    """Generate a series of lines, one for each line in `source`.

Each line is a list of pairs, each pair is a token::

[('key', 'def'), ('ws', ' '), ('nam', 'hello'), ('op', '('), ... ]

Each pair has a token class, and the token text.

If you concatenate all the token texts, and then join them with newlines,
    you should have your original `source` back, with two differences:
    trailing white space is not preserved, and a final line with no newline
    is indistinguishable from a final line with a newline.

"""

ws_tokens = {token.INDENT, token.DEDENT, token.NEWLINE, tokenize.NL}
    line: List[Tuple[str, str]] = []
    col = 0

source = source.expandtabs(8).replace("\r\n", "\n")
    tokgen = generate_tokens(source)

if env.PYBEHAVIOR.soft_keywords:
        match_case_lines = MatchCaseFinder(source).match_case_lines

for ttype, ttext, (sline, scol), (_, ecol), _ in _phys_tokens(tokgen):
        mark_start = True
        for part in re.split("(\n)", ttext):
            if part == "\n":
                yield line
                line = []
                col = 0
                mark_end = False
            elif part == "":
                mark_end = False
            elif ttype in ws_tokens:
                mark_end = False
            else:
                if mark_start and scol > col:
                    line.append(("ws", " " * (scol - col)))
                    mark_start = False
                tok_class = tokenize.tok_name.get(ttype, "xx").lower()[:3]
                if ttype == token.NAME:
                    if keyword.iskeyword(ttext):
                        # Hard keywords are always keywords.
                        tok_class = "key"
                    elif sys.version_info >= (3, 10):   # PYVERSIONS
                        # Need the version_info check to keep mypy from borking
                        # on issoftkeyword here.
                        if env.PYBEHAVIOR.soft_keywords and keyword.issoftkeyword(ttext):
                            # Soft keywords appear at the start of the line,
                            # on lines that start match or case statements.
                            if len(line) == 0:
                                is_start_of_line = True
                            elif (len(line) == 1) and line[0][0] == "ws":
                                is_start_of_line = True
                            else:
                                is_start_of_line = False
                            if is_start_of_line and sline in match_case_lines:
                                tok_class = "key"
                line.append((tok_class, part))
                mark_end = True
            scol = 0
        if mark_end:
            col = ecol

if line:
        yield line

class CachedTokenizer:
    """A one-element cache around tokenize.generate_tokens.

When reporting, coverage.py tokenizes files twice, once to find the
    structure of the file, and once to syntax-color it.  Tokenizing is
    expensive, and easily cached.

This is a one-element cache so that our twice-in-a-row tokenizing doesn't
    actually tokenize twice.

"""
    def __init__(self) -> None:
        self.last_text: Optional[str] = None
        self.last_tokens: List[tokenize.TokenInfo] = []

def generate_tokens(self, text: str) -> TokenInfos:
        """A stand-in for `tokenize.generate_tokens`."""
        if text != self.last_text:
            self.last_text = text
            readline = io.StringIO(text).readline
            try:
                self.last_tokens = list(tokenize.generate_tokens(readline))
            except:
                self.last_text = None
                raise
        return self.last_tokens

# Create our generate_tokens cache as a callable replacement function.
generate_tokens = CachedTokenizer().generate_tokens

def source_encoding(source: bytes) -> str:
    """Determine the encoding for `source`, according to PEP 263.

`source` is a byte string: the text of the program.

Returns a string, the name of the encoding.

"""
    readline = iter(source.splitlines(True)).__next__
    return tokenize.detect_encoding(readline)[0]

cmd: