Whitespace Counter Tokenizer

This class inherits from the main Tokenizer class and extends it by counting runs of spaces and line breaks.

__init__(**data) -> None

Initialise the Tokenizer class.

Source code in lexos/tokenizer/__init__.py

def __init__(self, **data) -> None:
    """Initialise the Tokenizer class."""
    super().__init__(**data)
    if self.nlp is None:
        try:
            self.nlp = spacy.load(self.model)
        except (OSError, ImportError):
            raise LexosException(
                f"Error loading model {self.model}. Please check the name and try again. You may need to install the model on your system."
            )
    self.nlp.max_length = self.max_length

_get_token_widths(text: str) -> tuple[list[str], list[int]]

Get the widths of tokens in a doc.

Parameters:

Name	Type	Description	Default
text	str	The input text.	required

Returns:

Type	Description
tuple[list[str], list[int]]	tuple[list[str], list[int]]: A tuple containing the tokens and widths.

Source code in lexos/tokenizer/whitespace_counter.py

def _get_token_widths(self, text: str) -> tuple[list[str], list[int]]:
    """Get the widths of tokens in a doc.

    Args:
        text (str): The input text.

    Returns:
        tuple[list[str], list[int]]: A tuple containing the tokens and widths.
    """
    tokens = []
    widths = []
    for match in self._token_widths_pattern.finditer(text):
        word, newline, multi_space, single_space = match.groups()
        if word:
            tokens.append(word)
            widths.append(len(word))  # Use number of characters in word
        elif newline:
            tokens.append("\n")
            widths.append(1)  # Use 1 to indicate a line break
        elif multi_space:
            tokens.append(" ")
            widths.append(len(multi_space))
        elif single_space:
            tokens.append(" ")
            widths.append(1)
    return tokens, widths

make_doc(text: str, max_length: int = None, disable: Optional[Iterable[str]] = None) -> Doc

Return a doc from a text.

Parameters:

Name	Type	Description	Default
text	str	The text to be parsed.	required
max_length	int	The maximum length of the doc.	None
disable	list[str]	A list of spaCy pipeline components to disable.	None

Returns:

Name	Type	Description
Doc	Doc	A spaCy doc object.

Source code in lexos/tokenizer/whitespace_counter.py

@validate_call
def make_doc(
    self, text: str, max_length: int = None, disable: Optional[Iterable[str]] = None
) -> Doc:
    """Return a doc from a text.

    Args:
        text (str): The text to be parsed.
        max_length (int): The maximum length of the doc.
        disable (list[str]): A list of spaCy pipeline components to disable.

    Returns:
        Doc: A spaCy doc object.
    """
    # Override instance settings with keyword arguments
    if max_length:
        self.max_length = max_length
        self.nlp.max_length = max_length
    disable = list(disable) if disable else []
    if disable:
        self.nlp.select_pipes(disable=disable)
    tokens, widths = self._get_token_widths(text)
    if not Token.has_extension("width"):
        Token.set_extension("width", default=0)
    doc = Doc(self.nlp.vocab, words=tokens)
    for token, count in zip(doc, widths):
        token._.width = count
    # Apply pipeline components manually, skipping those in 'disable'
    for name, proc in self.nlp.pipeline:
        if name not in disable:
            doc = proc(doc)
    return doc

make_docs(texts: Iterable[str], max_length: int = None, disable: Optional[Iterable[str]] = None, chunk_size: int = 1000) -> Iterable[Doc]

Return a generator of docs from an iterable of texts, processing in chunks.

Parameters:

Name	Type	Description	Default
texts	Iterable[str]	The texts to process.	required
max_length	int	Maximum doc length.	None
disable	Iterable[str]	Pipeline components to disable.	None
chunk_size	int	Number of docs to process per chunk.	1000

Yields:

Name	Type	Description
Doc	Iterable[Doc]	spaCy Doc objects.

Source code in lexos/tokenizer/whitespace_counter.py

@validate_call
def make_docs(
    self,
    texts: Iterable[str],
    max_length: int = None,
    disable: Optional[Iterable[str]] = None,
    chunk_size: int = 1000,
) -> Iterable[Doc]:
    """Return a generator of docs from an iterable of texts, processing in chunks.

    Args:
        texts (Iterable[str]): The texts to process.
        max_length (int, optional): Maximum doc length.
        disable (Iterable[str], optional): Pipeline components to disable.
        chunk_size (int, optional): Number of docs to process per chunk.

    Yields:
        Doc: spaCy Doc objects.
    """
    if max_length:
        self.max_length = max_length
        self.nlp.max_length = max_length

    disable = list(disable) if disable else []
    if not Token.has_extension("width"):
        Token.set_extension("width", default=0)
    enabled_pipes = [
        (name, proc) for name, proc in self.nlp.pipeline if name not in disable
    ]

    for text_chunk in batched(texts, chunk_size):
        docs = []
        for text in text_chunk:
            tokens, widths = self._get_token_widths(text)
            doc = Doc(self.nlp.vocab, words=tokens)
            for token, count in zip(doc, widths):
                token._.width = count
            docs.append(doc)
        for _, proc in enabled_pipes:
            docs = [proc(doc) for doc in docs]
        yield from docs