pytermgui.markup.tokens

The building blocks of the TIM language,

Use pytermgui.markup.parsing.tokenize_markup and pytermgui.markup.parsing.tokenize_ansi to generate.

  1"""The building blocks of the TIM language,
  2
  3Use `pytermgui.markup.parsing.tokenize_markup` and `pytermgui.markup.parsing.tokenize_ansi` to
  4generate.
  5"""
  6
  7from __future__ import annotations
  8
  9from dataclasses import dataclass
 10from functools import cached_property
 11from typing import TYPE_CHECKING, Any, Generator, Iterator
 12
 13from typing_extensions import TypeGuard
 14
 15from ..colors import Color
 16
 17if TYPE_CHECKING:
 18    from ..fancy_repr import FancyYield
 19
 20__all__ = [
 21    "Token",
 22    "PlainToken",
 23    "StyleToken",
 24    "ColorToken",
 25    "AliasToken",
 26    "ClearToken",
 27    "MacroToken",
 28    "HLinkToken",
 29    "CursorToken",
 30]
 31
 32
 33class Token:
 34    """A piece of markup information.
 35
 36    All tokens must have at least a `value` field, and have `markup` and `prettified_markup`
 37    properties derived from it in some manner.
 38
 39    They are meant to be immutable (frozen), and generated by some tokenization. They are also
 40    static representations of the data in its pre-parsed form.
 41    """
 42
 43    value: str
 44
 45    @cached_property
 46    def markup(self) -> str:
 47        """Returns markup representing this token."""
 48
 49        return self.value
 50
 51    @cached_property
 52    def prettified_markup(self) -> str:
 53        """Returns syntax-highlighted markup representing this token."""
 54
 55        return f"[{self.markup}]{self.markup}[/{self.markup}]"
 56
 57    def __eq__(self, other: object) -> bool:
 58        return isinstance(other, type(self)) and other.value == self.value
 59
 60    def __repr__(self) -> str:
 61        return f"<{type(self).__name__} markup: '{self.markup}'>"
 62
 63    def __fancy_repr__(self) -> Generator[FancyYield, None, None]:
 64        yield f"<{type(self).__name__} markup: "
 65        yield {
 66            "text": self.prettified_markup,
 67            "highlight": False,
 68        }
 69        yield ">"
 70
 71    def is_plain(self) -> TypeGuard["PlainToken"]:
 72        """Returns True if this token is an instance of PlainToken."""
 73
 74        return isinstance(self, PlainToken)
 75
 76    def is_color(self) -> TypeGuard["ColorToken"]:
 77        """Returns True if this token is an instance of ColorToken."""
 78
 79        return isinstance(self, ColorToken)
 80
 81    def is_style(self) -> TypeGuard["StyleToken"]:
 82        """Returns True if this token is an instance of StyleToken."""
 83
 84        return isinstance(self, StyleToken)
 85
 86    def is_alias(self) -> TypeGuard["AliasToken"]:
 87        """Returns True if this token is an instance of AliasToken."""
 88
 89        return isinstance(self, AliasToken)
 90
 91    def is_macro(self) -> TypeGuard["MacroToken"]:
 92        """Returns True if this token is an instance of MacroToken."""
 93
 94        return isinstance(self, MacroToken)
 95
 96    def is_clear(self) -> TypeGuard["ClearToken"]:
 97        """Returns True if this token is an instance of ClearToken."""
 98
 99        return isinstance(self, ClearToken)
100
101    def is_hyperlink(self) -> TypeGuard["HLinkToken"]:
102        """Returns True if this token is an instance of HLinkToken."""
103
104        return isinstance(self, HLinkToken)
105
106    def is_cursor(self) -> TypeGuard["CursorToken"]:
107        """Returns True if this token is an instance of CursorToken."""
108
109        return isinstance(self, CursorToken)
110
111
112@dataclass(frozen=True, repr=False)
113class PlainToken(Token):
114    """A plain piece of text.
115
116    These are the parts of data in-between markup tag groups.
117    """
118
119    __slots__ = ("value",)
120
121    value: str
122
123    def __repr__(self) -> str:
124        return f"<{type(self).__name__} markup: {self.markup!r}>"
125
126    def __fancy_repr__(self) -> Generator[FancyYield, None, None]:
127        yield f"<{type(self).__name__} markup: {self.markup!r}>"
128
129
130@dataclass(frozen=True, repr=False)
131class ColorToken(Token):
132    """A color identifier.
133
134    It stores the markup that created it, as well as the `pytermgui.colors.Color` object
135    that it represents.
136    """
137
138    __slots__ = ("value",)
139
140    value: str
141    color: Color
142
143    @cached_property
144    def markup(self) -> str:
145        return self.color.markup
146
147    @cached_property
148    def prettified_markup(self) -> str:
149        clearer = "bg" if self.color.background else "fg"
150
151        return f"[{self.markup}]{self.markup}[/{clearer}]"
152
153
154@dataclass(frozen=True, repr=False)
155class StyleToken(Token):
156    """A terminal-style identifier.
157
158    Most terminals support a set of 9 styles:
159
160    - bold
161    - dim
162    - italic
163    - underline
164    - blink
165    - blink2
166    - inverse
167    - invisible
168    - strikethrough
169
170    This token will store the style it represents by its name in the `value` field. Note
171    that other, less widely supported styles *may* be available; for an up-to-date list,
172    run `ptg -i pytermgui.markup.style_maps.STYLES`.
173    ```
174
175    """
176
177    __slots__ = ("value",)
178
179    value: str
180
181
182@dataclass(frozen=True, repr=False)
183class ClearToken(Token):
184    """A tag-clearer.
185
186    These tokens are prefixed by `/`, and followed by the name of the tag they target.
187
188    To reset color information in the current text, use the `/fg` and `/bg` special
189    tags. We cannot unset a specific color due to how the terminal works; all these do
190    is "reset" the current stroke color to the default of the terminal.
191
192    Additionally, there are some other special identifiers:
193
194    - `/`:  Clears all tags, including styles, colors, macros, links and more.
195    - `/!`: Clears all currently applied macros.
196    - `/~`: Clears all currently applied links.
197    """
198
199    __slots__ = ("value",)
200
201    value: str
202
203    @cached_property
204    def prettified_markup(self) -> str:
205        target = self.markup[1:]
206
207        return f"[210 strikethrough]/[/fg]{target}[/]"
208
209    def __eq__(self, other: object) -> bool:
210        if not isinstance(other, Token):
211            return False
212
213        return super().__eq__(other) or all(
214            obj.markup in ["/dim", "/bold"] for obj in [self, other]
215        )
216
217    def targets(  # pylint: disable=too-many-return-statements
218        self, token: Token
219    ) -> bool:
220        """Returns True if this token targets the one given as an argument."""
221
222        if token.is_clear() or token.is_cursor():
223            return False
224
225        if self.value in ("/", f"/{token.value}"):
226            return True
227
228        if token.is_hyperlink() and self.value == "/~":
229            return True
230
231        if token.is_macro() and self.value == "/!":
232            return True
233
234        if not Token.is_color(token):
235            return False
236
237        if self.value == "/fg" and not token.color.background:
238            return True
239
240        return self.value == "/bg" and token.color.background
241
242
243@dataclass(frozen=True, repr=False)
244class AliasToken(Token):
245    """A way to reference a set of tags from one central name."""
246
247    __slots__ = ("value",)
248
249    value: str
250
251
252@dataclass(frozen=True, repr=False)
253class MacroToken(Token):
254    """A binding of a Python function to a markup name.
255
256    See the docs on information about syntax & semantics.
257    """
258
259    __slots__ = ("value", "arguments")
260
261    value: str
262    arguments: tuple[str, ...]
263
264    def __iter__(self) -> Iterator[Any]:
265        return iter((self.value, self.arguments))
266
267    @cached_property
268    def prettified_markup(self) -> str:
269        target = self.markup[1:]
270
271        return f"[210 bold]![/]{target}"
272
273    @cached_property
274    def markup(self) -> str:
275        return f"{self.value}" + (
276            f"({':'.join(self.arguments)})" if len(self.arguments) > 0 else ""
277        )
278
279
280@dataclass(frozen=True, repr=False)
281class HLinkToken(Token):
282    """A terminal hyperlink.
283
284    See https://gist.github.com/egmontkob/eb114294efbcd5adb1944c9f3cb5feda.
285    """
286
287    __slots__ = ("value",)
288
289    value: str
290
291    @cached_property
292    def markup(self) -> str:
293        return f"~{self.value}"
294
295    @cached_property
296    def prettified_markup(self) -> str:
297        return f"[{self.markup}]~[blue underline]{self.value}[/fg /underline /~]"
298
299
300@dataclass(frozen=True, repr=False)
301class CursorToken(Token):
302    """A cursor location.
303
304    These can be used to move the terminal's cursor.
305    """
306
307    __slots__ = ("value", "y", "x")
308
309    value: str
310    y: int | None
311    x: int | None
312
313    def __iter__(self) -> Iterator[int | None]:
314        return iter((self.y, self.x))
315
316    def __repr__(self) -> str:
317        return f"<{type(self).__name__} position: {(';'.join(map(str, self)))}>"
318
319    def __fancy_repr__(self) -> Generator[FancyYield, None, None]:
320        yield self.__repr__()
321
322    @cached_property
323    def markup(self) -> str:
324        return f"({self.value})"
class Token:
 34class Token:
 35    """A piece of markup information.
 36
 37    All tokens must have at least a `value` field, and have `markup` and `prettified_markup`
 38    properties derived from it in some manner.
 39
 40    They are meant to be immutable (frozen), and generated by some tokenization. They are also
 41    static representations of the data in its pre-parsed form.
 42    """
 43
 44    value: str
 45
 46    @cached_property
 47    def markup(self) -> str:
 48        """Returns markup representing this token."""
 49
 50        return self.value
 51
 52    @cached_property
 53    def prettified_markup(self) -> str:
 54        """Returns syntax-highlighted markup representing this token."""
 55
 56        return f"[{self.markup}]{self.markup}[/{self.markup}]"
 57
 58    def __eq__(self, other: object) -> bool:
 59        return isinstance(other, type(self)) and other.value == self.value
 60
 61    def __repr__(self) -> str:
 62        return f"<{type(self).__name__} markup: '{self.markup}'>"
 63
 64    def __fancy_repr__(self) -> Generator[FancyYield, None, None]:
 65        yield f"<{type(self).__name__} markup: "
 66        yield {
 67            "text": self.prettified_markup,
 68            "highlight": False,
 69        }
 70        yield ">"
 71
 72    def is_plain(self) -> TypeGuard["PlainToken"]:
 73        """Returns True if this token is an instance of PlainToken."""
 74
 75        return isinstance(self, PlainToken)
 76
 77    def is_color(self) -> TypeGuard["ColorToken"]:
 78        """Returns True if this token is an instance of ColorToken."""
 79
 80        return isinstance(self, ColorToken)
 81
 82    def is_style(self) -> TypeGuard["StyleToken"]:
 83        """Returns True if this token is an instance of StyleToken."""
 84
 85        return isinstance(self, StyleToken)
 86
 87    def is_alias(self) -> TypeGuard["AliasToken"]:
 88        """Returns True if this token is an instance of AliasToken."""
 89
 90        return isinstance(self, AliasToken)
 91
 92    def is_macro(self) -> TypeGuard["MacroToken"]:
 93        """Returns True if this token is an instance of MacroToken."""
 94
 95        return isinstance(self, MacroToken)
 96
 97    def is_clear(self) -> TypeGuard["ClearToken"]:
 98        """Returns True if this token is an instance of ClearToken."""
 99
100        return isinstance(self, ClearToken)
101
102    def is_hyperlink(self) -> TypeGuard["HLinkToken"]:
103        """Returns True if this token is an instance of HLinkToken."""
104
105        return isinstance(self, HLinkToken)
106
107    def is_cursor(self) -> TypeGuard["CursorToken"]:
108        """Returns True if this token is an instance of CursorToken."""
109
110        return isinstance(self, CursorToken)

A piece of markup information.

All tokens must have at least a value field, and have markup and prettified_markup properties derived from it in some manner.

They are meant to be immutable (frozen), and generated by some tokenization. They are also static representations of the data in its pre-parsed form.

Token()
markup: str

Returns markup representing this token.

prettified_markup: str

Returns syntax-highlighted markup representing this token.

def is_plain(self) -> TypeGuard[pytermgui.markup.tokens.PlainToken]:
72    def is_plain(self) -> TypeGuard["PlainToken"]:
73        """Returns True if this token is an instance of PlainToken."""
74
75        return isinstance(self, PlainToken)

Returns True if this token is an instance of PlainToken.

def is_color(self) -> TypeGuard[pytermgui.markup.tokens.ColorToken]:
77    def is_color(self) -> TypeGuard["ColorToken"]:
78        """Returns True if this token is an instance of ColorToken."""
79
80        return isinstance(self, ColorToken)

Returns True if this token is an instance of ColorToken.

def is_style(self) -> TypeGuard[pytermgui.markup.tokens.StyleToken]:
82    def is_style(self) -> TypeGuard["StyleToken"]:
83        """Returns True if this token is an instance of StyleToken."""
84
85        return isinstance(self, StyleToken)

Returns True if this token is an instance of StyleToken.

def is_alias(self) -> TypeGuard[pytermgui.markup.tokens.AliasToken]:
87    def is_alias(self) -> TypeGuard["AliasToken"]:
88        """Returns True if this token is an instance of AliasToken."""
89
90        return isinstance(self, AliasToken)

Returns True if this token is an instance of AliasToken.

def is_macro(self) -> TypeGuard[pytermgui.markup.tokens.MacroToken]:
92    def is_macro(self) -> TypeGuard["MacroToken"]:
93        """Returns True if this token is an instance of MacroToken."""
94
95        return isinstance(self, MacroToken)

Returns True if this token is an instance of MacroToken.

def is_clear(self) -> TypeGuard[pytermgui.markup.tokens.ClearToken]:
 97    def is_clear(self) -> TypeGuard["ClearToken"]:
 98        """Returns True if this token is an instance of ClearToken."""
 99
100        return isinstance(self, ClearToken)

Returns True if this token is an instance of ClearToken.

def is_cursor(self) -> TypeGuard[pytermgui.markup.tokens.CursorToken]:
107    def is_cursor(self) -> TypeGuard["CursorToken"]:
108        """Returns True if this token is an instance of CursorToken."""
109
110        return isinstance(self, CursorToken)

Returns True if this token is an instance of CursorToken.

@dataclass(frozen=True, repr=False)
class PlainToken(Token):
113@dataclass(frozen=True, repr=False)
114class PlainToken(Token):
115    """A plain piece of text.
116
117    These are the parts of data in-between markup tag groups.
118    """
119
120    __slots__ = ("value",)
121
122    value: str
123
124    def __repr__(self) -> str:
125        return f"<{type(self).__name__} markup: {self.markup!r}>"
126
127    def __fancy_repr__(self) -> Generator[FancyYield, None, None]:
128        yield f"<{type(self).__name__} markup: {self.markup!r}>"

A plain piece of text.

These are the parts of data in-between markup tag groups.

PlainToken(value: str)
value: str
@dataclass(frozen=True, repr=False)
class StyleToken(Token):
155@dataclass(frozen=True, repr=False)
156class StyleToken(Token):
157    """A terminal-style identifier.
158
159    Most terminals support a set of 9 styles:
160
161    - bold
162    - dim
163    - italic
164    - underline
165    - blink
166    - blink2
167    - inverse
168    - invisible
169    - strikethrough
170
171    This token will store the style it represents by its name in the `value` field. Note
172    that other, less widely supported styles *may* be available; for an up-to-date list,
173    run `ptg -i pytermgui.markup.style_maps.STYLES`.
174    ```
175
176    """
177
178    __slots__ = ("value",)
179
180    value: str

A terminal-style identifier.

Most terminals support a set of 9 styles:

  • bold
  • dim
  • italic
  • underline
  • blink
  • blink2
  • inverse
  • invisible
  • strikethrough

This token will store the style it represents by its name in the value field. Note that other, less widely supported styles may be available; for an up-to-date list, run ptg -i pytermgui.markup.style_maps.STYLES. ```

StyleToken(value: str)
value: str
@dataclass(frozen=True, repr=False)
class ColorToken(Token):
131@dataclass(frozen=True, repr=False)
132class ColorToken(Token):
133    """A color identifier.
134
135    It stores the markup that created it, as well as the `pytermgui.colors.Color` object
136    that it represents.
137    """
138
139    __slots__ = ("value",)
140
141    value: str
142    color: Color
143
144    @cached_property
145    def markup(self) -> str:
146        return self.color.markup
147
148    @cached_property
149    def prettified_markup(self) -> str:
150        clearer = "bg" if self.color.background else "fg"
151
152        return f"[{self.markup}]{self.markup}[/{clearer}]"

A color identifier.

It stores the markup that created it, as well as the pytermgui.colors.Color object that it represents.

ColorToken(value: str, color: pytermgui.colors.Color)
value: str
markup: str
prettified_markup: str
@dataclass(frozen=True, repr=False)
class AliasToken(Token):
244@dataclass(frozen=True, repr=False)
245class AliasToken(Token):
246    """A way to reference a set of tags from one central name."""
247
248    __slots__ = ("value",)
249
250    value: str

A way to reference a set of tags from one central name.

AliasToken(value: str)
value: str
@dataclass(frozen=True, repr=False)
class ClearToken(Token):
183@dataclass(frozen=True, repr=False)
184class ClearToken(Token):
185    """A tag-clearer.
186
187    These tokens are prefixed by `/`, and followed by the name of the tag they target.
188
189    To reset color information in the current text, use the `/fg` and `/bg` special
190    tags. We cannot unset a specific color due to how the terminal works; all these do
191    is "reset" the current stroke color to the default of the terminal.
192
193    Additionally, there are some other special identifiers:
194
195    - `/`:  Clears all tags, including styles, colors, macros, links and more.
196    - `/!`: Clears all currently applied macros.
197    - `/~`: Clears all currently applied links.
198    """
199
200    __slots__ = ("value",)
201
202    value: str
203
204    @cached_property
205    def prettified_markup(self) -> str:
206        target = self.markup[1:]
207
208        return f"[210 strikethrough]/[/fg]{target}[/]"
209
210    def __eq__(self, other: object) -> bool:
211        if not isinstance(other, Token):
212            return False
213
214        return super().__eq__(other) or all(
215            obj.markup in ["/dim", "/bold"] for obj in [self, other]
216        )
217
218    def targets(  # pylint: disable=too-many-return-statements
219        self, token: Token
220    ) -> bool:
221        """Returns True if this token targets the one given as an argument."""
222
223        if token.is_clear() or token.is_cursor():
224            return False
225
226        if self.value in ("/", f"/{token.value}"):
227            return True
228
229        if token.is_hyperlink() and self.value == "/~":
230            return True
231
232        if token.is_macro() and self.value == "/!":
233            return True
234
235        if not Token.is_color(token):
236            return False
237
238        if self.value == "/fg" and not token.color.background:
239            return True
240
241        return self.value == "/bg" and token.color.background

A tag-clearer.

These tokens are prefixed by /, and followed by the name of the tag they target.

To reset color information in the current text, use the /fg and /bg special tags. We cannot unset a specific color due to how the terminal works; all these do is "reset" the current stroke color to the default of the terminal.

Additionally, there are some other special identifiers:

  • /: Clears all tags, including styles, colors, macros, links and more.
  • /!: Clears all currently applied macros.
  • /~: Clears all currently applied links.
ClearToken(value: str)
value: str
prettified_markup: str
def targets(self, token: pytermgui.markup.tokens.Token) -> bool:
218    def targets(  # pylint: disable=too-many-return-statements
219        self, token: Token
220    ) -> bool:
221        """Returns True if this token targets the one given as an argument."""
222
223        if token.is_clear() or token.is_cursor():
224            return False
225
226        if self.value in ("/", f"/{token.value}"):
227            return True
228
229        if token.is_hyperlink() and self.value == "/~":
230            return True
231
232        if token.is_macro() and self.value == "/!":
233            return True
234
235        if not Token.is_color(token):
236            return False
237
238        if self.value == "/fg" and not token.color.background:
239            return True
240
241        return self.value == "/bg" and token.color.background

Returns True if this token targets the one given as an argument.

@dataclass(frozen=True, repr=False)
class MacroToken(Token):
253@dataclass(frozen=True, repr=False)
254class MacroToken(Token):
255    """A binding of a Python function to a markup name.
256
257    See the docs on information about syntax & semantics.
258    """
259
260    __slots__ = ("value", "arguments")
261
262    value: str
263    arguments: tuple[str, ...]
264
265    def __iter__(self) -> Iterator[Any]:
266        return iter((self.value, self.arguments))
267
268    @cached_property
269    def prettified_markup(self) -> str:
270        target = self.markup[1:]
271
272        return f"[210 bold]![/]{target}"
273
274    @cached_property
275    def markup(self) -> str:
276        return f"{self.value}" + (
277            f"({':'.join(self.arguments)})" if len(self.arguments) > 0 else ""
278        )

A binding of a Python function to a markup name.

See the docs on information about syntax & semantics.

MacroToken(value: str, arguments: tuple[str, ...])
value: str
arguments: tuple[str, ...]
prettified_markup: str
markup: str
@dataclass(frozen=True, repr=False)
class HLinkToken(Token):
281@dataclass(frozen=True, repr=False)
282class HLinkToken(Token):
283    """A terminal hyperlink.
284
285    See https://gist.github.com/egmontkob/eb114294efbcd5adb1944c9f3cb5feda.
286    """
287
288    __slots__ = ("value",)
289
290    value: str
291
292    @cached_property
293    def markup(self) -> str:
294        return f"~{self.value}"
295
296    @cached_property
297    def prettified_markup(self) -> str:
298        return f"[{self.markup}]~[blue underline]{self.value}[/fg /underline /~]"
HLinkToken(value: str)
value: str
markup: str
prettified_markup: str
@dataclass(frozen=True, repr=False)
class CursorToken(Token):
301@dataclass(frozen=True, repr=False)
302class CursorToken(Token):
303    """A cursor location.
304
305    These can be used to move the terminal's cursor.
306    """
307
308    __slots__ = ("value", "y", "x")
309
310    value: str
311    y: int | None
312    x: int | None
313
314    def __iter__(self) -> Iterator[int | None]:
315        return iter((self.y, self.x))
316
317    def __repr__(self) -> str:
318        return f"<{type(self).__name__} position: {(';'.join(map(str, self)))}>"
319
320    def __fancy_repr__(self) -> Generator[FancyYield, None, None]:
321        yield self.__repr__()
322
323    @cached_property
324    def markup(self) -> str:
325        return f"({self.value})"

A cursor location.

These can be used to move the terminal's cursor.

CursorToken(value: str, y: int | None, x: int | None)
value: str
y: int | None
x: int | None
markup: str