Quellcode-Bibliothek token.rs

//! Lexed TOML tokens

use super::Span;
use super::APOSTROPHE;
use super::COMMENT_START_SYMBOL;
use super::QUOTATION_MARK;
use super::WSCHAR;
use crate::decoder::Encoding;

/// An unvalidated TOML Token
#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)]
pub struct Token {
    pub(super) kind: TokenKind,
    pub(super) span: Span,
}

impl Token {
    pub(super) fn new(kind: TokenKind, span: Span) -> Self {
        Self { kind, span }
    }

    #[inline(always)]
    pub fn kind(&self) -> TokenKind {
        self.kind
    }

    #[inline(always)]
    pub fn span(&self) -> Span {
        self.span
    }
}

#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)]
#[repr(u8)]
pub enum TokenKind {
    /// Either for dotted-key or float
    Dot = b'.',
    /// Key-value separator
    Equals = b'=',
    /// Value separator
    Comma = b',',
    /// Either array or standard-table start
    LeftSquareBracket = b'[',
    /// Either array or standard-table end
    RightSquareBracket = b']',
    /// Inline table start
    LeftCurlyBracket = b'{',
    /// Inline table end
    RightCurlyBracket = b'}',
    Whitespace = WSCHAR.0,
    Comment = COMMENT_START_SYMBOL,
    Newline = b'\n',
    LiteralString = APOSTROPHE,
    BasicString = QUOTATION_MARK,
    MlLiteralString = 1,
    MlBasicString,

    /// Anything else
    Atom,

    Eof,
}

impl TokenKind {
    pub const fn description(&self) -> &'static str {
        match self {
            Self::Dot => "`.`",
            Self::Equals => "`=`",
            Self::Comma => "`,`",
            Self::LeftSquareBracket => "`[`",
            Self::RightSquareBracket => "`]`",
            Self::LeftCurlyBracket => "`{`",
            Self::RightCurlyBracket => "`}`",
            Self::Whitespace => "whitespace",
            Self::Comment => "comment",
            Self::Newline => "newline",
            Self::LiteralString => "literal string",
            Self::BasicString => "basic string",
            Self::MlLiteralString => "multi-line literal string",
            Self::MlBasicString => "multi-line basic string",
            Self::Atom => "token",
            Self::Eof => "end-of-input",
        }
    }

    pub fn encoding(&self) -> Option<Encoding> {
        match self {
            Self::LiteralString => Some(Encoding::LiteralString),
            Self::BasicString => Some(Encoding::BasicString),
            Self::MlLiteralString => Some(Encoding::MlLiteralString),
            Self::MlBasicString => Some(Encoding::MlBasicString),
            Self::Atom
            | Self::LeftSquareBracket
            | Self::RightSquareBracket
            | Self::Dot
            | Self::Equals
            | Self::Comma
            | Self::RightCurlyBracket
            | Self::LeftCurlyBracket
            | Self::Whitespace
            | Self::Newline
            | Self::Comment
            | Self::Eof => None,
        }
    }
}