pubmod ascii_str; pubmod expr; mod memchr_splitter; pubmod node; mod target; #[cfg(test)] mod tests;
use std::borrow::Cow; use std::cell::Cell; use std::env::current_dir; use std::ops::{Deref, DerefMut}; use std::path::Path; use std::sync::Arc; use std::{fmt, str};
use winnow::ascii::take_escaped; use winnow::combinator::{alt, cut_err, delimited, fail, not, opt, peek, preceded, repeat}; use winnow::error::FromExternalError; use winnow::stream::{AsChar, Stream as _}; use winnow::token::{any, one_of, take_till, take_while}; use winnow::{ModalParser, Parser};
pubstruct Parsed { // `source` must outlive `ast`, so `ast` must be declared before `source`
ast: Ast<'static>, #[allow(dead_code)]
source: Arc<str>,
}
impl Parsed { /// If `file_path` is `None`, it means the `source` is an inline template. Therefore, if /// a parsing error occurs, we won't display the path as it wouldn't be useful. pubfn new(
source: Arc<str>,
file_path: Option<Arc<Path>>,
syntax: &Syntax<'_>,
) -> Result<Self, ParseError> { // Self-referential borrowing: `self` will keep the source alive as `String`, // internally we will transmute it to `&'static str` to satisfy the compiler. // However, we only expose the nodes with a lifetime limited to `self`. let src = unsafe { mem::transmute::<&str, &'static str>(source.as_ref()) }; let ast = Ast::from_str(src, file_path, syntax)?;
Ok(Self { ast, source })
}
// The return value's lifetime must be limited to `self` to uphold the unsafe invariant. #[must_use] pubfn nodes(&self) -> &[Node<'_>] {
&self.ast.nodes
}
/// Struct used to wrap types with their associated "span" which is used when generating errors /// in the code generation. pubstruct WithSpan<'a, T> {
inner: T,
span: Span<'a>,
}
/// An location in `&'a str` #[derive(Debug, Clone, Copy)] pubstruct Span<'a>(&'a [u8; 0]);
let path = file_path
.as_ref()
.and_then(|path| Some(strip_common(¤t_dir().ok()?, path))); match path {
Some(path) => write!(f, "failed to parse template source\n --> {path}@{offset}"),
None => write!(f, "failed to parse template source near offset {offset}"),
}
}
}
pub(crate) type ParseErr<'a> = winnow::error::ErrMode<ErrorContext<'a>>; pub(crate) type ParseResult<'a, T = &'a str> = Result<T, ParseErr<'a>>;
/// This type is used to handle `nom` errors and in particular to add custom error messages. /// It used to generate `ParserError`. /// /// It cannot be used to replace `ParseError` because it expects a generic, which would make /// `askama`'s users experience less good (since this generic is only needed for `nom`). #[derive(Debug)] pub(crate) struct ErrorContext<'a> { pub(crate) span: Span<'a>, pub(crate) message: Option<Cow<'static, str>>,
}
// Information about allowed character escapes is available at: // <https://doc.rust-lang.org/reference/tokens.html#character-literals>. fn char_lit<'a>(i: &mut &'a str) -> ParseResult<'a, CharLit<'a>> { let start = i.checkpoint(); let (b_prefix, s) = (
opt('b'),
delimited( '\'',
opt(take_escaped(take_till(1.., ['\\', '\'']), '\\', any)), '\'',
),
)
.parse_next(i)?;
let Some(s) = s else {
i.reset(&start); return Err(winnow::error::ErrMode::Cut(ErrorContext::new( "empty character literal",
*i,
)));
}; letmut is = s; let Ok(c) = Char::parse(&mut is) else {
i.reset(&start); return Err(winnow::error::ErrMode::Cut(ErrorContext::new( "invalid character",
*i,
)));
};
let (nb, max_value, err1, err2) = match c {
Char::Literal | Char::Escaped => { return Ok(CharLit {
prefix: b_prefix.map(|_| CharPrefix::Binary),
content: s,
});
}
Char::AsciiEscape(nb) => (
nb, // `0x7F` is the maximum value for a `\x` escaped character. 0x7F, "invalid character in ascii escape", "must be a character in the range [\\x00-\\x7f]",
),
Char::UnicodeEscape(nb) => (
nb, // `0x10FFFF` is the maximum value for a `\u` escaped character. 0x0010_FFFF, "invalid character in unicode escape", "unicode escape must be at most 10FFFF",
),
};
let Ok(nb) = u32::from_str_radix(nb, 16) else {
i.reset(&start); return Err(winnow::error::ErrMode::Cut(ErrorContext::new(err1, *i)));
}; if nb > max_value {
i.reset(&start); return Err(winnow::error::ErrMode::Cut(ErrorContext::new(err2, *i)));
}
/// Represents the different kinds of char declarations: #[derive(Copy, Clone)] enum Char<'a> { /// Any character that is not escaped.
Literal, /// An escaped character (like `\n`) which doesn't require any extra check.
Escaped, /// Ascii escape (like `\x12`).
AsciiEscape(&'a str), /// Unicode escape (like `\u{12}`).
UnicodeEscape(&'a str),
}
let (root, start, rest) = (root, identifier, tail).parse_next(i)?; let rest = rest.as_deref().unwrap_or_default();
// The returned identifier can be assumed to be path if: // - it is an absolute path (starts with `::`), or // - it has multiple components (at least one `::`), or // - the first letter is uppercase match (root, start, rest) {
(Some(_), start, tail) => { letmut path = Vec::with_capacity(2 + tail.len());
path.push("");
path.push(start);
path.extend(rest);
Ok(PathOrIdentifier::Path(path))
}
(None, name, []) if name
.chars()
.next()
.is_none_or(|c| c == '_' || c.is_lowercase()) =>
{
Ok(PathOrIdentifier::Identifier(name))
}
(None, start, tail) => { letmut path = Vec::with_capacity(1 + tail.len());
path.push(start);
path.extend(rest);
Ok(PathOrIdentifier::Path(path))
}
}
}
for (s, k, is_closing) in [
(syntax.block_start, "opening block", false),
(syntax.block_end, "closing block", true),
(syntax.expr_start, "opening expression", false),
(syntax.expr_end, "closing expression", true),
(syntax.comment_start, "opening comment", false),
(syntax.comment_end, "closing comment", true),
] { if s.len() < 2 { return Err(format!( "delimiters must be at least two characters long. \
The {k} delimiter ({s:?}) is too short",
));
} elseif s.len() > 32 { return Err(format!( "delimiters must be at most 32 characters long. \
The {k} delimiter ({:?}...) is too long",
&s[..(16..=s.len())
.find(|&i| s.is_char_boundary(i))
.unwrap_or(s.len())],
));
} elseif s.chars().any(char::is_whitespace) { return Err(format!( "delimiters may not contain white spaces. \
The {k} delimiter ({s:?}) contains white spaces",
));
} elseif is_closing
&& ['(', '-', '+', '~', '.', '>', '<', '&', '|', '!']
.contains(&s.chars().next().unwrap())
{ return Err(format!( "closing delimiters may not start with operators. \
The {k} delimiter ({s:?}) starts with operator `{}`",
s.chars().next().unwrap(),
));
}
}
for ((s1, k1), (s2, k2)) in [
(
(syntax.block_start, "block"),
(syntax.expr_start, "expression"),
),
(
(syntax.block_start, "block"),
(syntax.comment_start, "comment"),
),
(
(syntax.expr_start, "expression"),
(syntax.comment_start, "comment"),
),
] { if s1.starts_with(s2) || s2.starts_with(s1) { let (s1, k1, s2, k2) = match s1.len() < s2.len() { true => (s1, k1, s2, k2), false => (s2, k2, s1, k1),
}; return Err(format!( "an opening delimiter may not be the prefix of another delimiter. \
The {k1} delimiter ({s1:?}) clashes with the {k2} delimiter ({s2:?})",
));
}
}
Ok(syntax)
}
}
/// The nesting level of nodes and expressions. /// /// The level counts down from [`Level::MAX_DEPTH`] to 0. Once the value would reach below 0, /// [`Level::nest()`] / [`LevelGuard::nest()`] will return an error. The same [`Level`] instance is /// shared across all usages in a [`Parsed::new()`] / [`Ast::from_str()`] call, using a reference /// to an interior mutable counter. #[derive(Debug, Clone, Copy)] struct Level<'l>(&'l Cell<usize>);
impl Level<'_> { const MAX_DEPTH: usize = 128;
/// Acquire a [`LevelGuard`] without decrementing the counter, to be used with loops. fn guard(&self) -> LevelGuard<'_> {
LevelGuard {
level: *self,
count: 0,
}
}
/// Decrement the remaining level counter, and return a [`LevelGuard`] that increments it again /// when it's dropped. fn nest<'a>(&self, i: &'a str) -> ParseResult<'a, LevelGuard<'_>> { iflet Some(new_level) = self.0.get().checked_sub(1) { self.0.set(new_level);
Ok(LevelGuard {
level: *self,
count: 1,
})
} else {
Err(Self::_fail(i))
}
}
#[cold] #[inline(never)] fn _fail(i: &str) -> ParseErr<'_> {
winnow::error::ErrMode::Cut(ErrorContext::new( "your template code is too deeply nested, or the last expression is too complex",
i,
))
}
}
/// Used to keep track how often [`LevelGuard::nest()`] was called and to re-increment the /// remaining level counter when it is dropped / falls out of scope. #[must_use] struct LevelGuard<'l> {
level: Level<'l>,
count: usize,
}
impl Drop for LevelGuard<'_> { fn drop(&mutself) { self.level.0.set(self.level.0.get() + self.count);
}
}
impl LevelGuard<'_> { /// Used to decrement the level multiple times, e.g. for every iteration of a loop. fn nest<'a>(&mut self, i: &'a str) -> ParseResult<'a, ()> { iflet Some(new_level) = self.level.0.get().checked_sub(1) { self.level.0.set(new_level); self.count += 1;
Ok(())
} else {
Err(Level::_fail(i))
}
}
}
/// Returns the common parts of two paths. /// /// The goal of this function is to reduce the path length based on the `base` argument /// (generally the path where the program is running into). For example: /// /// ```text /// current dir: /a/b/c /// path: /a/b/c/d/e.txt /// ``` /// /// `strip_common` will return `d/e.txt`. #[must_use] pubfn strip_common(base: &Path, path: &Path) -> String { let path = match path.canonicalize() {
Ok(path) => path,
Err(_) => return path.display().to_string(),
}; letmut components_iter = path.components().peekable();
for current_path_component in base.components() { let Some(path_component) = components_iter.peek() else { return path.display().to_string();
}; if current_path_component != *path_component { break;
}
components_iter.next();
} let path_parts = components_iter
.map(|c| c.as_os_str().to_string_lossy())
.collect::<Vec<_>>(); if path_parts.is_empty() {
path.display().to_string()
} else {
path_parts.join(std::path::MAIN_SEPARATOR_STR)
}
}
/// Primitive integer types. Also used as number suffixes. const INTEGER_TYPES: &[(&str, IntKind)] = &[
("i8", IntKind::I8),
("i16", IntKind::I16),
("i32", IntKind::I32),
("i64", IntKind::I64),
("i128", IntKind::I128),
("isize", IntKind::Isize),
("u8", IntKind::U8),
("u16", IntKind::U16),
("u32", IntKind::U32),
("u64", IntKind::U64),
("u128", IntKind::U128),
("usize", IntKind::Usize),
];
/// Primitive floating point types. Also used as number suffixes. const FLOAT_TYPES: &[(&str, FloatKind)] = &[
("f16", FloatKind::F16),
("f32", FloatKind::F32),
("f64", FloatKind::F64),
("f128", FloatKind::F128),
];
/// Primitive numeric types. Also used as number suffixes. const NUM_TYPES: &[(&str, NumKind)] = &{ letmut list = [("", NumKind::Int(IntKind::I8)); INTEGER_TYPES.len() + FLOAT_TYPES.len()]; letmut i = 0; letmut o = 0; while i < INTEGER_TYPES.len() { let (name, value) = INTEGER_TYPES[i];
list[o] = (name, NumKind::Int(value));
i += 1;
o += 1;
} letmut i = 0; while i < FLOAT_TYPES.len() { let (name, value) = FLOAT_TYPES[i];
list[o] = (name, NumKind::Float(value));
i += 1;
o += 1;
}
list
};
/// Complete list of named primitive types. const PRIMITIVE_TYPES: &[&str] = &{ letmut list = [""; NUM_TYPES.len() + 1]; letmut i = 0; letmut o = 0; while i < NUM_TYPES.len() {
list[o] = NUM_TYPES[i].0;
i += 1;
o += 1;
}
list[o] = "bool";
list
};
// Since the individual buckets are quite short, a linear search is faster than a binary search. for probe in kws { if padded_ident == *AsciiChar::slice_as_bytes(probe[2..].try_into().unwrap()) { returntrue;
}
} false
}
#[cfg(not(windows))] #[cfg(test)] mod test { use std::path::Path;
usesuper::*;
#[test] fn test_strip_common() { // Full path is returned instead of empty when the entire path is in common.
assert_eq!(strip_common(Path::new("home"), Path::new("home")), "home");
let cwd = std::env::current_dir().expect("current_dir failed");
// We need actual existing paths for `canonicalize` to work, so let's do that. let entry = cwd
.read_dir()
.expect("read_dir failed")
.filter_map(std::result::Result::ok)
.find(|f| f.path().is_file())
.expect("no entry");
// Since they have the complete path in common except for the folder entry name, it should // return only the folder entry name.
assert_eq!(
strip_common(&cwd, &entry.path()),
entry.file_name().to_string_lossy()
);
// In this case it cannot canonicalize `/a/b/c` so it returns the path as is.
assert_eq!(strip_common(&cwd, Path::new("/a/b/c")), "/a/b/c");
}
#[test] fn test_num_lit() { // Should fail.
assert!(num_lit.parse_peek(".").is_err()); // Should succeed.
assert_eq!(
num_lit.parse_peek("1.2E-02").unwrap(),
("", Num::Float("1.2E-02", None))
);
assert_eq!(
num_lit.parse_peek("4e3").unwrap(),
("", Num::Float("4e3", None)),
);
assert_eq!(
num_lit.parse_peek("4e+_3").unwrap(),
("", Num::Float("4e+_3", None)),
); // Not supported because Rust wants a number before the `.`.
assert!(num_lit.parse_peek(".1").is_err());
assert!(num_lit.parse_peek(".1E-02").is_err()); // A `_` directly after the `.` denotes a field.
assert_eq!(
num_lit.parse_peek("1._0").unwrap(),
("._0", Num::Int("1", None))
);
assert_eq!(
num_lit.parse_peek("1_.0").unwrap(),
("", Num::Float("1_.0", None))
); // Not supported (voluntarily because of `1..` syntax).
assert_eq!(
num_lit.parse_peek("1.").unwrap(),
(".", Num::Int("1", None))
);
assert_eq!(
num_lit.parse_peek("1_.").unwrap(),
(".", Num::Int("1_", None))
);
assert_eq!(
num_lit.parse_peek("1_2.").unwrap(),
(".", Num::Int("1_2", None))
); // Numbers with suffixes
assert_eq!(
num_lit.parse_peek("-1usize").unwrap(),
("", Num::Int("-1", Some(IntKind::Usize)))
);
assert_eq!(
num_lit.parse_peek("123_f32").unwrap(),
("", Num::Float("123_", Some(FloatKind::F32)))
);
assert_eq!(
num_lit.parse_peek("1_.2_e+_3_f64|into_isize").unwrap(),
( "|into_isize",
Num::Float("1_.2_e+_3_", Some(FloatKind::F64))
)
);
assert_eq!(
num_lit.parse_peek("4e3f128").unwrap(),
("", Num::Float("4e3", Some(FloatKind::F128))),
);
}
Die Informationen auf dieser Webseite wurden
nach bestem Wissen sorgfältig zusammengestellt. Es wird jedoch weder Vollständigkeit, noch Richtigkeit,
noch Qualität der bereit gestellten Informationen zugesichert.
Bemerkung:
Die farbliche Syntaxdarstellung und die Messung sind noch experimentell.