"" " for Mojo and related languages.for details."" " import keywordfrom pygments import unistring as unifrom pygments.lexer import (from pygments.token import (# Error, from pygments.util import shebang_matches"MojoLexer" ]class MojoLexer(RegexLexer):"" " For Mojo source code (version 24.2.1)."" " "Mojo" "https://docs.modular.com/mojo/ " "mojo" , "🔥" ]"*.mojo" ,"*.🔥" ,"text/x-mojo" ,"application/x-mojo" ,"2.18" "[{uni.xid_start}][{uni.xid_continue}]*" def innerstring_rules(ttype):return [# the old style '%s' % (...) string formatting (still valid in Py3) "%(\(\w+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?" "[hlL]?[E-GXc-giorsaux%]" ,# the new style '{}'.format(...) string formatting "\{" "((\w+)((\.\w+)|(\[[^\]]+\]))*)?" # field name "(\![sra])?" # conversion "(\:(.?[<>=\^])?[-+ ]?#?0?(\d+)?,?(\.\d+)?[E-GXb-gnosx%]?)?" "\}" ,# backslashes, quotes and formatting signs must be parsed one at a time '[^\\\' "%{\n]+', ttype), '[\' "\\]', ttype), # unhandled string formatting sign "%|(\{{1,2})" , ttype),# newlines are an error (use "nl" state) def fstring_rules(ttype):return [# Assuming that a '}' is the closing brace after format specifier. # Sadly, this means that we won't detect syntax error. But it's # more important to parse correct syntax correctly, than to # highlight invalid syntax. "\}" , String.Interpol),"\{" , String.Interpol, "expr-inside-fstring" ),# backslashes, quotes and formatting signs must be parsed one at a time '[^\\\' "{}\n]+', ttype), '[\' "\\]', ttype), # newlines are an error (use "nl" state) "root" : ["\s+" , Whitespace),'^(\s*)([rRuUbB]{,2})("""(?:.|\n)*?""")' ,"^(\s*)([rRuUbB]{,2})('''(?:.|\n)*?''')" ,"\A#!.+$", Comment.Hashbang), "#.*$", Comment.Single), "\\\n" , Whitespace),"\\" , Whitespace),"keywords" ),"soft-keywords" ),# In the original PR, all the below here used ((?:\s|\\\s)+) to # designate whitespace, but I can't find any example of this being # needed in the example file, so we're replacing it with `\s+`. "(alias)(\s+)" ,"varname" , # TODO varname the right fit? "(var)(\s+)" , bygroups(Keyword, Whitespace), "varname" ),"(def)(\s+)" , bygroups(Keyword, Whitespace), "funcname" ),"(fn)(\s+)" , bygroups(Keyword, Whitespace), "funcname" ),"(class)(\s+)" ,"classname" ,# not implemented yet "(struct)(\s+)" , bygroups(Keyword, Whitespace), "structname" ),"(trait)(\s+)" , bygroups(Keyword, Whitespace), "structname" ),"(from)(\s+)" , bygroups(Keyword.Namespace, Whitespace), "fromimport" ),"(import)(\s+)" , bygroups(Keyword.Namespace, Whitespace), "import" ),"expr" ),"expr" : [# raw f-strings '(?i)(rf|fr)(""")' ,"rfstringescape" , "tdqf" ),"(?i)(rf|fr)(''')" ,"rfstringescape" , "tsqf" ),'(?i)(rf|fr)(")' ,"rfstringescape" , "dqf" ),"(?i)(rf|fr)(')" ,"rfstringescape" , "sqf" ),# non-raw f-strings '([fF])(""")' ,"fstringescape" , "tdqf" ),"([fF])(''')" ,"fstringescape" , "tsqf" ),'([fF])(")' ,"fstringescape" , "dqf" ),"([fF])(')" ,"fstringescape" , "sqf" ),# raw bytes and strings '(?i)(rb|br|r)(""")' , bygroups(String.Affix, String.Double), "tdqs" ),"(?i)(rb|br|r)(''')" , bygroups(String.Affix, String.Single), "tsqs" ),'(?i)(rb|br|r)(")' , bygroups(String.Affix, String.Double), "dqs" ),"(?i)(rb|br|r)(')" , bygroups(String.Affix, String.Single), "sqs" ),# non-raw strings '([uU]?)(""")' ,"stringescape" , "tdqs" ),"([uU]?)(''')" ,"stringescape" , "tsqs" ),'([uU]?)(")' ,"stringescape" , "dqs" ),"([uU]?)(')" ,"stringescape" , "sqs" ),# non-raw bytes '([bB])(""")' ,"bytesescape" , "tdqs" ),"([bB])(''')" ,"bytesescape" , "tsqs" ),'([bB])(")' ,"bytesescape" , "dqs" ),"([bB])(')" ,"bytesescape" , "sqs" ),"[^\S\n]+" , Text),"numbers" ),"!=|==|<<|>>|:=|[-~+/*%=<>&^|.]" , Operator),"([]{}:\(\),;[])+" , Punctuation),"(in|is|and|or|not)\b" , Operator.Word),"expr-keywords" ),"builtins" ),"magicfuncs" ),"magicvars" ),"name" ),"expr-inside-fstring" : ["[{([]" , Punctuation, "expr-inside-fstring-inner" ),# without format specifier "(=\s*)?" # debug (https://bugs.python.org/issue36817) "(\![sraf])?" # conversion "\}" ,"#pop", # with format specifier # we'll catch the remaining '}' in the outer scope "(=\s*)?" # debug (https://bugs.python.org/issue36817) "(\![sraf])?" # conversion ":" ,"#pop", "\s+" , Whitespace), # allow new lines "expr" ),"expr-inside-fstring-inner" : ["[{([]" , Punctuation, "expr-inside-fstring-inner" ),"[])}]" , Punctuation, "#pop"), "\s+" , Whitespace), # allow new lines "expr" ),"expr-keywords" : [# Based on https://docs.python.org/3/reference/expressions.html "async for" , # TODO https://docs.modular.com/mojo/roadmap#no-async-for-or-async-with "async with" , # TODO https://docs.modular.com/mojo/roadmap#no-async-for-or-async-with "await" ,"else" ,"for" ,"if" ,"lambda" ,"yield" ,"yield from" ,"\b" ,"True" , "False" , "None" ), suffix=r"\b" ), Keyword.Constant),"keywords" : ["assert" ,"async" ,"await" ,"borrowed" ,"break" ,"continue" ,"del" ,"elif" ,"else" ,"except" ,"finally" ,"for" ,"global" ,"if" ,"lambda" ,"pass" ,"raise" ,"nonlocal" ,"return" ,"try" ,"while" ,"yield" ,"yield from" ,"as" ,"with" ,"\b" ,"True" , "False" , "None" ), suffix=r"\b" ), Keyword.Constant),"soft-keywords" : [# `match`, `case` and `_` soft keywords "(^[ \t]*)" # at beginning of line + possible indentation "(match|case)\b" # a possible keyword "(?![ \t]*(?:" # not followed by... "[:,;=^&|@~)\]}]|(?:" + # characters and keywords that mean this isn't # pattern matching (but None/True/False is ok) "|" .join(k for k in keyword.kwlist if k[0].islower())")\b))" ,"soft-keywords-inner" ,"soft-keywords-inner" : [# optional `_` keyword "(\s+)([^\n_]*)(_\b)" , bygroups(Whitespace, using(this), Keyword)),"#pop"), "builtins" : ["__import__" ,"abs" ,"aiter" ,"all" ,"any" ,"bin" ,"bool" ,"bytearray" ,"breakpoint" ,"bytes" ,"callable" ,"chr" ,"classmethod" ,"compile" ,"complex" ,"delattr" ,"dict" ,"dir" ,"divmod" ,"enumerate" ,"eval" ,"filter" ,"float" ,"format" ,"frozenset" ,"getattr" ,"globals" ,"hasattr" ,"hash" ,"hex" ,"id" ,"input" ,"int" ,"isinstance" ,"issubclass" ,"iter" ,"len" ,"list" ,"locals" ,"map" ,"max" ,"memoryview" ,"min" ,"next" ,"object" ,"oct" ,"open" ,"ord" ,"pow" ,"print" ,"property" ,"range" ,"repr" ,"reversed" ,"round" ,"set" ,"setattr" ,"slice" ,"sorted" ,"staticmethod" ,"str" ,"sum" ,"super" ,"tuple" ,"type" ,"vars" ,"zip" ,# Mojo builtin types: https://docs.modular.com/mojo/stdlib/builtin/ "AnyType" ,"Coroutine" ,"DType" ,"Error" ,"Int" ,"List" ,"ListLiteral" ,"Scalar" ,"Int8" ,"UInt8" ,"Int16" ,"UInt16" ,"Int32" ,"UInt32" ,"Int64" ,"UInt64" ,"BFloat16" ,"Float16" ,"Float32" ,"Float64" ,"SIMD" ,"String" ,"Tensor" ,"Tuple" ,"Movable" ,"Copyable" ,"CollectionElement" ,"(?,"\b" ,"(?, Name.Builtin.Pseudo),"Error" ,),"(?,"\b" ,"magicfuncs" : ["__abs__" ,"__add__" ,"__aenter__" ,"__aexit__" ,"__aiter__" ,"__and__" ,"__anext__" ,"__await__" ,"__bool__" ,"__bytes__" ,"__call__" ,"__complex__" ,"__contains__" ,"__del__" ,"__delattr__" ,"__delete__" ,"__delitem__" ,"__dir__" ,"__divmod__" ,"__enter__" ,"__eq__" ,"__exit__" ,"__float__" ,"__floordiv__" ,"__format__" ,"__ge__" ,"__get__" ,"__getattr__" ,"__getattribute__" ,"__getitem__" ,"__gt__" ,"__hash__" ,"__iadd__" ,"__iand__" ,"__ifloordiv__" ,"__ilshift__" ,"__imatmul__" ,"__imod__" ,"__imul__" ,"__index__" ,"__init__" ,"__instancecheck__" ,"__int__" ,"__invert__" ,"__ior__" ,"__ipow__" ,"__irshift__" ,"__isub__" ,"__iter__" ,"__itruediv__" ,"__ixor__" ,"__le__" ,"__len__" ,"__length_hint__" ,"__lshift__" ,"__lt__" ,"__matmul__" ,"__missing__" ,"__mod__" ,"__mul__" ,"__ne__" ,"__neg__" ,"__new__" ,"__next__" ,"__or__" ,"__pos__" ,"__pow__" ,"__prepare__" ,"__radd__" ,"__rand__" ,"__rdivmod__" ,"__repr__" ,"__reversed__" ,"__rfloordiv__" ,"__rlshift__" ,"__rmatmul__" ,"__rmod__" ,"__rmul__" ,"__ror__" ,"__round__" ,"__rpow__" ,"__rrshift__" ,"__rshift__" ,"__rsub__" ,"__rtruediv__" ,"__rxor__" ,"__set__" ,"__setattr__" ,"__setitem__" ,"__str__" ,"__sub__" ,"__subclasscheck__" ,"__truediv__" ,"__xor__" ,"\b" ,"magicvars" : ["__annotations__" ,"__bases__" ,"__class__" ,"__closure__" ,"__code__" ,"__defaults__" ,"__dict__" ,"__doc__" ,"__file__" ,"__func__" ,"__globals__" ,"__kwdefaults__" ,"__module__" ,"__mro__" ,"__name__" ,"__objclass__" ,"__qualname__" ,"__self__" ,"__slots__" ,"__weakref__" ,"\b" ,"numbers" : ["(\d(?:_?\d)*\.(?:\d(?:_?\d)*)?|(?:\d(?:_?\d)*)?\.\d(?:_?\d)*)" "([eE][+-]?\d(?:_?\d)*)?" ,"\d(?:_?\d)*[eE][+-]?\d(?:_?\d)*j?" , Number.Float),"0[oO](?:_?[0-7])+" , Number.Oct),"0[bB](?:_?[01])+" , Number.Bin),"0[xX](?:_?[a-fA-F0-9])+" , Number.Hex),"\d(?:_?\d)*" , Number.Integer),"name" : ["@" + uni_name, Name.Decorator),"@" , Operator), # new matrix multiplication operator "varname" : ["#pop"), "funcname" : ["magicfuncs" ),"#pop"), "#pop"), "classname" : [Class , "#pop"), "structname" : ["#pop"), "import" : ["(\s+)(as)(\s+)" , bygroups(Whitespace, Keyword, Whitespace)),"\." , Name.Namespace),"(\s*)(,)(\s*)" , bygroups(Whitespace, Operator, Whitespace)),"#pop"), # all else: go back "fromimport" : ["(\s+)(import)\b" , bygroups(Whitespace, Keyword.Namespace), "#pop"), "\." , Name.Namespace),# if None occurs here, it's "raise x from None", since None can # never be a module name "None\b" , Keyword.Constant, "#pop"), "#pop"), "rfstringescape" : ["\{\{" , String.Escape),"\}\}" , String.Escape),"fstringescape" : ["rfstringescape" ),"stringescape" ),"bytesescape" : ['\\([\\abfnrtv"\' ]|\n|x[a-fA-F0-9]{2}|[0-7]{1,3})', String.Escape) "stringescape" : ["\\(N\{.*?\}|u[a-fA-F0-9]{4}|U[a-fA-F0-9]{8})" , String.Escape),"bytesescape" ),"fstrings-single" : fstring_rules(String.Single),"fstrings-double" : fstring_rules(String.Double),"strings-single" : innerstring_rules(String.Single),"strings-double" : innerstring_rules(String.Double),"dqf" : ['"' , String.Double, "#pop"), '\\\\|\\"|\\\n' , String.Escape), # included here for raw strings "fstrings-double" ),"sqf" : ["'" , String.Single, "#pop"), "\\\\|\\'|\\\n" , String.Escape), # included here for raw strings "fstrings-single" ),"dqs" : ['"' , String.Double, "#pop"), '\\\\|\\"|\\\n' , String.Escape), # included here for raw strings "strings-double" ),"sqs" : ["'" , String.Single, "#pop"), "\\\\|\\'|\\\n" , String.Escape), # included here for raw strings "strings-single" ),"tdqf" : ['"""' , String.Double, "#pop"), "fstrings-double" ),"\n" , String.Double),"tsqf" : ["'''" , String.Single, "#pop"), "fstrings-single" ),"\n" , String.Single),"tdqs" : ['"""' , String.Double, "#pop"), "strings-double" ),"\n" , String.Double),"tsqs" : ["'''" , String.Single, "#pop"), "strings-single" ),"\n" , String.Single),def analyse_text(text):return ("mojo?" ) or "import " in text[:1000]# TODO supported? quality 84%
¤ Dauer der Verarbeitung: 0.18 Sekunden
¤ *© Formatika GbR, Deutschland
