language-python-0.2: Parsing and pretty printing of Python code. Source code Contents Index

Language.Python.Common.Token Portability ghc Stability experimental Maintainer bjpop@csse.unimelb.edu.au

Contents The tokens String conversion Classification

Description

Lexical tokens for the Python lexer. Contains the superset of tokens from version 2 and version 3 of Python (they are mostly the same).

Synopsis

data Token = IndentToken { token_span :: !SrcSpan } | DedentToken { token_span :: !SrcSpan } | NewlineToken { token_span :: !SrcSpan } | LineJoinToken { token_span :: !SrcSpan } | CommentToken { token_span :: !SrcSpan token_literal :: !String } | IdentifierToken { token_span :: !SrcSpan token_literal :: !String } | StringToken { token_span :: !SrcSpan token_literal :: !String } | ByteStringToken { token_span :: !SrcSpan token_literal :: !String } | IntegerToken { token_span :: !SrcSpan token_literal :: !String token_integer :: !Integer } | LongIntegerToken { token_span :: !SrcSpan token_literal :: !String token_integer :: !Integer } | FloatToken { token_span :: !SrcSpan token_literal :: !String token_double :: !Double } | ImaginaryToken { token_span :: !SrcSpan token_literal :: !String token_double :: !Double } | DefToken { token_span :: !SrcSpan } | WhileToken { token_span :: !SrcSpan } | IfToken { token_span :: !SrcSpan } | TrueToken { token_span :: !SrcSpan } | FalseToken { token_span :: !SrcSpan } | ReturnToken { token_span :: !SrcSpan } | TryToken { token_span :: !SrcSpan } | ExceptToken { token_span :: !SrcSpan } | RaiseToken { token_span :: !SrcSpan } | InToken { token_span :: !SrcSpan } | IsToken { token_span :: !SrcSpan } | LambdaToken { token_span :: !SrcSpan } | ClassToken { token_span :: !SrcSpan } | FinallyToken { token_span :: !SrcSpan } | NoneToken { token_span :: !SrcSpan } | ForToken { token_span :: !SrcSpan } | FromToken { token_span :: !SrcSpan } | GlobalToken { token_span :: !SrcSpan } | WithToken { token_span :: !SrcSpan } | AsToken { token_span :: !SrcSpan } | ElifToken { token_span :: !SrcSpan } | YieldToken { token_span :: !SrcSpan } | AssertToken { token_span :: !SrcSpan } | ImportToken { token_span :: !SrcSpan } | PassToken { token_span :: !SrcSpan } | BreakToken { token_span :: !SrcSpan } | ContinueToken { token_span :: !SrcSpan } | DeleteToken { token_span :: !SrcSpan } | ElseToken { token_span :: !SrcSpan } | NotToken { token_span :: !SrcSpan } | AndToken { token_span :: !SrcSpan } | OrToken { token_span :: !SrcSpan } | NonLocalToken { token_span :: !SrcSpan } | PrintToken { token_span :: !SrcSpan } | ExecToken { token_span :: !SrcSpan } | AtToken { token_span :: !SrcSpan } | LeftRoundBracketToken { token_span :: !SrcSpan } | RightRoundBracketToken { token_span :: !SrcSpan } | LeftSquareBracketToken { token_span :: !SrcSpan } | RightSquareBracketToken { token_span :: !SrcSpan } | LeftBraceToken { token_span :: !SrcSpan } | RightBraceToken { token_span :: !SrcSpan } | DotToken { token_span :: !SrcSpan } | CommaToken { token_span :: !SrcSpan } | SemiColonToken { token_span :: !SrcSpan } | ColonToken { token_span :: !SrcSpan } | EllipsisToken { token_span :: !SrcSpan } | RightArrowToken { token_span :: !SrcSpan } | AssignToken { token_span :: !SrcSpan } | PlusAssignToken { token_span :: !SrcSpan } | MinusAssignToken { token_span :: !SrcSpan } | MultAssignToken { token_span :: !SrcSpan } | DivAssignToken { token_span :: !SrcSpan } | ModAssignToken { token_span :: !SrcSpan } | PowAssignToken { token_span :: !SrcSpan } | BinAndAssignToken { token_span :: !SrcSpan } | BinOrAssignToken { token_span :: !SrcSpan } | BinXorAssignToken { token_span :: !SrcSpan } | LeftShiftAssignToken { token_span :: !SrcSpan } | RightShiftAssignToken { token_span :: !SrcSpan } | FloorDivAssignToken { token_span :: !SrcSpan } | BackQuoteToken { token_span :: !SrcSpan } | PlusToken { token_span :: !SrcSpan } | MinusToken { token_span :: !SrcSpan } | MultToken { token_span :: !SrcSpan } | DivToken { token_span :: !SrcSpan } | GreaterThanToken { token_span :: !SrcSpan } | LessThanToken { token_span :: !SrcSpan } | EqualityToken { token_span :: !SrcSpan } | GreaterThanEqualsToken { token_span :: !SrcSpan } | LessThanEqualsToken { token_span :: !SrcSpan } | ExponentToken { token_span :: !SrcSpan } | BinaryOrToken { token_span :: !SrcSpan } | XorToken { token_span :: !SrcSpan } | BinaryAndToken { token_span :: !SrcSpan } | ShiftLeftToken { token_span :: !SrcSpan } | ShiftRightToken { token_span :: !SrcSpan } | ModuloToken { token_span :: !SrcSpan } | FloorDivToken { token_span :: !SrcSpan } | TildeToken { token_span :: !SrcSpan } | NotEqualsToken { token_span :: !SrcSpan } | NotEqualsV2Token { token_span :: !SrcSpan } | EOFToken { token_span :: !SrcSpan } debugTokenString :: Token -> String tokenString :: Token -> String hasLiteral :: Token -> Bool data TokenClass = Comment | Number | Identifier | Punctuation | Bracket | Layout | Keyword | String | Operator | Assignment classifyToken :: Token -> TokenClass

The tokens

data Token Source

Lexical tokens. Constructors IndentToken Indentation: increase. token_span :: !SrcSpan DedentToken Indentation: decrease. token_span :: !SrcSpan NewlineToken Newline. token_span :: !SrcSpan LineJoinToken Line join (backslash at end of line). token_span :: !SrcSpan CommentToken Single line comment. token_span :: !SrcSpan token_literal :: !String IdentifierToken Identifier. token_span :: !SrcSpan token_literal :: !String StringToken Literal: string. token_span :: !SrcSpan token_literal :: !String ByteStringToken Literal: byte string. token_span :: !SrcSpan token_literal :: !String IntegerToken Literal: integer. token_span :: !SrcSpan token_literal :: !String token_integer :: !Integer LongIntegerToken Literal: long integer. Version 2 only. token_span :: !SrcSpan token_literal :: !String token_integer :: !Integer FloatToken Literal: floating point. token_span :: !SrcSpan token_literal :: !String token_double :: !Double ImaginaryToken Literal: imaginary number. token_span :: !SrcSpan token_literal :: !String token_double :: !Double DefToken Keyword: 'def'. token_span :: !SrcSpan WhileToken Keyword: 'while'. token_span :: !SrcSpan IfToken Keyword: 'if'. token_span :: !SrcSpan TrueToken Keyword: 'True'. token_span :: !SrcSpan FalseToken Keyword: 'False'. token_span :: !SrcSpan ReturnToken Keyword: 'Return'. token_span :: !SrcSpan TryToken Keyword: 'try'. token_span :: !SrcSpan ExceptToken Keyword: 'except'. token_span :: !SrcSpan RaiseToken Keyword: 'raise'. token_span :: !SrcSpan InToken Keyword: 'in'. token_span :: !SrcSpan IsToken Keyword: 'is'. token_span :: !SrcSpan LambdaToken Keyword: 'lambda'. token_span :: !SrcSpan ClassToken Keyword: 'class'. token_span :: !SrcSpan FinallyToken Keyword: 'finally'. token_span :: !SrcSpan NoneToken Keyword: 'None'. token_span :: !SrcSpan ForToken Keyword: 'for'. token_span :: !SrcSpan FromToken Keyword: 'from'. token_span :: !SrcSpan GlobalToken Keyword: 'global'. token_span :: !SrcSpan WithToken Keyword: 'with'. token_span :: !SrcSpan AsToken Keyword: 'as'. token_span :: !SrcSpan ElifToken Keyword: 'elif'. token_span :: !SrcSpan YieldToken Keyword: 'yield'. token_span :: !SrcSpan AssertToken Keyword: 'assert'. token_span :: !SrcSpan ImportToken Keyword: 'import'. token_span :: !SrcSpan PassToken Keyword: 'pass'. token_span :: !SrcSpan BreakToken Keyword: 'break'. token_span :: !SrcSpan ContinueToken Keyword: 'continue'. token_span :: !SrcSpan DeleteToken Keyword: 'del'. token_span :: !SrcSpan ElseToken Keyword: 'else'. token_span :: !SrcSpan NotToken Keyword: 'not'. token_span :: !SrcSpan AndToken Keyword: boolean conjunction 'and'. token_span :: !SrcSpan OrToken Keyword: boolean disjunction 'or'. Version 3.x only: token_span :: !SrcSpan NonLocalToken Keyword: 'nonlocal' (Python 3.x only) Version 2.x only: token_span :: !SrcSpan PrintToken Keyword: 'print'. (Python 2.x only) token_span :: !SrcSpan ExecToken Keyword: 'exec'. (Python 2.x only) token_span :: !SrcSpan AtToken Delimiter: at sign '@'. token_span :: !SrcSpan LeftRoundBracketToken Delimiter: left round bracket '('. token_span :: !SrcSpan RightRoundBracketToken Delimiter: right round bracket ')'. token_span :: !SrcSpan LeftSquareBracketToken Delimiter: left square bracket '['. token_span :: !SrcSpan RightSquareBracketToken Delimiter: right square bracket ']'. token_span :: !SrcSpan LeftBraceToken Delimiter: left curly bracket '{'. token_span :: !SrcSpan RightBraceToken Delimiter: right curly bracket '}'. token_span :: !SrcSpan DotToken Delimiter: dot (full stop) '.'. token_span :: !SrcSpan CommaToken Delimiter: comma ','. token_span :: !SrcSpan SemiColonToken Delimiter: semicolon ';'. token_span :: !SrcSpan ColonToken Delimiter: colon ':'. token_span :: !SrcSpan EllipsisToken Delimiter: ellipses (three dots) '...'. token_span :: !SrcSpan RightArrowToken Delimiter: right facing arrow '->'. token_span :: !SrcSpan AssignToken Delimiter: assignment '='. token_span :: !SrcSpan PlusAssignToken Delimiter: plus assignment '+='. token_span :: !SrcSpan MinusAssignToken Delimiter: minus assignment '-='. token_span :: !SrcSpan MultAssignToken Delimiter: multiply assignment '*=' token_span :: !SrcSpan DivAssignToken Delimiter: divide assignment '/='. token_span :: !SrcSpan ModAssignToken Delimiter: modulus assignment '%='. token_span :: !SrcSpan PowAssignToken Delimiter: power assignment '**='. token_span :: !SrcSpan BinAndAssignToken Delimiter: binary-and assignment '&='. token_span :: !SrcSpan BinOrAssignToken Delimiter: binary-or assignment '|='. token_span :: !SrcSpan BinXorAssignToken Delimiter: binary-xor assignment '^='. token_span :: !SrcSpan LeftShiftAssignToken Delimiter: binary-left-shift assignment '<<='. token_span :: !SrcSpan RightShiftAssignToken Delimiter: binary-right-shift assignment '>>='. token_span :: !SrcSpan FloorDivAssignToken Delimiter: floor-divide assignment '='. token_span :: !SrcSpan BackQuoteToken Delimiter: back quote character '`\'. token_span :: !SrcSpan PlusToken Operator: plus '+'. token_span :: !SrcSpan MinusToken Operator: minus: '-'. token_span :: !SrcSpan MultToken Operator: multiply '*'. token_span :: !SrcSpan DivToken Operator: divide '/'. token_span :: !SrcSpan GreaterThanToken Operator: greater-than '>'. token_span :: !SrcSpan LessThanToken Operator: less-than '<'. token_span :: !SrcSpan EqualityToken Operator: equals '=='. token_span :: !SrcSpan GreaterThanEqualsToken Operator: greater-than-or-equals '>='. token_span :: !SrcSpan LessThanEqualsToken Operator: less-than-or-equals '<='. token_span :: !SrcSpan ExponentToken Operator: exponential '**'. token_span :: !SrcSpan BinaryOrToken Operator: binary-or '|'. token_span :: !SrcSpan XorToken Operator: binary-xor '^'. token_span :: !SrcSpan BinaryAndToken Operator: binary-and '&'. token_span :: !SrcSpan ShiftLeftToken Operator: binary-shift-left '<<'. token_span :: !SrcSpan ShiftRightToken Operator: binary-shift-right '>>'. token_span :: !SrcSpan ModuloToken Operator: modulus '%'. token_span :: !SrcSpan FloorDivToken Operator: floor-divide ''. token_span :: !SrcSpan TildeToken Operator: tilde '~'. token_span :: !SrcSpan NotEqualsToken Operator: not-equals '!='. token_span :: !SrcSpan NotEqualsV2Token Operator: not-equals ''. Version 2 only. token_span :: !SrcSpan EOFToken End of file token_span :: !SrcSpan Instances Eq Token Data Token Ord Token Show Token Typeable Token Pretty Token Span Token

String conversion

debugTokenString :: Token -> String Source

Produce a string from a token containing detailed information. Mainly intended for debugging.

tokenString :: Token -> String Source

Produce a string from a token which is suitable for printing as Python concrete syntax. Invisible tokens yield an empty string.

Classification

hasLiteral :: Token -> Bool Source

Test if a token contains its literal source text.

data TokenClass Source

Classification of tokens Constructors Comment Number Identifier Punctuation Bracket Layout Keyword String Operator Assignment Instances Eq TokenClass Ord TokenClass Show TokenClass

classifyToken :: Token -> TokenClass Source

Produced by Haddock version 2.4.2