Copyright	(c) 2009 Bernie Pope
License	BSD-style
Maintainer	bjpop@csse.unimelb.edu.au
Stability	experimental
Portability	ghc
Safe Haskell	Safe
Language	Haskell2010

Language.Python.Common.Token

Contents

The tokens
String conversion
Classification

Description

Lexical tokens for the Python lexer. Contains the superset of tokens from version 2 and version 3 of Python (they are mostly the same).

Synopsis

data Token
- = IndentToken {
  - token_span :: !SrcSpan
  }
- | DedentToken {
  - token_span :: !SrcSpan
  }
- | NewlineToken {
  - token_span :: !SrcSpan
  }
- | LineJoinToken {
  - token_span :: !SrcSpan
  }
- | CommentToken {
  - token_span :: !SrcSpan
  - token_literal :: !String
  }
- | IdentifierToken {
  - token_span :: !SrcSpan
  - token_literal :: !String
  }
- | StringToken {
  - token_span :: !SrcSpan
  - token_literal :: !String
  }
- | ByteStringToken {
  - token_span :: !SrcSpan
  - token_literal :: !String
  }
- | UnicodeStringToken {
  - token_span :: !SrcSpan
  - token_literal :: !String
  }
- | IntegerToken {
  - token_span :: !SrcSpan
  - token_literal :: !String
  - token_integer :: !Integer
  }
- | LongIntegerToken {
  - token_span :: !SrcSpan
  - token_literal :: !String
  - token_integer :: !Integer
  }
- | FloatToken {
  - token_span :: !SrcSpan
  - token_literal :: !String
  - token_double :: !Double
  }
- | ImaginaryToken {
  - token_span :: !SrcSpan
  - token_literal :: !String
  - token_double :: !Double
  }
- | DefToken {
  - token_span :: !SrcSpan
  }
- | WhileToken {
  - token_span :: !SrcSpan
  }
- | IfToken {
  - token_span :: !SrcSpan
  }
- | TrueToken {
  - token_span :: !SrcSpan
  }
- | FalseToken {
  - token_span :: !SrcSpan
  }
- | ReturnToken {
  - token_span :: !SrcSpan
  }
- | TryToken {
  - token_span :: !SrcSpan
  }
- | ExceptToken {
  - token_span :: !SrcSpan
  }
- | RaiseToken {
  - token_span :: !SrcSpan
  }
- | InToken {
  - token_span :: !SrcSpan
  }
- | IsToken {
  - token_span :: !SrcSpan
  }
- | LambdaToken {
  - token_span :: !SrcSpan
  }
- | ClassToken {
  - token_span :: !SrcSpan
  }
- | FinallyToken {
  - token_span :: !SrcSpan
  }
- | NoneToken {
  - token_span :: !SrcSpan
  }
- | ForToken {
  - token_span :: !SrcSpan
  }
- | FromToken {
  - token_span :: !SrcSpan
  }
- | GlobalToken {
  - token_span :: !SrcSpan
  }
- | WithToken {
  - token_span :: !SrcSpan
  }
- | AsToken {
  - token_span :: !SrcSpan
  }
- | ElifToken {
  - token_span :: !SrcSpan
  }
- | YieldToken {
  - token_span :: !SrcSpan
  }
- | AssertToken {
  - token_span :: !SrcSpan
  }
- | ImportToken {
  - token_span :: !SrcSpan
  }
- | PassToken {
  - token_span :: !SrcSpan
  }
- | BreakToken {
  - token_span :: !SrcSpan
  }
- | ContinueToken {
  - token_span :: !SrcSpan
  }
- | DeleteToken {
  - token_span :: !SrcSpan
  }
- | ElseToken {
  - token_span :: !SrcSpan
  }
- | NotToken {
  - token_span :: !SrcSpan
  }
- | AndToken {
  - token_span :: !SrcSpan
  }
- | OrToken {
  - token_span :: !SrcSpan
  }
- | NonLocalToken {
  - token_span :: !SrcSpan
  }
- | AsyncToken {
  - token_span :: !SrcSpan
  }
- | AwaitToken {
  - token_span :: !SrcSpan
  }
- | PrintToken {
  - token_span :: !SrcSpan
  }
- | ExecToken {
  - token_span :: !SrcSpan
  }
- | AtToken {
  - token_span :: !SrcSpan
  }
- | LeftRoundBracketToken {
  - token_span :: !SrcSpan
  }
- | RightRoundBracketToken {
  - token_span :: !SrcSpan
  }
- | LeftSquareBracketToken {
  - token_span :: !SrcSpan
  }
- | RightSquareBracketToken {
  - token_span :: !SrcSpan
  }
- | LeftBraceToken {
  - token_span :: !SrcSpan
  }
- | RightBraceToken {
  - token_span :: !SrcSpan
  }
- | DotToken {
  - token_span :: !SrcSpan
  }
- | CommaToken {
  - token_span :: !SrcSpan
  }
- | SemiColonToken {
  - token_span :: !SrcSpan
  }
- | ColonToken {
  - token_span :: !SrcSpan
  }
- | EllipsisToken {
  - token_span :: !SrcSpan
  }
- | RightArrowToken {
  - token_span :: !SrcSpan
  }
- | AssignToken {
  - token_span :: !SrcSpan
  }
- | PlusAssignToken {
  - token_span :: !SrcSpan
  }
- | MinusAssignToken {
  - token_span :: !SrcSpan
  }
- | MultAssignToken {
  - token_span :: !SrcSpan
  }
- | DivAssignToken {
  - token_span :: !SrcSpan
  }
- | ModAssignToken {
  - token_span :: !SrcSpan
  }
- | PowAssignToken {
  - token_span :: !SrcSpan
  }
- | BinAndAssignToken {
  - token_span :: !SrcSpan
  }
- | BinOrAssignToken {
  - token_span :: !SrcSpan
  }
- | BinXorAssignToken {
  - token_span :: !SrcSpan
  }
- | LeftShiftAssignToken {
  - token_span :: !SrcSpan
  }
- | RightShiftAssignToken {
  - token_span :: !SrcSpan
  }
- | FloorDivAssignToken {
  - token_span :: !SrcSpan
  }
- | MatrixMultAssignToken {
  - token_span :: !SrcSpan
  }
- | BackQuoteToken {
  - token_span :: !SrcSpan
  }
- | PlusToken {
  - token_span :: !SrcSpan
  }
- | MinusToken {
  - token_span :: !SrcSpan
  }
- | MultToken {
  - token_span :: !SrcSpan
  }
- | DivToken {
  - token_span :: !SrcSpan
  }
- | GreaterThanToken {
  - token_span :: !SrcSpan
  }
- | LessThanToken {
  - token_span :: !SrcSpan
  }
- | EqualityToken {
  - token_span :: !SrcSpan
  }
- | GreaterThanEqualsToken {
  - token_span :: !SrcSpan
  }
- | LessThanEqualsToken {
  - token_span :: !SrcSpan
  }
- | ExponentToken {
  - token_span :: !SrcSpan
  }
- | BinaryOrToken {
  - token_span :: !SrcSpan
  }
- | XorToken {
  - token_span :: !SrcSpan
  }
- | BinaryAndToken {
  - token_span :: !SrcSpan
  }
- | ShiftLeftToken {
  - token_span :: !SrcSpan
  }
- | ShiftRightToken {
  - token_span :: !SrcSpan
  }
- | ModuloToken {
  - token_span :: !SrcSpan
  }
- | FloorDivToken {
  - token_span :: !SrcSpan
  }
- | TildeToken {
  - token_span :: !SrcSpan
  }
- | NotEqualsToken {
  - token_span :: !SrcSpan
  }
- | NotEqualsV2Token {
  - token_span :: !SrcSpan
  }
- | EOFToken {
  - token_span :: !SrcSpan
  }
debugTokenString :: Token -> String
tokenString :: Token -> String
hasLiteral :: Token -> Bool
data TokenClass
- = Comment
- | Number
- | Identifier
- | Punctuation
- | Bracket
- | Layout
- | Keyword
- | String
- | Operator
- | Assignment
classifyToken :: Token -> TokenClass

The tokens

data Token Source #

Lexical tokens.

Constructors

IndentToken	Indentation: increase.
Fields token_span :: !SrcSpan
DedentToken	Indentation: decrease.
Fields token_span :: !SrcSpan
NewlineToken	Newline.
Fields token_span :: !SrcSpan
LineJoinToken	Line join (backslash at end of line).
Fields token_span :: !SrcSpan
CommentToken	Single line comment.
Fields token_span :: !SrcSpan token_literal :: !String
IdentifierToken	Identifier.
Fields token_span :: !SrcSpan token_literal :: !String
StringToken	Literal: string.
Fields token_span :: !SrcSpan token_literal :: !String
ByteStringToken	Literal: byte string.
Fields token_span :: !SrcSpan token_literal :: !String
UnicodeStringToken	Literal: unicode string, version 2 only.
Fields token_span :: !SrcSpan token_literal :: !String
IntegerToken	Literal: integer.
Fields token_span :: !SrcSpan token_literal :: !String token_integer :: !Integer
LongIntegerToken	Literal: long integer. Version 2 only.
Fields token_span :: !SrcSpan token_literal :: !String token_integer :: !Integer
FloatToken	Literal: floating point.
Fields token_span :: !SrcSpan token_literal :: !String token_double :: !Double
ImaginaryToken	Literal: imaginary number.
Fields token_span :: !SrcSpan token_literal :: !String token_double :: !Double
DefToken	Keyword: 'def'.
Fields token_span :: !SrcSpan
WhileToken	Keyword: 'while'.
Fields token_span :: !SrcSpan
IfToken	Keyword: 'if'.
Fields token_span :: !SrcSpan
TrueToken	Keyword: 'True'.
Fields token_span :: !SrcSpan
FalseToken	Keyword: 'False'.
Fields token_span :: !SrcSpan
ReturnToken	Keyword: 'Return'.
Fields token_span :: !SrcSpan
TryToken	Keyword: 'try'.
Fields token_span :: !SrcSpan
ExceptToken	Keyword: 'except'.
Fields token_span :: !SrcSpan
RaiseToken	Keyword: 'raise'.
Fields token_span :: !SrcSpan
InToken	Keyword: 'in'.
Fields token_span :: !SrcSpan
IsToken	Keyword: 'is'.
Fields token_span :: !SrcSpan
LambdaToken	Keyword: 'lambda'.
Fields token_span :: !SrcSpan
ClassToken	Keyword: 'class'.
Fields token_span :: !SrcSpan
FinallyToken	Keyword: 'finally'.
Fields token_span :: !SrcSpan
NoneToken	Keyword: 'None'.
Fields token_span :: !SrcSpan
ForToken	Keyword: 'for'.
Fields token_span :: !SrcSpan
FromToken	Keyword: 'from'.
Fields token_span :: !SrcSpan
GlobalToken	Keyword: 'global'.
Fields token_span :: !SrcSpan
WithToken	Keyword: 'with'.
Fields token_span :: !SrcSpan
AsToken	Keyword: 'as'.
Fields token_span :: !SrcSpan
ElifToken	Keyword: 'elif'.
Fields token_span :: !SrcSpan
YieldToken	Keyword: 'yield'.
Fields token_span :: !SrcSpan
AssertToken	Keyword: 'assert'.
Fields token_span :: !SrcSpan
ImportToken	Keyword: 'import'.
Fields token_span :: !SrcSpan
PassToken	Keyword: 'pass'.
Fields token_span :: !SrcSpan
BreakToken	Keyword: 'break'.
Fields token_span :: !SrcSpan
ContinueToken	Keyword: 'continue'.
Fields token_span :: !SrcSpan
DeleteToken	Keyword: 'del'.
Fields token_span :: !SrcSpan
ElseToken	Keyword: 'else'.
Fields token_span :: !SrcSpan
NotToken	Keyword: 'not'.
Fields token_span :: !SrcSpan
AndToken	Keyword: boolean conjunction 'and'.
Fields token_span :: !SrcSpan
OrToken	Keyword: boolean disjunction 'or'. Version 3.x only:
Fields token_span :: !SrcSpan
NonLocalToken	Keyword: 'nonlocal' (Python 3.x only)
Fields token_span :: !SrcSpan
AsyncToken	Keyword: 'async' (Python 3.x only)
Fields token_span :: !SrcSpan
AwaitToken	Keyword: 'await' (Python 3.x only) Version 2.x only:
Fields token_span :: !SrcSpan
PrintToken	Keyword: 'print'. (Python 2.x only)
Fields token_span :: !SrcSpan
ExecToken	Keyword: 'exec'. (Python 2.x only)
Fields token_span :: !SrcSpan
AtToken	Delimiter: at sign '@'.
Fields token_span :: !SrcSpan
LeftRoundBracketToken	Delimiter: left round bracket '('.
Fields token_span :: !SrcSpan
RightRoundBracketToken	Delimiter: right round bracket ')'.
Fields token_span :: !SrcSpan
LeftSquareBracketToken	Delimiter: left square bracket '['.
Fields token_span :: !SrcSpan
RightSquareBracketToken	Delimiter: right square bracket ']'.
Fields token_span :: !SrcSpan
LeftBraceToken	Delimiter: left curly bracket '{'.
Fields token_span :: !SrcSpan
RightBraceToken	Delimiter: right curly bracket '}'.
Fields token_span :: !SrcSpan
DotToken	Delimiter: dot (full stop) '.'.
Fields token_span :: !SrcSpan
CommaToken	Delimiter: comma ','.
Fields token_span :: !SrcSpan
SemiColonToken	Delimiter: semicolon ';'.
Fields token_span :: !SrcSpan
ColonToken	Delimiter: colon ':'.
Fields token_span :: !SrcSpan
EllipsisToken	Delimiter: ellipses (three dots) '...'.
Fields token_span :: !SrcSpan
RightArrowToken	Delimiter: right facing arrow '->'.
Fields token_span :: !SrcSpan
AssignToken	Delimiter: assignment '='.
Fields token_span :: !SrcSpan
PlusAssignToken	Delimiter: plus assignment '+='.
Fields token_span :: !SrcSpan
MinusAssignToken	Delimiter: minus assignment '-='.
Fields token_span :: !SrcSpan
MultAssignToken	Delimiter: multiply assignment '*='
Fields token_span :: !SrcSpan
DivAssignToken	Delimiter: divide assignment '/='.
Fields token_span :: !SrcSpan
ModAssignToken	Delimiter: modulus assignment '%='.
Fields token_span :: !SrcSpan
PowAssignToken	Delimiter: power assignment '**='.
Fields token_span :: !SrcSpan
BinAndAssignToken	Delimiter: binary-and assignment '&='.
Fields token_span :: !SrcSpan
BinOrAssignToken	Delimiter: binary-or assignment '\|='.
Fields token_span :: !SrcSpan
BinXorAssignToken	Delimiter: binary-xor assignment '^='.
Fields token_span :: !SrcSpan
LeftShiftAssignToken	Delimiter: binary-left-shift assignment '<<='.
Fields token_span :: !SrcSpan
RightShiftAssignToken	Delimiter: binary-right-shift assignment '>>='.
Fields token_span :: !SrcSpan
FloorDivAssignToken	Delimiter: floor-divide assignment '//='.
Fields token_span :: !SrcSpan
MatrixMultAssignToken	Delimiter: matrix multiplication assignment '@='.
Fields token_span :: !SrcSpan
BackQuoteToken	Delimiter: back quote character '`\'.
Fields token_span :: !SrcSpan
PlusToken	Operator: plus '+'.
Fields token_span :: !SrcSpan
MinusToken	Operator: minus: '-'.
Fields token_span :: !SrcSpan
MultToken	Operator: multiply '*'.
Fields token_span :: !SrcSpan
DivToken	Operator: divide '/'.
Fields token_span :: !SrcSpan
GreaterThanToken	Operator: greater-than '>'.
Fields token_span :: !SrcSpan
LessThanToken	Operator: less-than '<'.
Fields token_span :: !SrcSpan
EqualityToken	Operator: equals '=='.
Fields token_span :: !SrcSpan
GreaterThanEqualsToken	Operator: greater-than-or-equals '>='.
Fields token_span :: !SrcSpan
LessThanEqualsToken	Operator: less-than-or-equals '<='.
Fields token_span :: !SrcSpan
ExponentToken	Operator: exponential '**'.
Fields token_span :: !SrcSpan
BinaryOrToken	Operator: binary-or '\|'.
Fields token_span :: !SrcSpan
XorToken	Operator: binary-xor '^'.
Fields token_span :: !SrcSpan
BinaryAndToken	Operator: binary-and '&'.
Fields token_span :: !SrcSpan
ShiftLeftToken	Operator: binary-shift-left '<<'.
Fields token_span :: !SrcSpan
ShiftRightToken	Operator: binary-shift-right '>>'.
Fields token_span :: !SrcSpan
ModuloToken	Operator: modulus '%'.
Fields token_span :: !SrcSpan
FloorDivToken	Operator: floor-divide '//'.
Fields token_span :: !SrcSpan
TildeToken	Operator: tilde '~'.
Fields token_span :: !SrcSpan
NotEqualsToken	Operator: not-equals '!='.
Fields token_span :: !SrcSpan
NotEqualsV2Token	Operator: not-equals '<>'. Version 2 only.
Fields token_span :: !SrcSpan
EOFToken	End of file
Fields token_span :: !SrcSpan

Instances

Eq Token Source #
Instance details Defined in Language.Python.Common.Token Methods (==) :: Token -> Token -> Bool # (/=) :: Token -> Token -> Bool #
Data Token Source #
Instance details Defined in Language.Python.Common.Token Methods gfoldl :: (forall d b. Data d => c (d -> b) -> d -> c b) -> (forall g. g -> c g) -> Token -> c Token # gunfold :: (forall b r. Data b => c (b -> r) -> c r) -> (forall r. r -> c r) -> Constr -> c Token # toConstr :: Token -> Constr # dataTypeOf :: Token -> DataType # dataCast1 :: Typeable t => (forall d. Data d => c (t d)) -> Maybe (c Token) # dataCast2 :: Typeable t => (forall d e. (Data d, Data e) => c (t d e)) -> Maybe (c Token) # gmapT :: (forall b. Data b => b -> b) -> Token -> Token # gmapQl :: (r -> r' -> r) -> r -> (forall d. Data d => d -> r') -> Token -> r # gmapQr :: (r' -> r -> r) -> r -> (forall d. Data d => d -> r') -> Token -> r # gmapQ :: (forall d. Data d => d -> u) -> Token -> [u] # gmapQi :: Int -> (forall d. Data d => d -> u) -> Token -> u # gmapM :: Monad m => (forall d. Data d => d -> m d) -> Token -> m Token # gmapMp :: MonadPlus m => (forall d. Data d => d -> m d) -> Token -> m Token # gmapMo :: MonadPlus m => (forall d. Data d => d -> m d) -> Token -> m Token #
Ord Token Source #
Instance details Defined in Language.Python.Common.Token Methods compare :: Token -> Token -> Ordering # (<) :: Token -> Token -> Bool # (<=) :: Token -> Token -> Bool # (>) :: Token -> Token -> Bool # (>=) :: Token -> Token -> Bool # max :: Token -> Token -> Token # min :: Token -> Token -> Token #
Show Token Source #
Instance details Defined in Language.Python.Common.Token Methods showsPrec :: Int -> Token -> ShowS # show :: Token -> String # showList :: [Token] -> ShowS #
Pretty Token Source #
Instance details Defined in Language.Python.Common.PrettyToken Methods pretty :: Token -> Doc Source #
Span Token Source #
Instance details Defined in Language.Python.Common.Token Methods getSpan :: Token -> SrcSpan Source #

String conversion

debugTokenString :: Token -> String Source #

Produce a string from a token containing detailed information. Mainly intended for debugging.

tokenString :: Token -> String Source #

Produce a string from a token which is suitable for printing as Python concrete syntax. Invisible tokens yield an empty string.

Classification

hasLiteral :: Token -> Bool Source #

Test if a token contains its literal source text.

data TokenClass Source #

Classification of tokens

Constructors

Comment
Number
Identifier
Punctuation
Bracket
Layout
Keyword
String
Operator
Assignment

Instances

Eq TokenClass Source #
Instance details Defined in Language.Python.Common.Token Methods (==) :: TokenClass -> TokenClass -> Bool # (/=) :: TokenClass -> TokenClass -> Bool #
Ord TokenClass Source #
Instance details Defined in Language.Python.Common.Token Methods compare :: TokenClass -> TokenClass -> Ordering # (<) :: TokenClass -> TokenClass -> Bool # (<=) :: TokenClass -> TokenClass -> Bool # (>) :: TokenClass -> TokenClass -> Bool # (>=) :: TokenClass -> TokenClass -> Bool # max :: TokenClass -> TokenClass -> TokenClass # min :: TokenClass -> TokenClass -> TokenClass #
Show TokenClass Source #
Instance details Defined in Language.Python.Common.Token Methods showsPrec :: Int -> TokenClass -> ShowS # show :: TokenClass -> String # showList :: [TokenClass] -> ShowS #

classifyToken :: Token -> TokenClass Source #