{-# LANGUAGE CPP, DeriveDataTypeable #-}
-----------------------------------------------------------------------------
-- |
-- Module      : Language.Python.Common.Token 
-- Copyright   : (c) 2009 Bernie Pope 
-- License     : BSD-style
-- Maintainer  : bjpop@csse.unimelb.edu.au
-- Stability   : experimental
-- Portability : ghc
--
-- Lexical tokens for the Python lexer. Contains the superset of tokens from
-- version 2 and version 3 of Python (they are mostly the same).
-----------------------------------------------------------------------------

module Language.Python.Common.Token (
   -- * The tokens 
   Token (..),
   -- * String conversion
   debugTokenString,
   tokenString,
   -- * Classification
   hasLiteral,
   TokenClass (..),
   classifyToken
   ) where

import Language.Python.Common.Pretty
import Language.Python.Common.SrcLocation (SrcSpan (..), Span(getSpan))
import Data.Data

-- | Lexical tokens.
data Token
   -- Whitespace
   = IndentToken { token_span :: !SrcSpan }                       -- ^ Indentation: increase.
   | DedentToken { token_span :: !SrcSpan }                       -- ^ Indentation: decrease.
   | NewlineToken { token_span :: !SrcSpan }                      -- ^ Newline.
   | LineJoinToken { token_span :: !SrcSpan }                     -- ^ Line join (backslash at end of line).

   -- Comment
   | CommentToken { token_span :: !SrcSpan, token_literal :: !String } -- ^ Single line comment.

   -- Identifiers 
   | IdentifierToken { token_span :: !SrcSpan, token_literal :: !String }            -- ^ Identifier.

   -- Literals
   | StringToken { token_span :: !SrcSpan, token_literal :: !String }                   -- ^ Literal: string.
   | ByteStringToken { token_span :: !SrcSpan, token_literal :: !String }    -- ^ Literal: byte string.
   | UnicodeStringToken { token_span :: !SrcSpan, token_literal :: !String } -- ^ Literal: unicode string, version 2 only.
   | IntegerToken { token_span :: !SrcSpan, token_literal :: !String, token_integer :: !Integer }                 -- ^ Literal: integer.
   | LongIntegerToken { token_span :: !SrcSpan, token_literal :: !String, token_integer :: !Integer }             -- ^ Literal: long integer. /Version 2 only/.
   | FloatToken { token_span :: !SrcSpan, token_literal :: !String, token_double :: !Double }                     -- ^ Literal: floating point.
   | ImaginaryToken { token_span :: !SrcSpan, token_literal :: !String, token_double :: !Double }                 -- ^ Literal: imaginary number.

   -- Keywords
   | DefToken { token_span :: !SrcSpan }                          -- ^ Keyword: \'def\'. 
   | WhileToken { token_span :: !SrcSpan }                        -- ^ Keyword: \'while\'.
   | IfToken { token_span :: !SrcSpan }                           -- ^ Keyword: \'if\'.
   | TrueToken { token_span :: !SrcSpan }                         -- ^ Keyword: \'True\'.
   | FalseToken { token_span :: !SrcSpan }                        -- ^ Keyword: \'False\'.
   | ReturnToken { token_span :: !SrcSpan }                       -- ^ Keyword: \'Return\'.
   | TryToken { token_span :: !SrcSpan }                          -- ^ Keyword: \'try\'.
   | ExceptToken { token_span :: !SrcSpan }                       -- ^ Keyword: \'except\'.
   | RaiseToken { token_span :: !SrcSpan }                        -- ^ Keyword: \'raise\'.
   | InToken { token_span :: !SrcSpan }                           -- ^ Keyword: \'in\'.
   | IsToken { token_span :: !SrcSpan }                           -- ^ Keyword: \'is\'.
   | LambdaToken { token_span :: !SrcSpan }                       -- ^ Keyword: \'lambda\'.
   | ClassToken { token_span :: !SrcSpan }                        -- ^ Keyword: \'class\'.
   | FinallyToken { token_span :: !SrcSpan }                      -- ^ Keyword: \'finally\'.
   | NoneToken { token_span :: !SrcSpan }                         -- ^ Keyword: \'None\'. 
   | ForToken { token_span :: !SrcSpan }                          -- ^ Keyword: \'for\'.
   | FromToken { token_span :: !SrcSpan }                         -- ^ Keyword: \'from\'.
   | GlobalToken { token_span :: !SrcSpan }                       -- ^ Keyword: \'global\'.
   | WithToken { token_span :: !SrcSpan }                         -- ^ Keyword: \'with\'.
   | AsToken { token_span :: !SrcSpan }                           -- ^ Keyword: \'as\'.
   | ElifToken { token_span :: !SrcSpan }                         -- ^ Keyword: \'elif\'.
   | YieldToken { token_span :: !SrcSpan }                        -- ^ Keyword: \'yield\'.
   | AssertToken { token_span :: !SrcSpan }                       -- ^ Keyword: \'assert\'.
   | ImportToken { token_span :: !SrcSpan }                       -- ^ Keyword: \'import\'.
   | PassToken { token_span :: !SrcSpan }                         -- ^ Keyword: \'pass\'.
   | BreakToken { token_span :: !SrcSpan }                        -- ^ Keyword: \'break\'.
   | ContinueToken { token_span :: !SrcSpan }                     -- ^ Keyword: \'continue\'.
   | DeleteToken { token_span :: !SrcSpan }                       -- ^ Keyword: \'del\'.
   | ElseToken { token_span :: !SrcSpan }                         -- ^ Keyword: \'else\'.
   | NotToken { token_span :: !SrcSpan }                          -- ^ Keyword: \'not\'.
   | AndToken { token_span :: !SrcSpan }                          -- ^ Keyword: boolean conjunction \'and\'.
   | OrToken { token_span :: !SrcSpan }                           -- ^ Keyword: boolean disjunction \'or\'.
   -- Version 3.x only:
   | NonLocalToken { token_span :: !SrcSpan }                     -- ^ Keyword: \'nonlocal\' (Python 3.x only)
   | AsyncToken { token_span :: !SrcSpan }                        -- ^ Keyword: \'async\' (Python 3.x only)
   | AwaitToken { token_span :: !SrcSpan }                        -- ^ Keyword: \'await\' (Python 3.x only)
   -- Version 2.x only:
   | PrintToken { token_span :: !SrcSpan }                        -- ^ Keyword: \'print\'. (Python 2.x only)
   | ExecToken { token_span :: !SrcSpan }                         -- ^ Keyword: \'exec\'. (Python 2.x only)

   -- Delimiters
   | AtToken { token_span :: !SrcSpan }                           -- ^ Delimiter: at sign \'\@\'. 
   | LeftRoundBracketToken { token_span :: !SrcSpan }             -- ^ Delimiter: left round bracket \'(\'.
   | RightRoundBracketToken { token_span :: !SrcSpan }            -- ^ Delimiter: right round bracket \')\'.
   | LeftSquareBracketToken { token_span :: !SrcSpan }            -- ^ Delimiter: left square bracket \'[\'.
   | RightSquareBracketToken { token_span :: !SrcSpan }           -- ^ Delimiter: right square bracket \']\'.
   | LeftBraceToken { token_span :: !SrcSpan }                    -- ^ Delimiter: left curly bracket \'{\'.
   | RightBraceToken { token_span :: !SrcSpan }                   -- ^ Delimiter: right curly bracket \'}\'.
   | DotToken { token_span :: !SrcSpan }                          -- ^ Delimiter: dot (full stop) \'.\'.
   | CommaToken { token_span :: !SrcSpan }                        -- ^ Delimiter: comma \',\'.
   | SemiColonToken { token_span :: !SrcSpan }                    -- ^ Delimiter: semicolon \';\'.
   | ColonToken { token_span :: !SrcSpan }                        -- ^ Delimiter: colon \':\'.
   | EllipsisToken { token_span :: !SrcSpan }                     -- ^ Delimiter: ellipses (three dots) \'...\'.
   | RightArrowToken { token_span :: !SrcSpan }                   -- ^ Delimiter: right facing arrow \'->\'.
   | AssignToken { token_span :: !SrcSpan }                       -- ^ Delimiter: assignment \'=\'.
   | PlusAssignToken { token_span :: !SrcSpan }                   -- ^ Delimiter: plus assignment \'+=\'.
   | MinusAssignToken { token_span :: !SrcSpan }                  -- ^ Delimiter: minus assignment \'-=\'.
   | MultAssignToken { token_span :: !SrcSpan }                   -- ^ Delimiter: multiply assignment \'*=\'
   | DivAssignToken { token_span :: !SrcSpan }                    -- ^ Delimiter: divide assignment \'/=\'.
   | ModAssignToken { token_span :: !SrcSpan }                    -- ^ Delimiter: modulus assignment \'%=\'.
   | PowAssignToken { token_span :: !SrcSpan }                    -- ^ Delimiter: power assignment \'**=\'.
   | BinAndAssignToken { token_span :: !SrcSpan }                 -- ^ Delimiter: binary-and assignment \'&=\'.
   | BinOrAssignToken { token_span :: !SrcSpan }                  -- ^ Delimiter: binary-or assignment \'|=\'.
   | BinXorAssignToken { token_span :: !SrcSpan }                 -- ^ Delimiter: binary-xor assignment \'^=\'.
   | LeftShiftAssignToken { token_span :: !SrcSpan }              -- ^ Delimiter: binary-left-shift assignment \'<<=\'.
   | RightShiftAssignToken { token_span :: !SrcSpan }             -- ^ Delimiter: binary-right-shift assignment \'>>=\'.
   | FloorDivAssignToken { token_span :: !SrcSpan }               -- ^ Delimiter: floor-divide assignment \'//=\'.
   | MatrixMultAssignToken { token_span :: !SrcSpan }             -- ^ Delimiter: matrix multiplication assignment \'@=\'.
   | BackQuoteToken { token_span :: !SrcSpan }                    -- ^ Delimiter: back quote character \'`\'.

   -- Operators
   | PlusToken { token_span :: !SrcSpan }                         -- ^ Operator: plus \'+\'.
   | MinusToken { token_span :: !SrcSpan }                        -- ^ Operator: minus: \'-\'.
   | MultToken { token_span :: !SrcSpan }                         -- ^ Operator: multiply \'*\'.
   | DivToken { token_span :: !SrcSpan }                          -- ^ Operator: divide \'/\'.
   | GreaterThanToken { token_span :: !SrcSpan }                  -- ^ Operator: greater-than \'>\'.
   | LessThanToken { token_span :: !SrcSpan }                     -- ^ Operator: less-than \'<\'.
   | EqualityToken { token_span :: !SrcSpan }                     -- ^ Operator: equals \'==\'.
   | GreaterThanEqualsToken { token_span :: !SrcSpan }            -- ^ Operator: greater-than-or-equals \'>=\'.
   | LessThanEqualsToken { token_span :: !SrcSpan }               -- ^ Operator: less-than-or-equals \'<=\'.
   | ExponentToken { token_span :: !SrcSpan }                     -- ^ Operator: exponential \'**\'.
   | BinaryOrToken { token_span :: !SrcSpan }                     -- ^ Operator: binary-or \'|\'.
   | XorToken { token_span :: !SrcSpan }                          -- ^ Operator: binary-xor \'^\'.
   | BinaryAndToken { token_span :: !SrcSpan }                    -- ^ Operator: binary-and \'&\'.
   | ShiftLeftToken { token_span :: !SrcSpan }                    -- ^ Operator: binary-shift-left \'<<\'.
   | ShiftRightToken { token_span :: !SrcSpan }                   -- ^ Operator: binary-shift-right \'>>\'.
   | ModuloToken { token_span :: !SrcSpan }                       -- ^ Operator: modulus \'%\'.
   | FloorDivToken { token_span :: !SrcSpan }                     -- ^ Operator: floor-divide \'//\'.
   | TildeToken { token_span :: !SrcSpan }                        -- ^ Operator: tilde \'~\'.
   | NotEqualsToken { token_span :: !SrcSpan }                    -- ^ Operator: not-equals \'!=\'.
   | NotEqualsV2Token { token_span :: !SrcSpan }                  -- ^ Operator: not-equals \'<>\'. Version 2 only.

   -- Special cases
   | EOFToken { token_span :: !SrcSpan }                          -- ^ End of file 
   deriving (Eq,Ord,Show,Typeable,Data)

instance Span Token where
  getSpan = token_span

-- | Produce a string from a token containing detailed information. Mainly intended for debugging. 
debugTokenString :: Token -> String
debugTokenString token =
   render (text (show $ toConstr token) <+> pretty (token_span token) <+>
          if hasLiteral token then text (token_literal token) else empty)

-- | Test if a token contains its literal source text.
hasLiteral :: Token -> Bool
hasLiteral token =
   case token of
      CommentToken {}       -> True
      IdentifierToken {}    -> True
      StringToken {}        -> True
      ByteStringToken {}    -> True
      UnicodeStringToken {} -> True
      IntegerToken {}       -> True
      LongIntegerToken {}   -> True
      FloatToken {}         -> True
      ImaginaryToken  {}    -> True
      other                 -> False

-- | Classification of tokens
data TokenClass
   = Comment
   | Number
   | Identifier
   | Punctuation
   | Bracket
   | Layout
   | Keyword
   | String
   | Operator
   | Assignment
   deriving (Show, Eq, Ord)

classifyToken :: Token -> TokenClass
classifyToken token =
   case token of
      IndentToken {} -> Layout
      DedentToken {} -> Layout
      NewlineToken {} -> Layout
      CommentToken {} -> Comment
      IdentifierToken {} -> Identifier
      StringToken {} -> String
      ByteStringToken {} -> String
      UnicodeStringToken {} -> String
      IntegerToken {} -> Number
      LongIntegerToken {} -> Number
      FloatToken {} -> Number
      ImaginaryToken {} -> Number
      DefToken {} -> Keyword
      WhileToken {} -> Keyword
      IfToken {} ->  Keyword
      TrueToken {} -> Keyword
      FalseToken {} -> Keyword
      ReturnToken {} -> Keyword
      TryToken {} -> Keyword
      ExceptToken {} -> Keyword
      RaiseToken {} -> Keyword
      InToken {} -> Keyword
      IsToken {} -> Keyword
      LambdaToken {} -> Keyword
      ClassToken {} -> Keyword
      FinallyToken {} -> Keyword
      NoneToken {} -> Keyword
      ForToken {} -> Keyword
      FromToken {} -> Keyword
      GlobalToken {} -> Keyword
      WithToken {} -> Keyword
      AsToken {} -> Keyword
      ElifToken {} -> Keyword
      YieldToken {} -> Keyword
      AsyncToken {} -> Keyword
      AwaitToken {} -> Keyword
      AssertToken {} -> Keyword
      ImportToken {} -> Keyword
      PassToken {} -> Keyword
      BreakToken {} -> Keyword
      ContinueToken {} -> Keyword
      DeleteToken {} -> Keyword
      ElseToken {} -> Keyword
      NotToken {} -> Keyword
      AndToken {} -> Keyword
      OrToken {} -> Keyword
      NonLocalToken {} -> Keyword
      PrintToken {} -> Keyword
      ExecToken {} -> Keyword
      AtToken {} -> Keyword
      LeftRoundBracketToken {} -> Bracket
      RightRoundBracketToken {} -> Bracket
      LeftSquareBracketToken {} -> Bracket
      RightSquareBracketToken {} -> Bracket
      LeftBraceToken {} -> Bracket
      RightBraceToken {} -> Bracket
      DotToken {} -> Operator
      CommaToken {} -> Punctuation
      SemiColonToken {} -> Punctuation
      ColonToken {} -> Punctuation
      EllipsisToken {} -> Keyword  -- What kind of thing is an ellipsis?
      RightArrowToken {} -> Punctuation
      AssignToken {} -> Assignment
      PlusAssignToken {} -> Assignment
      MinusAssignToken {} -> Assignment
      MultAssignToken {} -> Assignment
      DivAssignToken {} -> Assignment
      ModAssignToken {} -> Assignment
      PowAssignToken {} -> Assignment
      BinAndAssignToken {} -> Assignment
      BinOrAssignToken {} -> Assignment
      BinXorAssignToken {} -> Assignment
      LeftShiftAssignToken {} -> Assignment
      RightShiftAssignToken {} -> Assignment
      FloorDivAssignToken {} -> Assignment
      MatrixMultAssignToken {} -> Assignment
      BackQuoteToken {} -> Punctuation
      PlusToken {} -> Operator
      MinusToken {} -> Operator
      MultToken {} -> Operator
      DivToken {} -> Operator
      GreaterThanToken {} -> Operator
      LessThanToken {} -> Operator
      EqualityToken {} -> Operator
      GreaterThanEqualsToken {} -> Operator
      LessThanEqualsToken {} -> Operator
      ExponentToken {} -> Operator
      BinaryOrToken {} -> Operator
      XorToken {} -> Operator
      BinaryAndToken {} -> Operator
      ShiftLeftToken {} -> Operator
      ShiftRightToken {} -> Operator
      ModuloToken {} -> Operator
      FloorDivToken {} -> Operator
      TildeToken {} -> Operator
      NotEqualsToken {} -> Operator
      NotEqualsV2Token {} -> Operator
      LineJoinToken {} -> Layout
      EOFToken {} -> Layout  -- maybe a spurious classification.

-- | Produce a string from a token which is suitable for printing as Python concrete syntax.
-- /Invisible/ tokens yield an empty string.
tokenString :: Token -> String
tokenString token =
   case token of
      IndentToken {} -> ""
      DedentToken {} -> ""
      NewlineToken {} -> ""
      CommentToken {} -> token_literal token
      IdentifierToken {} -> token_literal token
      StringToken {} -> token_literal token
      ByteStringToken {} -> token_literal token
      UnicodeStringToken {} -> token_literal token
      IntegerToken {} -> token_literal token
      LongIntegerToken {} -> token_literal token
      FloatToken {} -> token_literal token
      ImaginaryToken {} -> token_literal token
      DefToken {} -> "def"
      WhileToken {} -> "while"
      IfToken {} -> "if"
      TrueToken {} -> "True"
      FalseToken {} -> "False"
      ReturnToken {} -> "return"
      TryToken {} -> "try"
      ExceptToken {} -> "except"
      RaiseToken {} -> "raise"
      InToken {} -> "in"
      IsToken {} -> "is"
      LambdaToken {} -> "lambda"
      ClassToken {} -> "class"
      FinallyToken {} -> "finally"
      NoneToken {} -> "None"
      ForToken {} -> "for"
      FromToken {} -> "from"
      GlobalToken {} -> "global"
      WithToken {} -> "with"
      AsToken {} -> "as"
      ElifToken {} -> "elif"
      YieldToken {} -> "yield"
      AsyncToken {} -> "async"
      AwaitToken {} -> "await"
      AssertToken {} -> "assert"
      ImportToken {} -> "import"
      PassToken {} -> "pass"
      BreakToken {} -> "break"
      ContinueToken {} -> "continue"
      DeleteToken {} -> "delete"
      ElseToken {} -> "else"
      NotToken {} -> "not"
      AndToken {} -> "and"
      OrToken {} -> "or"
      NonLocalToken {} -> "nonlocal"
      PrintToken {} -> "print"
      ExecToken {} -> "exec"
      AtToken {} -> "at"
      LeftRoundBracketToken {} -> "("
      RightRoundBracketToken {} -> ")"
      LeftSquareBracketToken {} -> "["
      RightSquareBracketToken {} -> "]"
      LeftBraceToken {} -> "{"
      RightBraceToken {} -> "}"
      DotToken {} -> "."
      CommaToken {} -> ","
      SemiColonToken {} -> ";"
      ColonToken {} -> ":"
      EllipsisToken {} -> "..."
      RightArrowToken {} -> "->"
      AssignToken {} -> "="
      PlusAssignToken {} -> "+="
      MinusAssignToken {} -> "-="
      MultAssignToken {} -> "*="
      DivAssignToken {} -> "/="
      ModAssignToken {} -> "%="
      PowAssignToken {} -> "**="
      BinAndAssignToken {} -> "&="
      BinOrAssignToken {} -> "|="
      BinXorAssignToken {} -> "^="
      LeftShiftAssignToken {} -> "<<="
      RightShiftAssignToken {} -> ">>="
      FloorDivAssignToken {} -> "//="
      MatrixMultAssignToken {} -> "@="
      BackQuoteToken {} -> "`"
      PlusToken {} -> "+"
      MinusToken {} -> "-"
      MultToken {} -> "*"
      DivToken {} -> "/"
      GreaterThanToken {} -> ">"
      LessThanToken {} -> "<"
      EqualityToken {} -> "=="
      GreaterThanEqualsToken {} -> ">="
      LessThanEqualsToken {} -> "<="
      ExponentToken {} -> "**"
      BinaryOrToken {} -> "|"
      XorToken {} -> "^"
      BinaryAndToken {} -> "&"
      ShiftLeftToken {} -> "<<"
      ShiftRightToken {} -> ">>"
      ModuloToken {} -> "%"
      FloorDivToken {} -> "//"
      TildeToken {} -> "~"
      NotEqualsToken {} -> "!="
      NotEqualsV2Token {} -> "<>"
      LineJoinToken {} -> "\\"
      EOFToken {} -> ""