Copyright	(C) CSIRO 2017-2018
License	BSD3
Maintainer	Isaac Elliott <isaace71295@gmail.com>
Stability	experimental
Portability	non-portable
Safe Haskell	None
Language	Haskell2010

Language.Python.Internal.Lexer

Contents

Source Information
Errors
Miscellaneous
Megaparsec re-exports

Description

Synopsis

tokenizeWithTabs :: (AsLexicalError s Char, AsTabError s SrcInfo, AsIncorrectDedent s SrcInfo) => FilePath -> Text -> Either s [PyToken SrcInfo]
data SrcInfo = SrcInfo {
- _srcInfoName :: FilePath
- _srcInfoLineStart :: !Int
- _srcInfoLineEnd :: !Int
- _srcInfoColStart :: !Int
- _srcInfoColEnd :: !Int
- _srcInfoOffsetStart :: !Int
- _srcInfoOffsetEnd :: !Int
}
initialSrcInfo :: FilePath -> SrcInfo
withSrcInfo :: MonadParsec e s m => m (SrcInfo -> a) -> m a
class AsLexicalError s t | s -> t where
- _LexicalError :: Prism' s (NonEmpty SourcePos, Maybe (ErrorItem t), Set (ErrorItem t))
unsafeFromLexicalError :: (HasCallStack, AsLexicalError s t) => ParseError t Void -> s
class AsTabError s a | s -> a where
- _TabError :: Prism' s a
class AsIncorrectDedent s a | s -> a where
- _IncorrectDedent :: Prism' s a
fromTabError :: (AsTabError s a, AsIncorrectDedent s a) => TabError a -> s
data TabError a
- = TabError a
- | IncorrectDedent a
tokenize :: AsLexicalError e Char => FilePath -> Text -> Either e [PyToken SrcInfo]
insertTabs :: (Semigroup a, AsTabError s a, AsIncorrectDedent s a) => a -> [PyToken a] -> Either s [PyToken a]
data ParseError t e
- = TrivialError (NonEmpty SourcePos) (Maybe (ErrorItem t)) (Set (ErrorItem t))
- | FancyError (NonEmpty SourcePos) (Set (ErrorFancy e))

Documentation

tokenizeWithTabs Source #

Arguments

:: (AsLexicalError s Char, AsTabError s SrcInfo, AsIncorrectDedent s SrcInfo)
=> FilePath	File name
-> Text	Input to tokenize
-> Either s [PyToken SrcInfo]

Tokenize an input file, inserting indent/level/dedent tokens in appropriate positions according to the block structure.

Source Information

data SrcInfo Source #

Constructors

SrcInfo
Fields _srcInfoName :: FilePath _srcInfoLineStart :: !Int _srcInfoLineEnd :: !Int _srcInfoColStart :: !Int _srcInfoColEnd :: !Int _srcInfoOffsetStart :: !Int _srcInfoOffsetEnd :: !Int

Instances

Eq SrcInfo Source #
Instance details Defined in Language.Python.Internal.Lexer Methods (==) :: SrcInfo -> SrcInfo -> Bool # (/=) :: SrcInfo -> SrcInfo -> Bool #
Show SrcInfo Source #
Instance details Defined in Language.Python.Internal.Lexer Methods showsPrec :: Int -> SrcInfo -> ShowS # show :: SrcInfo -> String # showList :: [SrcInfo] -> ShowS #
Semigroup SrcInfo Source #
Instance details Defined in Language.Python.Internal.Lexer Methods (<>) :: SrcInfo -> SrcInfo -> SrcInfo # sconcat :: NonEmpty SrcInfo -> SrcInfo # stimes :: Integral b => b -> SrcInfo -> SrcInfo #

initialSrcInfo :: FilePath -> SrcInfo Source #

withSrcInfo :: MonadParsec e s m => m (SrcInfo -> a) -> m a Source #

Errors

class AsLexicalError s t | s -> t where Source #

Methods

_LexicalError :: Prism' s (NonEmpty SourcePos, Maybe (ErrorItem t), Set (ErrorItem t)) Source #

Instances

AsLexicalError (ParseError a) Char Source #
Instance details Defined in Language.Python.Parse.Error Methods _LexicalError :: Prism' (ParseError a) (NonEmpty SourcePos, Maybe (ErrorItem Char), Set (ErrorItem Char)) Source #

unsafeFromLexicalError :: (HasCallStack, AsLexicalError s t) => ParseError t Void -> s Source #

Convert a concrete ParseError to a value that has an instance of AsLexicalError

This function is partial, because our parser will never use FancyError

class AsTabError s a | s -> a where Source #

Methods

_TabError :: Prism' s a Source #

Instances

AsTabError (ParseError a) a Source #
Instance details Defined in Language.Python.Parse.Error Methods _TabError :: Prism' (ParseError a) a Source #
AsTabError (IndentationError a) a Source #
Instance details Defined in Language.Python.Validate.Indentation.Error Methods _TabError :: Prism' (IndentationError a) a Source #
AsTabError (ValidationError a) a Source #
Instance details Defined in Language.Python.Validate.Error Methods _TabError :: Prism' (ValidationError a) a Source #

class AsIncorrectDedent s a | s -> a where Source #

Methods

_IncorrectDedent :: Prism' s a Source #

Instances

AsIncorrectDedent (ParseError a) a Source #
Instance details Defined in Language.Python.Parse.Error Methods _IncorrectDedent :: Prism' (ParseError a) a Source #

fromTabError :: (AsTabError s a, AsIncorrectDedent s a) => TabError a -> s Source #

Convert a concrete TabError to a value that has an instance of AsTabError

data TabError a Source #

Constructors

TabError a

Tabs and spaces were used inconsistently

IncorrectDedent a

The dedent at the end of a block doesn't match and preceding indents

e.g.

def a():
    if b:
        pass
    else:
        pass
  pass

The final line will cause an IncorrectDedent error

Instances

Eq a => Eq (TabError a) Source #
Instance details Defined in Language.Python.Internal.Lexer Methods (==) :: TabError a -> TabError a -> Bool # (/=) :: TabError a -> TabError a -> Bool #
Show a => Show (TabError a) Source #
Instance details Defined in Language.Python.Internal.Lexer Methods showsPrec :: Int -> TabError a -> ShowS # show :: TabError a -> String # showList :: [TabError a] -> ShowS #

Miscellaneous

tokenize Source #

Arguments

:: AsLexicalError e Char
=> FilePath	File name
-> Text	Input to tokenize
-> Either e [PyToken SrcInfo]

Convert some input to a sequence of tokens. Indent and dedent tokens are not added (see insertTabs)

insertTabs Source #

Arguments

:: (Semigroup a, AsTabError s a, AsIncorrectDedent s a)
=> a	Initial source annotation
-> [PyToken a]	Token stream
-> Either s [PyToken a]

Insert indent and dedent tokens

https://docs.python.org/3.5/reference/lexical_analysis.html#indentation

Megaparsec re-exports

data ParseError t e #

Constructors

TrivialError (NonEmpty SourcePos) (Maybe (ErrorItem t)) (Set (ErrorItem t))
FancyError (NonEmpty SourcePos) (Set (ErrorFancy e))

Instances

(Eq t, Eq e) => Eq (ParseError t e)
Instance details Defined in Text.Megaparsec.Error Methods (==) :: ParseError t e -> ParseError t e -> Bool # (/=) :: ParseError t e -> ParseError t e -> Bool #
(Data t, Data e, Ord t, Ord e) => Data (ParseError t e)
Instance details Defined in Text.Megaparsec.Error Methods gfoldl :: (forall d b. Data d => c (d -> b) -> d -> c b) -> (forall g. g -> c g) -> ParseError t e -> c (ParseError t e) # gunfold :: (forall b r. Data b => c (b -> r) -> c r) -> (forall r. r -> c r) -> Constr -> c (ParseError t e) # toConstr :: ParseError t e -> Constr # dataTypeOf :: ParseError t e -> DataType # dataCast1 :: Typeable t0 => (forall d. Data d => c (t0 d)) -> Maybe (c (ParseError t e)) # dataCast2 :: Typeable t0 => (forall d e0. (Data d, Data e0) => c (t0 d e0)) -> Maybe (c (ParseError t e)) # gmapT :: (forall b. Data b => b -> b) -> ParseError t e -> ParseError t e # gmapQl :: (r -> r' -> r) -> r -> (forall d. Data d => d -> r') -> ParseError t e -> r # gmapQr :: (r' -> r -> r) -> r -> (forall d. Data d => d -> r') -> ParseError t e -> r # gmapQ :: (forall d. Data d => d -> u) -> ParseError t e -> [u] # gmapQi :: Int -> (forall d. Data d => d -> u) -> ParseError t e -> u # gmapM :: Monad m => (forall d. Data d => d -> m d) -> ParseError t e -> m (ParseError t e) # gmapMp :: MonadPlus m => (forall d. Data d => d -> m d) -> ParseError t e -> m (ParseError t e) # gmapMo :: MonadPlus m => (forall d. Data d => d -> m d) -> ParseError t e -> m (ParseError t e) #
(Read t, Read e, Ord t, Ord e) => Read (ParseError t e)
Instance details Defined in Text.Megaparsec.Error Methods readsPrec :: Int -> ReadS (ParseError t e) # readList :: ReadS [ParseError t e] # readPrec :: ReadPrec (ParseError t e) # readListPrec :: ReadPrec [ParseError t e] #
(Show t, Show e) => Show (ParseError t e)
Instance details Defined in Text.Megaparsec.Error Methods showsPrec :: Int -> ParseError t e -> ShowS # show :: ParseError t e -> String # showList :: [ParseError t e] -> ShowS #
Generic (ParseError t e)
Instance details Defined in Text.Megaparsec.Error Associated Types type Rep (ParseError t e) :: Type -> Type # Methods from :: ParseError t e -> Rep (ParseError t e) x # to :: Rep (ParseError t e) x -> ParseError t e #
(Ord t, Ord e) => Semigroup (ParseError t e)
Instance details Defined in Text.Megaparsec.Error Methods (<>) :: ParseError t e -> ParseError t e -> ParseError t e # sconcat :: NonEmpty (ParseError t e) -> ParseError t e # stimes :: Integral b => b -> ParseError t e -> ParseError t e #
(Ord t, Ord e) => Monoid (ParseError t e)
Instance details Defined in Text.Megaparsec.Error Methods mempty :: ParseError t e # mappend :: ParseError t e -> ParseError t e -> ParseError t e # mconcat :: [ParseError t e] -> ParseError t e #
(Show t, Ord t, ShowToken t, Typeable t, Show e, ShowErrorComponent e, Typeable e) => Exception (ParseError t e)
Instance details Defined in Text.Megaparsec.Error Methods toException :: ParseError t e -> SomeException # fromException :: SomeException -> Maybe (ParseError t e) # displayException :: ParseError t e -> String #
(NFData t, NFData e) => NFData (ParseError t e)
Instance details Defined in Text.Megaparsec.Error Methods rnf :: ParseError t e -> () #
type Rep (ParseError t e)
Instance details Defined in Text.Megaparsec.Error type Rep (ParseError t e) = D1 (MetaData "ParseError" "Text.Megaparsec.Error" "megaparsec-6.5.0-7b380a41c7db9c6a4aad5bc0349f059f6770288935e8a88dbe061dc3475e1e3e" False) (C1 (MetaCons "TrivialError" PrefixI False) (S1 (MetaSel (Nothing :: Maybe Symbol) NoSourceUnpackedness NoSourceStrictness DecidedLazy) (Rec0 (NonEmpty SourcePos)) :: (S1 (MetaSel (Nothing :: Maybe Symbol) NoSourceUnpackedness NoSourceStrictness DecidedLazy) (Rec0 (Maybe (ErrorItem t))) :: S1 (MetaSel (Nothing :: Maybe Symbol) NoSourceUnpackedness NoSourceStrictness DecidedLazy) (Rec0 (Set (ErrorItem t))))) :+: C1 (MetaCons "FancyError" PrefixI False) (S1 (MetaSel (Nothing :: Maybe Symbol) NoSourceUnpackedness NoSourceStrictness DecidedLazy) (Rec0 (NonEmpty SourcePos)) :*: S1 (MetaSel (Nothing :: Maybe Symbol) NoSourceUnpackedness NoSourceStrictness DecidedLazy) (Rec0 (Set (ErrorFancy e)))))