Copyright	© Oren Ben-Kiki 2007 © Herbert Valerio Riedel 2015-2018
License	GPL-2.0-or-later
Safe Haskell	Safe
Language	Haskell2010

Data.YAML.Token

Description

Tokenizer for the YAML 1.2 syntax as defined in http://yaml.org/spec/1.2/spec.html.

Synopsis

tokenize :: ByteString -> Bool -> [Token]
data Token = Token {
- tByteOffset :: !Int
- tCharOffset :: !Int
- tLine :: !Int
- tLineChar :: !Int
- tCode :: !Code
- tText :: !String
}
data Code
- = Bom
- | Text
- | Meta
- | Break
- | LineFeed
- | LineFold
- | Indicator
- | White
- | Indent
- | DirectivesEnd
- | DocumentEnd
- | BeginEscape
- | EndEscape
- | BeginComment
- | EndComment
- | BeginDirective
- | EndDirective
- | BeginTag
- | EndTag
- | BeginHandle
- | EndHandle
- | BeginAnchor
- | EndAnchor
- | BeginProperties
- | EndProperties
- | BeginAlias
- | EndAlias
- | BeginScalar
- | EndScalar
- | BeginSequence
- | EndSequence
- | BeginMapping
- | EndMapping
- | BeginPair
- | EndPair
- | BeginNode
- | EndNode
- | BeginDocument
- | EndDocument
- | BeginStream
- | EndStream
- | Error
- | Unparsed
- | Detected
data Encoding
- = UTF8
- | UTF16LE
- | UTF16BE
- | UTF32LE
- | UTF32BE

Documentation

tokenize :: ByteString -> Bool -> [Token] Source #

tokenize input emit_unparsed converts the Unicode input (using the UTF-8, UTF-16 (LE or BE), or UTF-32 (LE or BE) encoding) to a list of Token according to the YAML 1.2 specification.

Errors are reported as tokens with Error :: Code, and the unparsed text following an error may be attached as a final Unparsed token (if the emit_unparsed argument is True). Note that tokens are available "immediately", allowing for streaming of large YAML files with memory requirements depending only on the YAML nesting level.

data Token Source #

Parsed token.

Constructors

Token
Fields tByteOffset :: !Int 0-base byte offset in stream. tCharOffset :: !Int 0-base character offset in stream. tLine :: !Int 1-based line number. tLineChar :: !Int 0-based character in line. tCode :: !Code Specific token `Code`. tText :: !String Contained input chars, if any.

Instances

Instances details

Generic Token Source #

Instance details

Defined in Data.YAML.Token

Associated Types

type Rep Token

Instance details

Defined in Data.YAML.Token

type Rep Token = D1 ('MetaData "Token" "Data.YAML.Token" "HsYAML-0.2.1.2-5QnIVi3cktj3Prnkxvfyh5" 'False) (C1 ('MetaCons "Token" 'PrefixI 'True) ((S1 ('MetaSel ('Just "tByteOffset") 'NoSourceUnpackedness 'SourceStrict 'DecidedStrict) (Rec0 Int) :*: (S1 ('MetaSel ('Just "tCharOffset") 'NoSourceUnpackedness 'SourceStrict 'DecidedStrict) (Rec0 Int) :*: S1 ('MetaSel ('Just "tLine") 'NoSourceUnpackedness 'SourceStrict 'DecidedStrict) (Rec0 Int))) :*: (S1 ('MetaSel ('Just "tLineChar") 'NoSourceUnpackedness 'SourceStrict 'DecidedStrict) (Rec0 Int) :*: (S1 ('MetaSel ('Just "tCode") 'NoSourceUnpackedness 'SourceStrict 'DecidedStrict) (Rec0 Code) :*: S1 ('MetaSel ('Just "tText") 'NoSourceUnpackedness 'SourceStrict 'DecidedStrict) (Rec0 String)))))

Methods

from :: Token -> Rep Token x #

to :: Rep Token x -> Token #

Show Token Source #

Instance details

Defined in Data.YAML.Token

Methods

showsPrec :: Int -> Token -> ShowS #

show :: Token -> String #

showList :: [Token] -> ShowS #

NFData Token Source #

Since: 0.2.0

Instance details

Defined in Data.YAML.Token

Methods

rnf :: Token -> () #

type Rep Token Source #

Instance details

Defined in Data.YAML.Token

type Rep Token = D1 ('MetaData "Token" "Data.YAML.Token" "HsYAML-0.2.1.2-5QnIVi3cktj3Prnkxvfyh5" 'False) (C1 ('MetaCons "Token" 'PrefixI 'True) ((S1 ('MetaSel ('Just "tByteOffset") 'NoSourceUnpackedness 'SourceStrict 'DecidedStrict) (Rec0 Int) :*: (S1 ('MetaSel ('Just "tCharOffset") 'NoSourceUnpackedness 'SourceStrict 'DecidedStrict) (Rec0 Int) :*: S1 ('MetaSel ('Just "tLine") 'NoSourceUnpackedness 'SourceStrict 'DecidedStrict) (Rec0 Int))) :*: (S1 ('MetaSel ('Just "tLineChar") 'NoSourceUnpackedness 'SourceStrict 'DecidedStrict) (Rec0 Int) :*: (S1 ('MetaSel ('Just "tCode") 'NoSourceUnpackedness 'SourceStrict 'DecidedStrict) (Rec0 Code) :*: S1 ('MetaSel ('Just "tText") 'NoSourceUnpackedness 'SourceStrict 'DecidedStrict) (Rec0 String)))))

data Code Source #

Token codes.

Constructors

Bom	BOM, contains "`TF8`", "`TF16LE`", "`TF32BE`", etc.
Text	Content text characters.
Meta	Non-content (meta) text characters.
Break	Separation line break.
LineFeed	Line break normalized to content line feed.
LineFold	Line break folded to content space.
Indicator	Character indicating structure.
White	Separation white space.
Indent	Indentation spaces.
DirectivesEnd	Document start marker.
DocumentEnd	Document end marker.
BeginEscape	Begins escape sequence.
EndEscape	Ends escape sequence.
BeginComment	Begins comment.
EndComment	Ends comment.
BeginDirective	Begins directive.
EndDirective	Ends directive.
BeginTag	Begins tag.
EndTag	Ends tag.
BeginHandle	Begins tag handle.
EndHandle	Ends tag handle.
BeginAnchor	Begins anchor.
EndAnchor	Ends anchor.
BeginProperties	Begins node properties.
EndProperties	Ends node properties.
BeginAlias	Begins alias.
EndAlias	Ends alias.
BeginScalar	Begins scalar content.
EndScalar	Ends scalar content.
BeginSequence	Begins sequence content.
EndSequence	Ends sequence content.
BeginMapping	Begins mapping content.
EndMapping	Ends mapping content.
BeginPair	Begins mapping key:value pair.
EndPair	Ends mapping key:value pair.
BeginNode	Begins complete node.
EndNode	Ends complete node.
BeginDocument	Begins document.
EndDocument	Ends document.
BeginStream	Begins YAML stream.
EndStream	Ends YAML stream.
Error	Parsing error at this point.
Unparsed	Unparsed due to errors (or at end of test).
Detected	Detected parameter (for testing).