Network.HTTP.Robots

Synopsis

• type Robot = ([([UserAgent], [Directive])], [Unparsable])
• type Unparsable = ByteString
• data UserAgent
  • = Wildcard
  • | Literal ByteString
• type Path = ByteString
• type TimeInterval = (DiffTime, DiffTime)
• data Directive
  • = Allow Path
  • | Disallow Path
  • | CrawlDelay {
    • crawlDelay :: Rational
    • timeInterval :: TimeInterval
    }
  • | NoArchive Path
  • | NoSnippet Path
  • | NoTranslate Path
  • | NoIndex Path
• subParser :: Parser a -> ByteString -> Parser a
• safeParseRational :: Parser Rational
• dropUTF8BOM :: ByteString -> ByteString
• parseHourMinute :: Parser (Integer, Integer)
• parseTimeInterval :: Parser TimeInterval
• allDay :: TimeInterval
• parseRequestRate :: Parser Directive
• parseVisitTime :: Parser Directive
• parseCrawlDelay :: Parser Directive
• strip :: ByteString -> ByteString
• parseRobots :: ByteString -> Either String Robot
• robotP :: Parser Robot
• unparsableP :: Parser ByteString
• agentDirectiveP :: Parser ([UserAgent], [Directive])
• skipSpace :: Parser ()
• directiveP :: Parser Directive
• agentP :: Parser UserAgent
• commentsP :: Parser ()
• tokenP :: Parser ByteString
• tokenWithSpacesP :: Parser ByteString
• canAccess :: ByteString -> Robot -> Path -> Bool

Documentation