{-# LANGUAGE FlexibleContexts #-}
{-# LANGUAGE FlexibleInstances #-}
{-# LANGUAGE GeneralizedNewtypeDeriving #-}
{-# LANGUAGE MultiParamTypeClasses #-}
{-# LANGUAGE OverloadedStrings #-}
{-# LANGUAGE PatternGuards #-}
{-# LANGUAGE ScopedTypeVariables #-}
{-# LANGUAGE StandaloneDeriving #-}
{-# LANGUAGE TemplateHaskell #-}
{-# LANGUAGE TupleSections #-}
{-# LANGUAGE TypeFamilies #-}
module Database.Persist.Sqlite
( withSqlitePool
, withSqlitePoolInfo
, withSqliteConn
, withSqliteConnInfo
, createSqlitePool
, createSqlitePoolFromInfo
, module Database.Persist.Sql
, SqliteConf (..)
, SqliteConnectionInfo
, mkSqliteConnectionInfo
, sqlConnectionStr
, walEnabled
, fkEnabled
, extraPragmas
, runSqlite
, runSqliteInfo
, wrapConnection
, wrapConnectionInfo
, mockMigration
, retryOnBusy
, waitForDatabase
, RawSqlite
, persistentBackend
, rawSqliteConnection
, withRawSqliteConnInfo
) where
import Control.Concurrent (threadDelay)
import qualified Control.Exception as E
import Control.Monad (forM_)
import Control.Monad.IO.Unlift (MonadIO (..), MonadUnliftIO, withRunInIO, withUnliftIO, unliftIO, withRunInIO)
import Control.Monad.Logger (NoLoggingT, runNoLoggingT, MonadLogger, logWarn, runLoggingT)
import Control.Monad.Trans.Reader (ReaderT, runReaderT)
import Control.Monad.Trans.Writer (runWriterT)
import Data.Acquire (Acquire, mkAcquire, with)
import Data.Aeson
import Data.Aeson.Types (modifyFailure)
import Data.Conduit
import qualified Data.Conduit.List as CL
import qualified Data.HashMap.Lazy as HashMap
import Data.Int (Int64)
import Data.IORef
import qualified Data.Map as Map
import Data.Monoid ((<>))
import Data.Pool (Pool)
import Data.Text (Text)
import qualified Data.Text as T
import qualified Data.Text.IO as TIO
import Lens.Micro.TH (makeLenses)
import UnliftIO.Resource (ResourceT, runResourceT)
import Database.Persist.Sql
import qualified Database.Persist.Sql.Util as Util
import qualified Database.Sqlite as Sqlite
createSqlitePool :: (MonadLogger m, MonadUnliftIO m)
=> Text -> Int -> m (Pool SqlBackend)
createSqlitePool = createSqlitePoolFromInfo . conStringToInfo
createSqlitePoolFromInfo :: (MonadLogger m, MonadUnliftIO m)
=> SqliteConnectionInfo -> Int -> m (Pool SqlBackend)
createSqlitePoolFromInfo connInfo = createSqlPool $ openWith const connInfo
withSqlitePool :: (MonadUnliftIO m, MonadLogger m)
=> Text
-> Int
-> (Pool SqlBackend -> m a) -> m a
withSqlitePool connInfo = withSqlPool . openWith const $ conStringToInfo connInfo
withSqlitePoolInfo :: (MonadUnliftIO m, MonadLogger m)
=> SqliteConnectionInfo
-> Int
-> (Pool SqlBackend -> m a) -> m a
withSqlitePoolInfo connInfo = withSqlPool $ openWith const connInfo
withSqliteConn :: (MonadUnliftIO m, MonadLogger m)
=> Text -> (SqlBackend -> m a) -> m a
withSqliteConn = withSqliteConnInfo . conStringToInfo
withSqliteConnInfo :: (MonadUnliftIO m, MonadLogger m)
=> SqliteConnectionInfo -> (SqlBackend -> m a) -> m a
withSqliteConnInfo = withSqlConn . openWith const
openWith :: (SqlBackend -> Sqlite.Connection -> r)
-> SqliteConnectionInfo
-> LogFunc
-> IO r
openWith f connInfo logFunc = do
conn <- Sqlite.open $ _sqlConnectionStr connInfo
backend <- wrapConnectionInfo connInfo conn logFunc `E.onException` Sqlite.close conn
return $ f backend conn
wrapConnection :: Sqlite.Connection -> LogFunc -> IO SqlBackend
wrapConnection = wrapConnectionInfo (mkSqliteConnectionInfo "")
retryOnBusy :: (MonadUnliftIO m, MonadLogger m) => m a -> m a
retryOnBusy action =
start $ take 20 $ delays 1000
where
delays x
| x >= 1000000 = repeat x
| otherwise = x : delays (x * 2)
start [] = do
$logWarn "Out of retry attempts"
action
start (x:xs) = do
eres <- withRunInIO $ \run -> E.try $ run action
case eres of
Left (Sqlite.SqliteException { Sqlite.seError = Sqlite.ErrorBusy }) -> do
$logWarn "Encountered an SQLITE_BUSY, going to retry..."
liftIO $ threadDelay x
start xs
Left e -> liftIO $ E.throwIO e
Right y -> return y
waitForDatabase
:: (MonadUnliftIO m, MonadLogger m, BackendCompatible SqlBackend backend)
=> ReaderT backend m ()
waitForDatabase = retryOnBusy $ rawExecute "SELECT 42" []
wrapConnectionInfo
:: SqliteConnectionInfo
-> Sqlite.Connection
-> LogFunc
-> IO SqlBackend
wrapConnectionInfo connInfo conn logFunc = do
let
walPragma
| _walEnabled connInfo = (("PRAGMA journal_mode=WAL;", True):)
| otherwise = id
fkPragma
| _fkEnabled connInfo = (("PRAGMA foreign_keys = on;", False):)
| otherwise = id
pragmas = walPragma $ fkPragma $ map (, False) $ _extraPragmas connInfo
forM_ pragmas $ \(pragma, shouldRetry) -> flip runLoggingT logFunc $
(if shouldRetry then retryOnBusy else id) $ liftIO $ do
stmt <- Sqlite.prepare conn pragma
_ <- Sqlite.stepConn conn stmt
Sqlite.reset conn stmt
Sqlite.finalize stmt
smap <- newIORef $ Map.empty
return $ SqlBackend
{ connPrepare = prepare' conn
, connStmtMap = smap
, connInsertSql = insertSql'
, connUpsertSql = Nothing
, connPutManySql = Just putManySql
, connInsertManySql = Nothing
, connClose = Sqlite.close conn
, connMigrateSql = migrate'
, connBegin = \f _ -> helper "BEGIN" f
, connCommit = helper "COMMIT"
, connRollback = ignoreExceptions . helper "ROLLBACK"
, connEscapeName = escape
, connNoLimit = "LIMIT -1"
, connRDBMS = "sqlite"
, connLimitOffset = decorateSQLWithLimitOffset "LIMIT -1"
, connLogFunc = logFunc
, connMaxParams = Just 999
, connRepsertManySql = Just repsertManySql
}
where
helper t getter = do
stmt <- getter t
_ <- stmtExecute stmt []
stmtReset stmt
ignoreExceptions = E.handle (\(_ :: E.SomeException) -> return ())
runSqlite :: (MonadUnliftIO m)
=> Text
-> ReaderT SqlBackend (NoLoggingT (ResourceT m)) a
-> m a
runSqlite connstr = runResourceT
. runNoLoggingT
. withSqliteConn connstr
. runSqlConn
runSqliteInfo :: (MonadUnliftIO m)
=> SqliteConnectionInfo
-> ReaderT SqlBackend (NoLoggingT (ResourceT m)) a
-> m a
runSqliteInfo conInfo = runResourceT
. runNoLoggingT
. withSqliteConnInfo conInfo
. runSqlConn
prepare' :: Sqlite.Connection -> Text -> IO Statement
prepare' conn sql = do
stmt <- Sqlite.prepare conn sql
return Statement
{ stmtFinalize = Sqlite.finalize stmt
, stmtReset = Sqlite.reset conn stmt
, stmtExecute = execute' conn stmt
, stmtQuery = withStmt' conn stmt
}
insertSql' :: EntityDef -> [PersistValue] -> InsertSqlResult
insertSql' ent vals =
case entityPrimary ent of
Just _ ->
ISRManyKeys sql vals
where sql = T.concat
[ "INSERT INTO "
, escape $ entityDB ent
, "("
, T.intercalate "," $ map (escape . fieldDB) $ entityFields ent
, ") VALUES("
, T.intercalate "," (map (const "?") $ entityFields ent)
, ")"
]
Nothing ->
ISRInsertGet ins sel
where
sel = T.concat
[ "SELECT "
, escape $ fieldDB (entityId ent)
, " FROM "
, escape $ entityDB ent
, " WHERE _ROWID_=last_insert_rowid()"
]
ins = T.concat
[ "INSERT INTO "
, escape $ entityDB ent
, if null (entityFields ent)
then " VALUES(null)"
else T.concat
[ "("
, T.intercalate "," $ map (escape . fieldDB) $ entityFields ent
, ") VALUES("
, T.intercalate "," (map (const "?") $ entityFields ent)
, ")"
]
]
execute' :: Sqlite.Connection -> Sqlite.Statement -> [PersistValue] -> IO Int64
execute' conn stmt vals = flip finally (liftIO $ Sqlite.reset conn stmt) $ do
Sqlite.bind stmt vals
_ <- Sqlite.stepConn conn stmt
Sqlite.changes conn
withStmt'
:: MonadIO m
=> Sqlite.Connection
-> Sqlite.Statement
-> [PersistValue]
-> Acquire (ConduitM () [PersistValue] m ())
withStmt' conn stmt vals = do
_ <- mkAcquire
(Sqlite.bind stmt vals >> return stmt)
(Sqlite.reset conn)
return pull
where
pull = do
x <- liftIO $ Sqlite.stepConn conn stmt
case x of
Sqlite.Done -> return ()
Sqlite.Row -> do
cols <- liftIO $ Sqlite.columns stmt
yield cols
pull
showSqlType :: SqlType -> Text
showSqlType SqlString = "VARCHAR"
showSqlType SqlInt32 = "INTEGER"
showSqlType SqlInt64 = "INTEGER"
showSqlType SqlReal = "REAL"
showSqlType (SqlNumeric precision scale) = T.concat [ "NUMERIC(", T.pack (show precision), ",", T.pack (show scale), ")" ]
showSqlType SqlDay = "DATE"
showSqlType SqlTime = "TIME"
showSqlType SqlDayTime = "TIMESTAMP"
showSqlType SqlBlob = "BLOB"
showSqlType SqlBool = "BOOLEAN"
showSqlType (SqlOther t) = t
migrate' :: [EntityDef]
-> (Text -> IO Statement)
-> EntityDef
-> IO (Either [Text] [(Bool, Text)])
migrate' allDefs getter val = do
let (cols, uniqs, _) = mkColumns allDefs val
let newSql = mkCreateTable False def (filter (not . safeToRemove val . cName) cols, uniqs)
stmt <- getter "SELECT sql FROM sqlite_master WHERE type='table' AND name=?"
oldSql' <- with (stmtQuery stmt [PersistText $ unDBName table])
(\src -> runConduit $ src .| go)
case oldSql' of
Nothing -> return $ Right [(False, newSql)]
Just oldSql -> do
if oldSql == newSql
then return $ Right []
else do
sql <- getCopyTable allDefs getter val
return $ Right sql
where
def = val
table = entityDB def
go = do
x <- CL.head
case x of
Nothing -> return Nothing
Just [PersistText y] -> return $ Just y
Just y -> error $ "Unexpected result from sqlite_master: " ++ show y
mockMigration :: Migration -> IO ()
mockMigration mig = do
smap <- newIORef $ Map.empty
let sqlbackend = SqlBackend
{ connPrepare = \_ -> do
return Statement
{ stmtFinalize = return ()
, stmtReset = return ()
, stmtExecute = undefined
, stmtQuery = \_ -> return $ return ()
}
, connStmtMap = smap
, connInsertSql = insertSql'
, connInsertManySql = Nothing
, connClose = undefined
, connMigrateSql = migrate'
, connBegin = \f _ -> helper "BEGIN" f
, connCommit = helper "COMMIT"
, connRollback = ignoreExceptions . helper "ROLLBACK"
, connEscapeName = escape
, connNoLimit = "LIMIT -1"
, connRDBMS = "sqlite"
, connLimitOffset = decorateSQLWithLimitOffset "LIMIT -1"
, connLogFunc = undefined
, connUpsertSql = undefined
, connPutManySql = undefined
, connMaxParams = Just 999
, connRepsertManySql = Nothing
}
result = runReaderT . runWriterT . runWriterT $ mig
resp <- result sqlbackend
mapM_ TIO.putStrLn $ map snd $ snd resp
where
helper t getter = do
stmt <- getter t
_ <- stmtExecute stmt []
stmtReset stmt
ignoreExceptions = E.handle (\(_ :: E.SomeException) -> return ())
safeToRemove :: EntityDef -> DBName -> Bool
safeToRemove def (DBName colName)
= any (elem "SafeToRemove" . fieldAttrs)
$ filter ((== DBName colName) . fieldDB)
$ entityFields def
getCopyTable :: [EntityDef]
-> (Text -> IO Statement)
-> EntityDef
-> IO [(Bool, Text)]
getCopyTable allDefs getter def = do
stmt <- getter $ T.concat [ "PRAGMA table_info(", escape table, ")" ]
oldCols' <- with (stmtQuery stmt []) (\src -> runConduit $ src .| getCols)
let oldCols = map DBName $ filter (/= "id") oldCols'
let newCols = filter (not . safeToRemove def) $ map cName cols
let common = filter (`elem` oldCols) newCols
return [ (False, tmpSql)
, (False, copyToTemp $ addIdCol common)
, (common /= filter (not . safeToRemove def) oldCols, dropOld)
, (False, newSql)
, (False, copyToFinal $ addIdCol newCols)
, (False, dropTmp)
]
where
addIdCol = case entityPrimary def of
Nothing -> (fieldDB (entityId def) :)
Just _ -> id
getCols = do
x <- CL.head
case x of
Nothing -> return []
Just (_:PersistText name:_) -> do
names <- getCols
return $ name : names
Just y -> error $ "Invalid result from PRAGMA table_info: " ++ show y
table = entityDB def
tableTmp = DBName $ unDBName table <> "_backup"
(cols, uniqs, _) = mkColumns allDefs def
cols' = filter (not . safeToRemove def . cName) cols
newSql = mkCreateTable False def (cols', uniqs)
tmpSql = mkCreateTable True def { entityDB = tableTmp } (cols', uniqs)
dropTmp = "DROP TABLE " <> escape tableTmp
dropOld = "DROP TABLE " <> escape table
copyToTemp common = T.concat
[ "INSERT INTO "
, escape tableTmp
, "("
, T.intercalate "," $ map escape common
, ") SELECT "
, T.intercalate "," $ map escape common
, " FROM "
, escape table
]
copyToFinal newCols = T.concat
[ "INSERT INTO "
, escape table
, " SELECT "
, T.intercalate "," $ map escape newCols
, " FROM "
, escape tableTmp
]
mkCreateTable :: Bool -> EntityDef -> ([Column], [UniqueDef]) -> Text
mkCreateTable isTemp entity (cols, uniqs) =
case entityPrimary entity of
Just pdef ->
T.concat
[ "CREATE"
, if isTemp then " TEMP" else ""
, " TABLE "
, escape $ entityDB entity
, "("
, T.drop 1 $ T.concat $ map (sqlColumn isTemp) cols
, ", PRIMARY KEY "
, "("
, T.intercalate "," $ map (escape . fieldDB) $ compositeFields pdef
, ")"
, T.concat $ map sqlUnique uniqs
, ")"
]
Nothing -> T.concat
[ "CREATE"
, if isTemp then " TEMP" else ""
, " TABLE "
, escape $ entityDB entity
, "("
, escape $ fieldDB (entityId entity)
, " "
, showSqlType $ fieldSqlType $ entityId entity
," PRIMARY KEY"
, mayDefault $ defaultAttribute $ fieldAttrs $ entityId entity
, T.concat $ map (sqlColumn isTemp) cols
, T.concat $ map sqlUnique uniqs
, ")"
]
mayDefault :: Maybe Text -> Text
mayDefault def = case def of
Nothing -> ""
Just d -> " DEFAULT " <> d
sqlColumn :: Bool -> Column -> Text
sqlColumn noRef (Column name isNull typ def _cn _maxLen ref) = T.concat
[ ","
, escape name
, " "
, showSqlType typ
, if isNull then " NULL" else " NOT NULL"
, mayDefault def
, case ref of
Nothing -> ""
Just (table, _) -> if noRef then "" else " REFERENCES " <> escape table
]
sqlUnique :: UniqueDef -> Text
sqlUnique (UniqueDef _ cname cols _) = T.concat
[ ",CONSTRAINT "
, escape cname
, " UNIQUE ("
, T.intercalate "," $ map (escape . snd) cols
, ")"
]
escape :: DBName -> Text
escape (DBName s) =
T.concat [q, T.concatMap go s, q]
where
q = T.singleton '"'
go '"' = "\"\""
go c = T.singleton c
putManySql :: EntityDef -> Int -> Text
putManySql ent n = putManySql' conflictColumns fields ent n
where
fields = entityFields ent
conflictColumns = concatMap (map (escape . snd) . uniqueFields) (entityUniques ent)
repsertManySql :: EntityDef -> Int -> Text
repsertManySql ent n = putManySql' conflictColumns fields ent n
where
fields = keyAndEntityFields ent
conflictColumns = escape . fieldDB <$> entityKeyFields ent
putManySql' :: [Text] -> [FieldDef] -> EntityDef -> Int -> Text
putManySql' conflictColumns fields ent n = q
where
fieldDbToText = escape . fieldDB
mkAssignment f = T.concat [f, "=EXCLUDED.", f]
table = escape . entityDB $ ent
columns = Util.commaSeparated $ map fieldDbToText fields
placeholders = map (const "?") fields
updates = map (mkAssignment . fieldDbToText) fields
q = T.concat
[ "INSERT INTO "
, table
, Util.parenWrapped columns
, " VALUES "
, Util.commaSeparated . replicate n
. Util.parenWrapped . Util.commaSeparated $ placeholders
, " ON CONFLICT "
, Util.parenWrapped . Util.commaSeparated $ conflictColumns
, " DO UPDATE SET "
, Util.commaSeparated updates
]
data SqliteConf = SqliteConf
{ sqlDatabase :: Text
, sqlPoolSize :: Int
}
| SqliteConfInfo
{ sqlConnInfo :: SqliteConnectionInfo
, sqlPoolSize :: Int
} deriving Show
instance FromJSON SqliteConf where
parseJSON v = modifyFailure ("Persistent: error loading Sqlite conf: " ++) $ flip (withObject "SqliteConf") v parser where
parser o = if HashMap.member "database" o
then SqliteConf
<$> o .: "database"
<*> o .: "poolsize"
else SqliteConfInfo
<$> o .: "connInfo"
<*> o .: "poolsize"
instance PersistConfig SqliteConf where
type PersistConfigBackend SqliteConf = SqlPersistT
type PersistConfigPool SqliteConf = ConnectionPool
createPoolConfig (SqliteConf cs size) = runNoLoggingT $ createSqlitePoolFromInfo (conStringToInfo cs) size
createPoolConfig (SqliteConfInfo info size) = runNoLoggingT $ createSqlitePoolFromInfo info size
runPool _ = runSqlPool
loadConfig = parseJSON
finally :: MonadUnliftIO m
=> m a
-> m b
-> m a
finally a sequel = withUnliftIO $ \u ->
E.finally (unliftIO u a)
(unliftIO u sequel)
{-# INLINABLE finally #-}
mkSqliteConnectionInfo :: Text -> SqliteConnectionInfo
mkSqliteConnectionInfo fp = SqliteConnectionInfo fp True True []
conStringToInfo :: Text -> SqliteConnectionInfo
conStringToInfo connStr = SqliteConnectionInfo connStr' enableWal True [] where
(connStr', enableWal) = case () of
()
| Just cs <- T.stripPrefix "WAL=on " connStr -> (cs, True)
| Just cs <- T.stripPrefix "WAL=off " connStr -> (cs, False)
| otherwise -> (connStr, True)
data SqliteConnectionInfo = SqliteConnectionInfo
{ _sqlConnectionStr :: Text
, _walEnabled :: Bool
, _fkEnabled :: Bool
, _extraPragmas :: [Text]
} deriving Show
makeLenses ''SqliteConnectionInfo
instance FromJSON SqliteConnectionInfo where
parseJSON v = modifyFailure ("Persistent: error loading SqliteConnectionInfo: " ++) $
flip (withObject "SqliteConnectionInfo") v $ \o -> SqliteConnectionInfo
<$> o .: "connectionString"
<*> o .: "walEnabled"
<*> o .: "fkEnabled"
<*> o .:? "extraPragmas" .!= []
withRawSqliteConnInfo
:: (MonadUnliftIO m, MonadLogger m)
=> SqliteConnectionInfo
-> (RawSqlite SqlBackend -> m a)
-> m a
withRawSqliteConnInfo connInfo f = do
logFunc <- askLogFunc
withRunInIO $ \run -> E.bracket (openBackend logFunc) closeBackend $ run . f
where
openBackend = openWith RawSqlite connInfo
closeBackend = close' . _persistentBackend
data RawSqlite backend = RawSqlite
{ _persistentBackend :: backend
, _rawSqliteConnection :: Sqlite.Connection
}
makeLenses ''RawSqlite
instance HasPersistBackend b => HasPersistBackend (RawSqlite b) where
type BaseBackend (RawSqlite b) = BaseBackend b
persistBackend = persistBackend . _persistentBackend
instance BackendCompatible b (RawSqlite b) where
projectBackend = _persistentBackend
instance (PersistCore b) => PersistCore (RawSqlite b) where
newtype BackendKey (RawSqlite b) = RawSqliteKey (BackendKey b)
deriving instance (Show (BackendKey b)) => Show (BackendKey (RawSqlite b))
deriving instance (Read (BackendKey b)) => Read (BackendKey (RawSqlite b))
deriving instance (Eq (BackendKey b)) => Eq (BackendKey (RawSqlite b))
deriving instance (Ord (BackendKey b)) => Ord (BackendKey (RawSqlite b))
deriving instance (Num (BackendKey b)) => Num (BackendKey (RawSqlite b))
deriving instance (Integral (BackendKey b)) => Integral (BackendKey (RawSqlite b))
deriving instance (PersistField (BackendKey b)) => PersistField (BackendKey (RawSqlite b))
deriving instance (PersistFieldSql (BackendKey b)) => PersistFieldSql (BackendKey (RawSqlite b))
deriving instance (Real (BackendKey b)) => Real (BackendKey (RawSqlite b))
deriving instance (Enum (BackendKey b)) => Enum (BackendKey (RawSqlite b))
deriving instance (Bounded (BackendKey b)) => Bounded (BackendKey (RawSqlite b))
deriving instance (ToJSON (BackendKey b)) => ToJSON (BackendKey (RawSqlite b))
deriving instance (FromJSON (BackendKey b)) => FromJSON (BackendKey (RawSqlite b))
instance (PersistStoreRead b) => PersistStoreRead (RawSqlite b) where
get = liftPersist . get
getMany = liftPersist . getMany
instance (PersistQueryRead b) => PersistQueryRead (RawSqlite b) where
selectSourceRes filts opts = liftPersist $ selectSourceRes filts opts
selectFirst filts opts = liftPersist $ selectFirst filts opts
selectKeysRes filts opts = liftPersist $ selectKeysRes filts opts
count = liftPersist . count
instance (PersistQueryWrite b) => PersistQueryWrite (RawSqlite b) where
updateWhere filts updates = liftPersist $ updateWhere filts updates
deleteWhere = liftPersist . deleteWhere
instance (PersistUniqueRead b) => PersistUniqueRead (RawSqlite b) where
getBy = liftPersist . getBy
instance (PersistStoreWrite b) => PersistStoreWrite (RawSqlite b) where
insert = liftPersist . insert
insert_ = liftPersist . insert_
insertMany = liftPersist . insertMany
insertMany_ = liftPersist . insertMany_
insertEntityMany = liftPersist . insertEntityMany
insertKey k = liftPersist . insertKey k
repsert k = liftPersist . repsert k
repsertMany = liftPersist . repsertMany
replace k = liftPersist . replace k
delete = liftPersist . delete
update k = liftPersist . update k
updateGet k = liftPersist . updateGet k
instance (PersistUniqueWrite b) => PersistUniqueWrite (RawSqlite b) where
deleteBy = liftPersist . deleteBy
insertUnique = liftPersist . insertUnique
upsert rec = liftPersist . upsert rec
upsertBy uniq rec = liftPersist . upsertBy uniq rec
putMany = liftPersist . putMany