module Stackage.Upload
( UploadBundle (..)
, SnapshotIdent (..)
, uploadBundle
, UploadDocs (..)
, uploadDocs
, uploadHackageDistro
, uploadHackageDistroNamed
, UploadDocMap (..)
, uploadDocMap
, uploadBundleV2
, UploadBundleV2 (..)
, def
, StackageServer
, unStackageServer
) where
import Control.Monad.Writer.Strict (execWriter, tell)
import Data.Default.Class (Default (..))
import Data.Function (fix)
import Filesystem (isDirectory, isFile)
import Network.HTTP.Client
import qualified Network.HTTP.Client.Conduit as HCC
import Network.HTTP.Client.MultipartFormData
import Stackage.BuildPlan (BuildPlan)
import Stackage.Prelude
import Stackage.ServerBundle (bpAllPackages, docsListing, writeIndexStyle)
import System.IO.Temp (withSystemTempFile)
import qualified System.IO as IO
import qualified Data.Yaml as Y
newtype StackageServer = StackageServer { unStackageServer :: Text }
deriving (Show, Eq, Ord, Hashable, IsString)
instance Default StackageServer where
def = "http://www.stackage.org"
data UploadBundle = UploadBundle
{ ubServer :: StackageServer
, ubContents :: LByteString
, ubAlias :: Maybe Text
, ubNightly :: Maybe Text
, ubLTS :: Maybe Text
, ubAuthToken :: Text
}
instance Default UploadBundle where
def = UploadBundle
{ ubServer = def
, ubContents = mempty
, ubAlias = Nothing
, ubNightly = Nothing
, ubLTS = Nothing
, ubAuthToken = "no-auth-token-provided"
}
newtype SnapshotIdent = SnapshotIdent { unSnapshotIdent :: Text }
deriving (Show, Eq, Ord, Hashable, IsString)
uploadBundle :: UploadBundle -> Manager -> IO (SnapshotIdent, Maybe Text)
uploadBundle UploadBundle {..} man = do
req1 <- parseUrl $ unpack $ unStackageServer ubServer ++ "/upload"
req2 <- formDataBody formData req1
let req3 = req2
{ method = "PUT"
, requestHeaders =
[ ("Authorization", encodeUtf8 ubAuthToken)
, ("Accept", "application/json")
] ++ requestHeaders req2
, redirectCount = 0
, checkStatus = \_ _ _ -> Nothing
, responseTimeout = Just 300000000
}
res <- httpLbs req3 man
case lookup "x-stackage-ident" $ responseHeaders res of
Just snapid -> return
( SnapshotIdent $ decodeUtf8 snapid
, decodeUtf8 <$> lookup "location" (responseHeaders res)
)
Nothing -> error $ "An error occurred: " ++ show res
where
params = mapMaybe (\(x, y) -> (x, ) <$> y)
[ ("alias", ubAlias)
, ("nightly", ubNightly)
, ("lts", ubLTS)
]
formData = ($ []) $ execWriter $ do
forM_ params $ \(key, value) ->
tell' $ partBS key $ encodeUtf8 value
tell' $ partFileRequestBody "stackage" "stackage"
$ RequestBodyLBS ubContents
tell' x = tell (x:)
data UploadDocs = UploadDocs
{ udServer :: StackageServer
, udDocs :: FilePath
, udAuthToken :: Text
, udSnapshot :: SnapshotIdent
}
uploadDocs :: UploadDocs -> Manager -> IO (Response LByteString)
uploadDocs (UploadDocs (StackageServer host) fp0 token ident) man = do
fe <- isFile fp0
if fe
then uploadDocsFile $ fpToString fp0
else do
de <- isDirectory fp0
if de
then uploadDocsDir
else error $ "Path not found: " ++ fpToString fp0
where
uploadDocsDir = withSystemTempFile "haddocks.tar.xz" $ \fp h -> do
hClose h
dirs <- writeIndexStyle (Just $ unSnapshotIdent ident) fp0
let cp = (proc "tar" $ "cJf" : fp : "index.html" : "style.css" : dirs)
{ cwd = Just $ fpToString fp0
}
withCheckedProcess cp $ \Inherited Inherited Inherited -> return ()
uploadDocsFile fp
uploadDocsFile fp = do
req1 <- parseUrl $ unpack $ concat
[ host
, "/upload-haddock/"
, unSnapshotIdent ident
]
let formData =
[ partFileSource "tarball" fp
]
req2 <- formDataBody formData req1
let req3 = req2
{ method = "PUT"
, requestHeaders =
[ ("Authorization", encodeUtf8 token)
, ("Accept", "application/json")
] ++ requestHeaders req2
, redirectCount = 0
, checkStatus = \_ _ _ -> Nothing
, responseTimeout = Just 300000000
}
httpLbs req3 man
uploadHackageDistro :: BuildPlan
-> ByteString
-> ByteString
-> Manager
-> IO (Response LByteString)
uploadHackageDistro = uploadHackageDistroNamed "Stackage"
uploadHackageDistroNamed
:: Text
-> BuildPlan
-> ByteString
-> ByteString
-> Manager
-> IO (Response LByteString)
uploadHackageDistroNamed name bp username password manager = do
req1 <- parseUrl $ concat
[ "http://hackage.haskell.org/distro/"
, unpack name
, "/packages.csv"
]
let req2 = req1
{ requestHeaders = [("Content-Type", "text/csv")]
, requestBody = RequestBodyLBS csv
, checkStatus = \_ _ _ -> Nothing
, method = "PUT"
}
httpLbs (applyBasicAuth username password req2) manager
where
csv = encodeUtf8
$ builderToLazy
$ mconcat
$ intersperse "\n"
$ map go
$ mapToList
$ bpAllPackages bp
go (name, version) =
"\"" ++
(toBuilder $ display name) ++
"\",\"" ++
(toBuilder $ display version) ++
"\",\"http://www.stackage.org/package/" ++
(toBuilder $ display name) ++
"\""
data UploadDocMap = UploadDocMap
{ udmServer :: StackageServer
, udmAuthToken :: Text
, udmSnapshot :: SnapshotIdent
, udmDocDir :: FilePath
, udmPlan :: BuildPlan
}
uploadDocMap :: UploadDocMap -> Manager -> IO (Response LByteString)
uploadDocMap UploadDocMap {..} man = do
docmap <- docsListing udmPlan udmDocDir
req1 <- parseUrl $ unpack $ unStackageServer udmServer ++ "/upload-doc-map"
req2 <- formDataBody (formData $ Y.encode docmap) req1
let req3 = req2
{ method = "PUT"
, requestHeaders =
[ ("Authorization", encodeUtf8 udmAuthToken)
, ("Accept", "application/json")
] ++ requestHeaders req2
, redirectCount = 0
, checkStatus = \_ _ _ -> Nothing
, responseTimeout = Just 300000000
}
httpLbs req3 man
where
formData docmap =
[ partBS "snapshot" (encodeUtf8 $ unSnapshotIdent udmSnapshot)
, partFileRequestBody "docmap" "docmap" $ RequestBodyBS docmap
]
data UploadBundleV2 = UploadBundleV2
{ ub2Server :: StackageServer
, ub2AuthToken :: Text
, ub2Bundle :: FilePath
}
uploadBundleV2 :: UploadBundleV2 -> Manager -> IO Text
uploadBundleV2 UploadBundleV2 {..} man = IO.withBinaryFile (fpToString ub2Bundle) IO.ReadMode $ \h -> do
size <- IO.hFileSize h
putStrLn $ "Bundle size: " ++ tshow size
req1 <- parseUrl $ unpack $ unStackageServer ub2Server ++ "/upload2"
let req2 = req1
{ method = "PUT"
, requestHeaders =
[ ("Authorization", encodeUtf8 ub2AuthToken)
, ("Accept", "application/json")
, ("Content-Type", "application/x-tar")
]
, requestBody = HCC.requestBodySource (fromIntegral size)
$ sourceHandle h $= printProgress size
}
sink = decodeUtf8C =$ fix (\loop -> do
mx <- peekC
case mx of
Nothing -> error $ "uploadBundleV2: premature end of stream"
Just _ -> do
l <- lineC $ takeCE 4096 =$ foldC
let (cmd, msg') = break (== ':') l
msg = dropWhile (== ' ') $ dropWhile (== ':') msg'
case cmd of
"CONT" -> do
putStrLn msg
loop
"FAILURE" -> error $ "uploadBundleV2 failed: " ++ unpack msg
"SUCCESS" -> return msg
_ -> error $ "uploadBundleV2: unknown command " ++ unpack cmd
)
withResponse req2 man $ \res -> HCC.bodyReaderSource (responseBody res) $$ sink
where
printProgress total =
loop 0 0
where
loop sent lastPercent =
await >>= maybe (putStrLn "Upload complete") go
where
go bs = do
yield bs
let sent' = sent + fromIntegral (length bs)
percent = sent' * 100 `div` total
when (percent /= lastPercent)
$ putStrLn $ "Upload progress: " ++ tshow percent ++ "%"
loop sent' percent