{-# LANGUAGE RankNTypes #-} {-# LANGUAGE FlexibleContexts #-} {-# OPTIONS_GHC -fno-warn-unused-binds #-} {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE NoMonomorphismRestriction #-} {-# OPTIONS_HADDOCK ignore-exports #-} module Buchhaltung.Importers ( paypalImporter , aqbankingImporter , comdirectVisaImporter , monefyImporter , natwestIntlImporter , barclaysUkImporter , revolutImporter , barclaycardusImporter , pncbankImporter , module Buchhaltung.Import , getBayesFields ) where import Buchhaltung.Common import Buchhaltung.Import import Control.Arrow import Control.Monad.Cont import Control.Monad.RWS.Strict import Data.Foldable import Data.Functor.Identity import qualified Data.HashMap.Strict as HM import Data.List import qualified Data.ListLike as L import qualified Data.ListLike.String as L import qualified Data.Map.Strict as M import Data.Maybe import Data.Ord import Data.String import qualified Data.Text as T import qualified Data.Text.IO as T import qualified Data.Text.Lazy as TL import Data.Time.Calendar import Debug.Trace import Formatting (sformat, (%), shown) import qualified Formatting.ShortFormatters as F import Safe as S import Text.Parsec import qualified Text.ParserCombinators.Parsec as C import Text.Printf import qualified Text.Regex.TDFA as R import Text.Regex.TDFA.Text () -- * CSV -- findVersion -- :: (MonadError Msg m, Show k, Ord k) => Maybe k -> M.Map k b -> m b headerInfo :: MonadError Msg m => VersionedCSV a -> Maybe Version -> m (SFormat Version, CsvImport a) headerInfo g v = do (format, map) <- g let convert rh = (cVersion rh <$ format, rh) convert . cRaw <&> lookupErrM (printf "Version is not defined for format '%s%'" $ fName format) M.lookup (fromMaybe (fromDefaultVersion $ fVersion format) v) map -- | Data type for preprocessing and meta-data extraction of CSV files type Preprocessor env1 env2 = forall m. MonadError Msg m => (T.Text, env1) -> m (T.Text, env2) processLines :: ([T.Text] -> [T.Text]) -> Preprocessor env env processLines f = return . (first $ T.unlines . f . T.lines) csvImport = csvImportPreprocessed return csvImportPreprocessed :: Preprocessor env1 env2 -> VersionedCSV env2 -> T.Text -> CommonM (env1, Maybe Version) [ImportedEntry] csvImportPreprocessed pp versionedCsv csv1 = do (env, version) <- reader oEnv (csv2, env2) <- pp (csv1, env) (form, g@CSV{cHeader=expected, cDescription = desc, cVersion = version }) <- headerInfo versionedCsv version let toEntry x = ImportedEntry { ieT = genTrans date (vdate =<< cVDate g x) (getCsvConcatDescription env2 desc x) , ieSource = fromMapToSource form x , iePostings = (\p -> (AccountId (cBank g env2 x) (cAccount p x) , cAmount p x , ($ x) <$> cSuffix p , cNegate p x)) . ($ env2) <$> cPostings g } where vdate vd = if date == vd then Nothing else Just vd date = cDate g x (header, rows) = (if cStrip g then stripCsv else id) $ parseCsv (cSeparator g) . TL.fromStrict $ csv2 if expected == header then return $ fmap toEntry $ filter (cFilter g) rows else throwError $ L.unlines [sformat ("Headers do not match. Expected by format '"% F.st%"' version '"%F.st%"':") (fName form) version ,fshow expected ,"Given:" ,fshow header ] -- * AQBanking -- -- imports output from @aqbankingcli listtrans@ aqbankingImporter :: Importer env aqbankingImporter = Importer Nothing $ csvImport aqbankingImport aqbankingImport :: VersionedCSV env aqbankingImport = toVersionedCSV (SFormat "aqBanking" $ DefaultVersion "4") [CSV { cFilter = const True , cDate = readdate . getCsv "date" , cStrip = False , cVDate = Just . readdate . getCsv "valutadate" , cBank = const $ getCsv "localBankCode" , cPostings = [ const CsvPosting { cAccount = getCsv "localAccountNumber" , cAmount = getCsvConcat [ "value_value" , "value_currency"] , cSuffix = Nothing , cNegate = const False }] , cSeparator = ';' , cVersion= "4" , cHeader = ["transactionId" ,"localBankCode" ,"localAccountNumber" ,"remoteBankCode" ,"remoteAccountNumber" ,"date" ,"valutadate" ,"value_value" ,"value_currency" ,"localName" ,"remoteName" ,"remoteName1" ,"purpose" ,"purpose1" ,"purpose2" ,"purpose3" ,"purpose4" ,"purpose5" ,"purpose6" ,"purpose7" ,"purpose8" ,"purpose9" ,"purpose10" ,"purpose11" ,"category" ,"category1" ,"category2" ,"category3" ,"category4" ,"category5" ,"category6" ,"category7"] , cDescription = Field <$> desc , cBayes = ["remoteBankCode","remoteAccountNumber"] ++ desc }] where desc = concatMap (\(f,i) -> (f <>) <$> "":i) [ ("remoteName", ["1"]) , ("purpose", fshow <$> [1..11]) , ("category", fshow <$> [1..7])] -- * Postbank Germany Kontoauszüge (from PDF with @pdftotext@) -- fromPostbankPDF2 :: T.Text -> [ImportedEntry] -- fromPostbankPDF2 xx = fmap (f . mconcat) $ groupBy y $ readcsvrow ',' <$> L.lines xx -- where y a b = head b=="" -- f :: [T.Text] -> ImportedEntry -- f l@(dat:(_:(des:(am:rest)))) = ImportedEntry{ -- ieT = genTrans (parseDateDE $ dat <> "2014") Nothing (L.unwords s) -- ,ieSource = v $ T.intercalate (v $ T.singleton hbci_sep) l -- ,iePostings=("Aktiva:Konten:Giro", a <> " EUR") -- } -- where s = (c des):rest -- c = L.unwords . tail . L.words -- a = mconcat $ L.words $ comma am -- v r = "\"" <> r <> "\"" -- -- "/home/data/finanzen/imported/daniela_manuell1.csv" -- fromPostbankPDF1 :: T.Text -> [ImportedEntry] -- fromPostbankPDF1 xx = fmap (f.concat) $ groupBy y $ readcsvrow ',' <$> L.lines xx -- where y a b = head b=="" -- f :: [T.Text] -> ImportedEntry -- f l@(dat:(des:(am:rest))) = ImportedEntry{ -- ieT = genTrans (parseDateDE $ dat <> "2013") Nothing (L.unwords s) -- ,ieSource = v $ T.intercalate (v $ T.singleton hbci_sep) l -- ,iePostings=("Aktiva:Konten:Giro", a <> " EUR") -- } -- where s = (c des):rest -- c = L.unwords . tail . L.words -- a = mconcat $ L.words $ comma am -- v r = "\"" <> r <> "\"" -- * Comdirect Germany -- -- uses the old way. Do not adopt! comdirectToAqbanking :: IO () comdirectToAqbanking = toAqbanking2 ';' T.getContents comdirect_header comdirect_mapping $ const True comdirect_header :: [[Char]] comdirect_header = ["Buchungstag","Wertstellung (Valuta)","Vorgang","Buchungstext","Umsatz in EUR"] comdirect_header_visa :: [[Char]] comdirect_header_visa = ["Buchungstag","Umsatztag","Vorgang","Referenz","Buchungstext","Umsatz in EUR"] comdirect_mapping_visa :: [([Char], T.Text -> T.Text)] comdirect_mapping_visa = [ ("Referenz",const ""), ( "Referenz" , undefined), ( "Referenz" , const "Visa") ] ++ empty 2 ++ [ ( "Buchungstag", fshow . readdate2), ( "Umsatztag", fshow. readdate2), ( "Umsatz in EUR", comma ), ( "Umsatz in EUR", const "EUR" ), ( "Umsatz in EUR", const "Johannes Gerer" ), ( "Buchungstext", id), ("Referenz",const ""), ( "Vorgang",id), ( "Referenz",id)] ++ empty (32-14) where empty x = take x $ repeat ( "Referenz" , const "") comdirect_mapping :: [([Char], T.Text -> T.Text)] comdirect_mapping = [ ("Buchungstag",const ""), ( "Buchungstag" , undefined), ( "Buchungstag" , const "Visa") ] ++ empty 2 ++ [ ( "Buchungstag", fshow . readdate2), ( "Wertstellung (Valuta)", fshow . readdate2), ( "Umsatz in EUR", const "WTF" ), ( "Umsatz in EUR", const "EUR" ), ( "Umsatz in EUR", const "Johannes Gerer" ), ( "Buchungstext", id), ( "Vorgang",id)] ++ empty (32-10-2) where empty x = take x $ repeat ( "Buchungstag" , const "") -- comdirectVisaCSVImport :: AccountMap -> T.Text -> [ImportedEntry] -- comdirectVisaCSVImport accountMappings = -- aqbankingCsvImport accountMappings . toAqbanking2Pure ';' -- comdirect_header_visa comdirect_mapping_visa (const True) -- . T.replace "\n\"Neu\";" "" . L.unlines . ok ok :: (IsString b, Eq b, L.StringLike b) => b -> [b] ok = L.takeWhile (/= fromString "") . L.dropWhile (/= fromString "\"Buchungstag\";\"Umsatztag\";\"Vorgang\";\"Referenz\";\"Buchungstext\";\"Umsatz in EUR\";") . L.lines p :: ParsecT [Char] u Identity [Char] p = do a <- C.manyTill C.anyChar (C.try $ C.string "\n\"Neu\";") return a -- comdirectVisaImporter :: CustomImport2 -- comdirectVisaImporter = Importer windoof comdirectVisaCSVImport -- paypalImport :: AccountMap -> T.Text -> [ImportedEntry] -- paypalImport accountMappings = aqbankingCsvImport accountMappings . myf2 ',' -- paypal_header (paypal_mapping " Brutto" "") filt -- where filt x = not $ "Storniert" `elem` x -- myf2 sep header mapping' filtercond = -- T.intercalate "\n" . show2 . (:) csv_header -- . map appl . filter filtercond . drop 1 . (readcsv sep) -- where appl = map (\(a,b) -> a b) . zip transformation . description_list header mapping -- mapping = map fst mapping' hbci_sep = ';' -- | Descriptions create the description, by concatenation of all cols description :: (L.ListLike c item, L.StringLike c, Show a, Eq a) => [a] -> [c] -> c description cols r = L.unwords . filter (not . L.null) $ description_list csv_header cols r description_list :: (Show a, Eq a) => [a] -> [a] -> [b] -> [b] description_list t cols r = map fst $ sorted r where desc_indices = map (idx t) cols -- check if all desired columns exist magic v k = do i <- elemIndices k desc_indices return (v,i) -- for every col in desc_cols get a pair containing the value and -- the index in the desc_cols arrays sorted r = sortBy (comparing snd) $ mconcat $ zipWith magic r [0..] -- extract all desc_cols in the specified order -- transformation = map snd mapping' csv_header = undefined -- * PNC Bank USA transaction logs pncbankImporter :: Importer T.Text pncbankImporter = Importer windoof $ csvImport pncbank pncbank :: VersionedCSV T.Text pncbank = toVersionedCSV (SFormat "pncbank" $ DefaultVersion "May 2017") [CSV { cFilter =(/= "") . getCsv "Date" , cDate = parseDateUS . getCsv "Date" , cStrip = False , cVDate = Just . parseDateUS . getCsv "Date" , cBank = const $ const "PNC Bank" , cPostings = [ \env -> CsvPosting { cAccount = const env , cAmount = textstrip . (T.replace "$" "") . (T.replace "," "") . (<> " USD") . getCsvCreditDebit "Withdrawals" "Deposits" , cSuffix = Nothing , cNegate = const False }] , cSeparator = ',' , cHeader = ["Date" ,"Description" ,"Withdrawals" ,"Deposits" ,"Balance" ] , cDescription = Field <$> desc , cBayes = desc , cVersion = "May 2017" } ] where desc = ["Description"] -- * Barclaycard US transaction logs barclaycardusImporter :: Importer () barclaycardusImporter = Importer windoof $ csvImportPreprocessed barclaycardPreprocessor barclaycardus barclaycardPreprocessor :: Preprocessor () AccountId barclaycardPreprocessor (wholeFile, _) = maybe e (return . (,) (T.unlines body) . AccountId bank . T.dropWhile (== 'X')) $ T.stripPrefix accountPrefix accountLine where (bank : accountLine : _ : _ : body) = T.lines wholeFile accountPrefix = "Account Number: " e = throwError $ "Expected second line of file to begin with prefix " `T.append` accountPrefix barclaycardus :: VersionedCSV AccountId barclaycardus = toVersionedCSV (SFormat "barclaycard" $ DefaultVersion "May 2017") [CSV { cFilter =(/= "") . getCsv "Transaction Date" , cDate = parseDateUS . getCsv "Transaction Date" , cStrip = False , cVDate = Just . parseDateUS . getCsv "Transaction Date" , cBank = const . aBank , cPostings = [ \env -> CsvPosting { cAccount = const $ aAccount env , cAmount = textstrip . (<> " USD") . getCsv "Amount" , cSuffix = Nothing , cNegate = const False }] , cSeparator = ',' , cHeader = ["Transaction Date" ,"Description" ,"Category" ,"Amount" ] , cDescription = Field <$> desc , cBayes = "Category" : desc , cVersion = "May 2017" } ] where desc = ["Description"] -- * Revolut Csv revolutImporter :: Importer (RevolutSettings ()) revolutImporter = Importer Nothing $ csvImportPreprocessed extractCurrency revolut extractCurrency :: Preprocessor (RevolutSettings ()) (RevolutSettings T.Text) extractCurrency (text, env) = do cur <- maybe (throwError $ "Cannot find currency (regular expression: " <> T.pack (show currencyColumn) <> " in header:\n" <> header) (return . fst) $ (flip atMay 1 . toList =<<) $ listToMaybe $ currencyRegex header return (T.unlines $ T.replace (" (" <> cur <> ")") "" header:rest , const cur <$> env) where header:rest = T.lines text currencyRegex = R.matchAllText (R.makeRegex currencyColumn :: R.Regex) currencyColumn = "\\bPaid Out \\(([^)]+)\\)" :: T.Text revolut :: VersionedCSV (RevolutSettings T.Text) revolut = toVersionedCSV (SFormat "revolut" $ DefaultVersion "2017") [ CSV { cFilter = (/= "") . getCsv "Completed Date" , cDate = (\x -> headNote ("no parse of " <> T.unpack x) $ mapMaybe (\format -> parseDateM format x) ["%b %e, %Y", "%e %b %Y"]) . getCsv "Completed Date" , cStrip = True , cVDate = const Nothing , cBank = const $ const "Revolut" , cPostings = [ \env -> CsvPosting { cAccount = const $ revolutUser env , cAmount = (<> revolutCurrency env) . getCsvCreditDebit "Paid Out" "Paid In" , cSuffix = Just $ const $ revolutCurrency env , cNegate = const False } ] , cSeparator = ';' , cHeader = ["Completed Date" ,"Reference" ,"Paid Out" ,"Paid In" ,"Exchange Out" ,"Exchange In" ,"Balance" ,"Category" ] , cDescription = [Const "Revolut" , Field "Reference" , Const ", Category" , Field "Category" ] , cBayes = ["Reference", "Category" ] , cVersion = "2017" } ] -- remove currency signs and append corresponding currency name normalizeCurrency :: T.Text -> T.Text normalizeCurrency text = (`runCont` id) $ callCC $ \exit -> do let g (symbol, name) text1 = unless (T.length text2 == L.length text1) $ exit $ T.replace " " "" text2 <> " " <> name where text2 = T.replace symbol "" text1 mapM_ (\x -> g x text) currencySymbols return text currencySymbols = [ ("$", "USD") , ("€", "EUR") , ("£", "GBP") ] -- * Monefy Csv monefyImporter :: Importer MonefySettings monefyImporter = Importer Nothing $ csvImportPreprocessed unambiguousHeader monefy -- | replace the header by one with unique column names (currency2) unambiguousHeader :: Preprocessor env env unambiguousHeader = processLines $ (h2:) . tail where h2 = "date,account,category,amount,currency,converted amount,currency2,description" -- (T.replace "ü" "ue" <$> rest) monefy :: VersionedCSV MonefySettings monefy = toVersionedCSV (SFormat "monefy" $ DefaultVersion "2017") [CSV { cFilter = const True , cStrip = False , cDate = parseDate "%d/%m/%Y" . getCsv "date" , cVDate = const Nothing , cBank = const . monefyInstallation , cPostings = [ const CsvPosting { cAccount = getCsv "account" , cAmount = amt , cSuffix = Nothing , cNegate = const False } , \env -> let suf = monefyCategorySuffix env in CsvPosting { cAccount = if suf then const "Monefy Category Account" else getCsv "category" , cAmount = amt , cSuffix = if suf then Just $ getCsv "category" else Nothing , cNegate = const True } ] , cSeparator = ',' , cHeader = ["date" ,"account" ,"category" ,"amount" ,"currency" ,"converted amount" ,"currency2" ,"description" ] , cDescription = [Const "Monefy" , Read monefyInstallation , Field "description"] , cBayes = [] , cVersion = "2017" } ] where amt = (\a b -> textstrip $ T.replace "," "" a <> " " <> b) <$> getCsv "amount" <*> getCsv "currency" -- * BarclaysUk International CSV export barclaysUkImporter :: Importer () barclaysUkImporter = Importer Nothing $ csvImport barclaysUk barclaysUk :: VersionedCSV () barclaysUk = toVersionedCSV (SFormat "barclaysUk" $ DefaultVersion "2017") [CSV { cFilter = (/= "") . getCsv "Date" , cStrip = False , cDate = parseDate "%d/%m/%Y" . getCsv "Date" , cVDate = const Nothing , cBank = const $ fst <$> accountBank , cPostings = [ const CsvPosting { cAccount = snd <$> accountBank , cAmount = (<> " GBP") <$> getCsv "Amount" , cSuffix = Nothing , cNegate = const False } ] , cSeparator = ',' , cHeader = ["Number" ,"Date" ,"Account" ,"Amount" ,"Subcategory" ,"Memo" ] , cDescription = Field <$> desc , cBayes = desc , cVersion = "2017" } ] where accountBank = (g . T.splitOn " ") . getCsv "Account" g [acc,bank] = (acc, bank) g _ = error "Expected 'Account' to be of the format \"{sort code} {account number}\"" desc = ["Subcategory", "Memo"] -- * Natwest International CSV export natwestIntlImporter :: Importer () natwestIntlImporter = Importer Nothing $ csvImportPreprocessed (processLines $ tail) natwestIntl natwestIntl :: VersionedCSV () natwestIntl = toVersionedCSV (SFormat "natwestIntl" $ DefaultVersion "2017") [CSV { cFilter = (/= "") . getCsv "Date" , cStrip = False , cDate = parseDate "%d/%m/%Y" . getCsv "Date" , cVDate = const Nothing , cBank = const $ fst <$> accountBank , cPostings = [ const CsvPosting { cAccount = snd <$> accountBank , cAmount = (<> " GBP") <$> getCsv " Value" , cSuffix = Nothing , cNegate = const False } ] , cSeparator = ',' , cHeader = ["Date" ," Type" ," Description" ," Value" ," Balance" ," Account Name" ," Account Number" ] , cDescription = [Field " Description" ,Const ", Type" ,Field " Type"] , cBayes = [" Description" ," Type"] , cVersion = "2017" } ] where accountBank = (g . T.splitOn "-" . T.tail ) . getCsv " Account Number" g [acc,bank] = (acc, bank) g _ = error "Expected ' Account Number' to be of the format \"'{sort code}-{account number}\"" -- natwestTransactionType = -- [("103" ,"MT103 Payment") -- ,("ACI" ,"Interest on Account Balance") -- ,("ADV" ,"Separate Advice") -- ,("AMD" ,"Amendments History") -- ,("ATM" ,"Cash Withdrawal") -- ,("BAC" ,"Automated Credit") -- ,("BAE" ,"Branch Account Entry") -- ,("BCO" ,"Non Market Close Out") -- ,("BGC" ,"Bank Giro Credit") -- ,("BGT" ,"Guarantees") -- ,("BLN" ,"Bankline Charges") -- ,("BOE" ,"Bill of Exchange") -- ,("BSP" ,"Branch single payment") -- ,("C/R" ,"Credit") -- ,("C/L" ,"Automated teller machine cash withdrawal") -- ,("CAE" ,"Cheque Collection") -- ,("CCB" ,"Cheque Collection") -- ,("CDM" ,"Cash and Deposit Machine") -- ,("CHG" ,"Charges") -- ,("CHP" ,"CHAPS Transfer (NatWest only)") -- ,("CHQ" ,"Cheque") -- ,("CNA" ,"Clean Cheque Neg") -- ,("CND" ,"Cheque Negotiation") -- ,("COM" ,"Commission") -- ,("CRD" ,"Card Payment or Cash") -- ,("D/D" ,"Direct Debit") -- ,("D/R" ,"Debit") -- ,("DCR" ,"Documentary Credit") -- ,("DFT" ,"Foreign Draft") -- ,("DIV" ,"Dividend") -- ,("DPC" ,"Digital Banking Payment") -- ,("EBP" ,"Electronic Payment") -- ,("FPAY" ,"Faster Payment - Future Dated (Appears on statements as EBP)") -- ,("GSD" ,"Gov Stamp Duty") -- ,("IBP" ,"Inter Branch Payment") -- ,("ICP" ,"Inward Currency Payment") -- ,("INT" ,"Interest") -- ,("INV" ,"Investment") -- ,("IPAY" ,"Faster Payment -Immediate (Appears on statements as EBP)") -- ,("ISP" ,"Inward Sterling Payment") -- ,("ITL" ,"International Transfer & Treasury Settlements (and RBS CHAPS Payments)") -- ,("ITM" ,"Incoming CHAPS") -- ,("LON" ,"New Loan") -- ,("LST" ,"Supplementary List") -- ,("LVP" ,"Low Value Payment") -- ,("MEC" ,"Export Credits") -- ,("MFD" ,"Maturing Fwd Deal") -- ,("MGT" ,"Bonds & Guarantees") -- ,("MIB" ,"Inward Bills") -- ,("MIC" ,"Import Credits") -- ,("MKD" ,"Market Deal") -- ,("MOB" ,"Outward Bills") -- ,("MSC" ,"Miscellaneous Entry") -- ,("NDC" ,"No Dividend Counterfoil") -- ,("NPAY" ,"Faster Payment - Next Day (Appears on statement as EBP)") -- ,("POS" ,"Maestro Transaction") -- ,("RTF" ,"Relay Transfer") -- ,("S/O" ,"Standing Order") -- ,("SBT" ,"Funds Transfer") -- ,("SCR" ,"Sundry Credit Item") -- ,("SDE" ,"Urgent Euro Transfer") -- ,("SDR" ,"Sundry Debit Item") -- ,("STF" ,"Manually Keyed Standard Transfer") -- ,("STL" ,"Settlement") -- ,("TFP" ,"Trade Finance Product") -- ,("TFR" ,"Transfer") -- ,("TRF" ,"International Payment (NatWest only)") -- ,("TLR" ,"Card Payment or Cash") -- ,("TEL" ," Telephone Banking transaction") -- ,("TSU" ,"Telephone Banking") -- ,("U/D" ,"Unpaid Direct Debit") -- ,("UTF" ,"Urgent Transfer") -- ,("WSF" ,"Foreign Exchange Deal") -- ,("WSM" ,"Money Market Deal")] -- * Comdirect Visa Statements comdirectVisaImporter :: Importer T.Text comdirectVisaImporter = Importer windoof $ csvImport comdirectVisa comdirectVisa :: VersionedCSV T.Text comdirectVisa = toVersionedCSV (SFormat "visa" $ DefaultVersion "manuell") [CSV { cFilter =(/= "") . getCsv "Buchungstag" , cStrip = False , cDate = parseDateDE . getCsv "Buchungstag" , cVDate = Just . parseDateDE . getCsv "Valuta" , cBank = const , cPostings = [ const CsvPosting { cAccount = const "Visa" , cAmount = textstrip . comma . (<> " EUR") . getCsv "Ausgang" , cSuffix = Nothing , cNegate = const False }] , cSeparator = ',' , cHeader = ["Buchungstag" ,"Vorgang" ,"Buchungstext" ,"Ausgang" ,"Valuta" ,"Referenz" ,"Buchungstext2" ] , cDescription = Field <$> desc , cBayes = desc , cVersion = "manuell" -- hand extracted from @pdftotext -layout@ } , CSV { cFilter =(/= "") . getCsv "Buchungstag" , cDate = parseDateDE . getCsv "Buchungstag" , cStrip = False , cVDate = Just . parseDateDE . getCsv "Umsatztag" , cBank = const , cPostings = [ const CsvPosting { cAccount = const "Visa" , cAmount = comma . (<> " EUR") . getCsv "Umsatz in EUR" , cSuffix = Nothing , cNegate = const False }] , cSeparator = ',' , cHeader = ["Buchungstag" ,"Umsatztag" ,"Vorgang" ,"Referenz" ,"Buchungstext" ,"Umsatz in EUR"] , cDescription = Field <$> desc2 , cBayes = desc2 , cVersion = "export" } ] where desc = ["Vorgang" ,"Buchungstext" ,"Buchungstext2" ] desc2 = ["Vorgang" ,"Buchungstext" ] -- * Paypal (German) -- -- understands exports under the following setting: -- -- @ -- alle guthaben relevanten Zahlungen (kommagetrennt) ohne warenkorbdetails! -- @ paypalImporter :: Importer T.Text paypalImporter = Importer windoof $ csvImport paypalImport paypalImport :: VersionedCSV T.Text paypalImport = let base = CSV { cFilter = (/= "Storniert") . getCsv " Status" , cDate = parseDateDE . getCsv "Datum" , cStrip = False , cVDate = const Nothing , cBank = const $ const "Paypal" , cPostings = [ \env -> CsvPosting { cAccount = const env , cAmount = comma . getCsvConcat [ " Netto" , " Währung"] , cSuffix = Nothing , cNegate = const False }] , cSeparator = ',' , cVersion = "undefined" , cHeader = [] , cBayes = ["undefined"] , cDescription = [Field "undefined"] } desc = Field <$> [" Name" ," Verwendungszweck" ," Art" ," Zeit"] desc2 = Field <$> [" Name" ," Artikelbezeichnung" ," Typ" ," Zeit"] in toVersionedCSV (SFormat "paypal" $ DefaultVersion "2017") [base { cVersion = "2017" , cHeader = ["Datum" ," Zeit" ," Zeitzone" ," Name" ," Typ" ," Status" ," Betreff" ," W\195\164hrung" ," Brutto" ," Geb\195\188hr" ," Netto" ," Hinweis" ," Von E-Mail-Adresse" ," An E-Mail-Adresse" ," Transactionscode" ," Zahlungsart" ,"Status der Gegenpartei" ," Lieferadresse" ," Adressstatus" ," Artikelbezeichnung" ," Artikelnummer" ," Betrag f\195\188r Versandkosten" ," Versicherungsbetrag" ," Umsatzsteuer" ," Trinkgeld" ," Rabatt" ," Mitgliedsname des Verk\195\164ufers" ," Option 1 - Name" ," Option 1 - Wert" ," Option 2 - Name" ," Option 2 - Wert" ," Auktions-Site" ," K\195\164ufer-ID" ," Artikel-URL" ," Angebotsende" ," Txn-Referenzkennung" ," Rechnungsnummer" ," Abonnementnummer" ," Individuelle Nummer" ," Belegnummer" ," Guthaben" ," Adresszeile 1" ," Zus\195\164tzliche Angaben" ," Ort" ," Staat/Provinz/Region/Landkreis/Territorium/Pr\195\164fektur/Republik" ," PLZ" ," Land" ," Telefonnummer" ," Auswirkung auf Guthaben" ," "] , cBayes = [" Name" ," An E-Mail-Adresse" ," Von E-Mail-Adresse" ," Artikelbezeichnung" ," Typ" ," Status" ," K\195\164ufer-ID" , "Status der Gegenpartei"," Adressstatus" , " Option 1 - Name" , " Option 2 - Name" ," Auktions-Site" ," K\195\164ufer-ID" ," Artikel-URL" ," Adresszeile 1" ," Zus\195\164tzliche Angaben" ," Ort" ," Staat/Provinz/Region/Landkreis/Territorium/Pr\195\164fektur/Republik" ," PLZ" ," Land" ," Telefonnummer" ] , cDescription = desc2 } ,base { cVersion = "2016" , cHeader = ["Datum" ," Zeit" ," Zeitzone" ," Name" ," Typ" ," Status" ," W\228hrung" ," Brutto" ," Geb\252hr" ," Netto" ," Von E-Mail-Adresse" ," An E-Mail-Adresse" ," Transactionscode" ," Status der Gegenpartei" ," Adressstatus" ," Artikelbezeichnung" ," Artikelnummer" ," Betrag f\252r Versandkosten" ," Versicherungsbetrag" ," Umsatzsteuer" ," Option 1 - Name" ," Option 1 - Wert" ," Option 2 - Name" ," Option 2 - Wert" ," Auktions-Site" ," K\228ufer-ID" ," Artikel-URL" ," Angebotsende" ," Vorgangs-Nr." ," Rechnungs-Nr." ," Txn-Referenzkennung" ," Rechnungsnummer" ," Individuelle Nummer" ," Belegnummer" ," Guthaben" ," Adresszeile 1" ," Zus\228tzliche Angaben" ," Ort" ," Staat/Provinz/Region/Landkreis/Territorium/Pr\228fektur/Republik" ," PLZ" ," Land" ," Telefonnummer" ," "] , cBayes = [" Name" ," An E-Mail-Adresse" ," Von E-Mail-Adresse" ," Artikelbezeichnung" ," Typ" ," Status" ," Käufer-ID" , " Status der Gegenpartei"," Adressstatus" , " Option 1 - Name" , " Option 2 - Name" ," Auktions-Site" ," K\228ufer-ID" ," Artikel-URL" ," Adresszeile 1" ," Zus\228tzliche Angaben" ," Ort" ," Staat/Provinz/Region/Landkreis/Territorium/Pr\228fektur/Republik" ," PLZ" ," Land" ," Telefonnummer" ] , cDescription = desc2 } ,base { cVersion = "2014" , cHeader = ["Datum" ," Zeit" ," Zeitzone" ," Name" ," Art" ," Status" ," W\228hrung" ," Brutto" ," Geb\252hr" ," Netto" ," Von E-Mail-Adresse" ," An E-Mail-Adresse" ," Transaktionscode" ," Status der Gegenpartei" ," Adressstatus" ," Verwendungszweck" ," Artikelnummer" ," Betrag f\252r Versandkosten" ," Versicherungsbetrag" ," Umsatzsteuer" ," Option 1 - Name" ," Option 1 - Wert" ," Option 2 - Name" ," Option 2 - Wert" ," Auktions-Site" ," K\228ufer-ID" ," Artikel-URL" ," Angebotsende" ," Vorgangs-Nr." ," Rechnungs-Nr." ," Txn-Referenzkennung" ," Rechnungsnummer" ," Individuelle Nummer" ," Best\228tigungsnummer" ," Guthaben" ," Adresse" ," Zus\228tzliche Angaben" ," Ort" ," Staat/Provinz/Region/Landkreis/Territorium/Pr\228fektur/Republik" ," PLZ" ," Land" ," Telefonnummer der Kontaktperson" ," "] , cBayes = [" Name" ," An E-Mail-Adresse" ," Von E-Mail-Adresse" ," Verwendungszweck" ," Art" ," Status" ," Käufer-ID" , " Status der Gegenpartei"," Adressstatus" , " Option 1 - Name" , " Option 2 - Name" ," Auktions-Site" ," K\228ufer-ID" ," Artikel-URL" ," Adresse" ," Zus\228tzliche Angaben" ," Ort" ," Staat/Provinz/Region/Landkreis/Territorium/Pr\228fektur/Republik" ," PLZ" ," Land" ," Telefonnummer der Kontaktperson" ] , cDescription = desc } , base { cVersion = "2013" , cHeader = ["Datum" ," Zeit" ," Zeitzone" ," Name" ," Art" ," Status" ," W\228hrung" ," Brutto" ," Geb\252hr" ," Netto" ," Von E-Mail-Adresse" ," An E-Mail-Adresse" ," Transaktionscode" ," Status der Gegenpartei" ," Adressstatus" ," Verwendungszweck" ," Artikelnummer" ," Betrag f\252r Versandkosten" ," Versicherungsbetrag" ," Umsatzsteuer" ," Option 1 - Name" ," Option 1 - Wert" ," Option 2 - Name" ," Option 2 - Wert" ," Auktions-Site" ," K\228ufer-ID" ," Artikel-URL" ," Angebotsende" ," Vorgangs-Nr." ," Rechnungs-Nr." ," Txn-Referenzkennung" ," Rechnungsnummer" ," Individuelle Nummer" ," Menge" ," Best\228tigungsnummer" ," Guthaben" ," Adresse" ," Zus\228tzliche Angaben" ," Ort" ," Staat/Provinz/Region/Landkreis/Territorium/Pr\228fektur/Republik" ," PLZ" ," Land" ," Telefonnummer der Kontaktperson" ," "] , cBayes = [" Name" ," An E-Mail-Adresse" ," Von E-Mail-Adresse" ," Verwendungszweck" ," Art" ," Status" ," Käufer-ID" ," Status der Gegenpartei"," Adressstatus" ," Option 1 - Name" ," Option 2 - Name" ," Auktions-Site" ," K\228ufer-ID" ," Artikel-URL" ," Adresse" ," Zus\228tzliche Angaben" ," Ort" ," Staat/Provinz/Region/Landkreis/Territorium/Pr\228fektur/Republik" ," PLZ" ," Land" ," Telefonnummer der Kontaktperson" ] , cDescription = desc }] -- * other stuff show2 :: [[T.Text]] -> [T.Text] show2 = map (T.intercalate ";" . map (\x -> "\"" <> (quote x) <> "\"")) where quote x = T.replace "\"" escapedDoubleQuotes x hibiscusToAqbanking :: IO () hibiscusToAqbanking = toAqbanking ';' (T.readFile "/home/data/finanzen/imported/hibiscus-export-20120930.csv") hibicus_header hibiscus_mapping hibiscus_transf $ const True toAqbankingPure :: (Show a, Eq a) => Char -> [a] -> [a] -> [T.Text -> T.Text] -> ([T.Text] -> Bool) -> T.Text -> T.Text toAqbankingPure sep header mapping transformation filtercond = T.intercalate "\n" . show2 . (:) csv_header . map appl . filter filtercond . drop 1 . (readcsv sep) where appl = map (\(a,b) -> a b) . zip transformation . description_list header mapping toAqbanking :: (Show a, Eq a) => Char -> IO T.Text -> [a] -> [a] -> [T.Text -> T.Text] -> ([T.Text] -> Bool) -> IO () toAqbanking sep f header mapping transf filt = T.putStr =<< toAqbankingPure sep header mapping transf filt <$> f toAqbanking2Pure :: (Show a, Eq a) => Char -> [a] -> [(a, T.Text -> T.Text)] -> ([T.Text] -> Bool) -> T.Text -> T.Text toAqbanking2Pure sep header mapping filt = toAqbankingPure sep header mapping' transf filt where mapping' = map fst mapping transf = map snd mapping toAqbanking2 sep f header mapping filt = T.writeFile "/tmp/new" =<< toAqbanking2Pure sep header mapping filt <$> f hibicus_header :: [[Char]] hibicus_header =["Kontonummer","BLZ","Konto","Gegenkonto","Gegenkonto BLZ","Gegenkonto Inhaber","Betrag","Valuta","Datum","Verwendungszweck","Verwendungszweck 2","Zwischensumme","Primanota","Kundenreferenz","Kategorie","Kommentar","Weitere Verwendungszwecke"] hibiscus_mapping :: [[Char]] hibiscus_mapping = ["Primanota","BLZ","Kontonummer","Gegenkonto BLZ","Gegenkonto","Datum","Valuta","Betrag" , "Betrag" , "Betrag" , "Gegenkonto Inhaber", "Betrag", "Verwendungszweck","Verwendungszweck 2","Kommentar","Weitere Verwendungszwecke"] ++ (take 16 $ repeat "Betrag") hibiscus_transf :: [T.Text -> T.Text] hibiscus_transf = [id, id , id , id , id , fshow . readdate2, fshow . readdate2,comma ,const "EUR", const "Johannes Gerer", id , const ""] <> (take 4 $ repeat id) <> (take 16 $ repeat $ const "") :: [T.Text -> T.Text] comma :: T.Text -> T.Text comma = T.replace "," "." . T.replace "." "" :: T.Text -> T.Text readdate2 :: Stream t Data.Functor.Identity.Identity Char => t -> Date2 readdate2 s = either (error.show) id (parse p_date2 "date" s) p_date2 :: Stream t Data.Functor.Identity.Identity Char => Parsec t () Date2 p_date2 = do d <- many1 digit char '.' m <- many1 digit char '.' y <- many1 digit return (D y m d) readdate :: T.Text -> Day readdate s = either (error.show) id (parse p_date "date" s) p_date :: Stream t Data.Functor.Identity.Identity Char => Parsec t () Day p_date = do y <- many1 digit char '/' m <- many1 digit char '/' d <- many1 digit return $ fromGregorian (read y) (read m) (read d) instance Show Date2 where show (D y m d) = concat [y, "/", m, "/", d] data Date2 = D String String String -- year month day deriving (Eq) toBayes :: MonadError Msg m => VersionedCSV a -> m (SFormat DefaultVersion, M.Map Version [T.Text]) toBayes = fmap (second $ fmap $ cBayes . cRaw) -- create a map that associates each format with a map that associates -- each version with a list of fields defaultFields :: MonadError Msg m => m (M.Map (SFormat ()) (M.Map Version [T.Text])) defaultFields = fromListUnique . fmap (first $ (() <$)) =<< sequence [toBayes paypalImport ,toBayes aqbankingImport ,toBayes barclaycardus ,toBayes pncbank ,toBayes revolut ,toBayes natwestIntl ,toBayes barclaysUk ,toBayes monefy] -- extract the values of all available bayes fields of a given -- source. getBayesFields :: MonadError Msg m => Source -> m [T.Text] getBayesFields source = do -- extract the correct bayes fields that correspond to the source's -- format and version. fields <- lookupErrM "Version has to be configured" M.lookup (fVersion format) =<< lookupErrM "Format has to be configured" M.lookup (() <$ format) =<< defaultFields return $ mapMaybe (flip M.lookup $ sStore source) fields where format = sFormat source