{-# LANGUAGE DeriveDataTypeable #-} {-# LANGUAGE FlexibleInstances #-} {-# LANGUAGE GADTs #-} {-# LANGUAGE QuasiQuotes #-} {-# LANGUAGE RecordWildCards #-} {-# LANGUAGE StandaloneDeriving #-} {-# LANGUAGE TemplateHaskell #-} {-# LANGUAGE TypeFamilies #-} -- | Parse TSN XML for the DTD \"Odds_XML.dtd\". Each document -- contains a root element \ that contains a bunch of -- other... disorganized... information. -- module TSN.XML.Odds ( dtd, pickle_message, -- * Tests odds_tests, -- * WARNING: these are private but exported to silence warnings OddsCasinoConstructor(..), OddsConstructor(..), OddsGame_OddsGameTeamConstructor(..), OddsGameConstructor(..), OddsGameLineConstructor(..), OddsGameTeamConstructor(..) ) where -- System imports. import Control.Monad ( forM_, join ) import Data.Time ( UTCTime(..) ) import Data.Tuple.Curry ( uncurryN ) import Database.Groundhog ( (=.), (==.), countAll, executeRaw, insert_, migrate, runMigration, silentMigrationLogger, update ) import Database.Groundhog.Core ( DefaultKey ) import Database.Groundhog.Generic ( runDbConn ) import Database.Groundhog.Sqlite ( withSqliteConn ) import Database.Groundhog.TH ( groundhog, mkPersist ) import Test.Tasty ( TestTree, testGroup ) import Test.Tasty.HUnit ( (@?=), testCase ) import Text.Read ( readMaybe ) import Text.XML.HXT.Core ( PU, xp5Tuple, xp6Tuple, xp8Tuple, xpAttr, xpElem, xpInt, xpList, xpOption, xpPair, xpText, xpTriple, xpWrap ) -- Local imports. import TSN.Codegen ( tsn_codegen_config ) import TSN.DbImport ( DbImport(..), ImportResult(..), run_dbmigrate ) import TSN.Picklers ( xp_date, xp_time, xp_time_stamp ) import TSN.XmlImport ( XmlImport(..), XmlImportFk(..) ) import Xml ( FromXml(..), FromXmlFk(..), ToDb(..), pickle_unpickle, unpickleable, unsafe_unpickle ) -- | The DTD to which this module corresponds. Used to invoke dbimport. -- dtd :: String dtd = "Odds_XML.dtd" -- -- DB/XML data types -- -- * OddsGameCasino/OddsGameCasinoXml -- | The casinos should have their own table, but the lines don't -- belong in that table (there is a separate table for -- 'OddsGameLine' which associates the two). -- -- We drop the \"Game\" prefix because the casinos really aren't -- children of the games; the XML just makes it seem that way. -- data OddsCasino = OddsCasino { casino_client_id :: Int, casino_name :: String } deriving (Eq, Show) -- | The home/away lines are 'Double's, but the over/under lines are -- textual. If we want to use one data type for both, we have to go -- with a 'String' and then attempt to 'read' a 'Double' later when we -- go to insert the thing. -- data OddsGameCasinoXml = OddsGameCasinoXml { xml_casino_client_id :: Int, xml_casino_name :: String, xml_casino_line :: Maybe String } deriving (Eq, Show) -- | Try to get a 'Double' out of the 'xml_casino_line' which is a -- priori textual (because it might be an over/under line). -- home_away_line :: OddsGameCasinoXml -> Maybe Double home_away_line = join . (fmap readMaybe) . xml_casino_line instance ToDb OddsGameCasinoXml where -- | The database representation of an 'OddsGameCasinoXml' is an -- 'OddsCasino'. -- type Db OddsGameCasinoXml = OddsCasino instance FromXml OddsGameCasinoXml where -- | We convert from XML to the database by dropping the line field. -- from_xml OddsGameCasinoXml{..} = OddsCasino { casino_client_id = xml_casino_client_id, casino_name = xml_casino_name } -- | This allows us to insert the XML representation 'OddsGameCasinoXml' -- directly. -- instance XmlImport OddsGameCasinoXml -- * OddsGameTeam -- | The database representation of teams as they appear in odds -- games. -- data OddsGameTeam = OddsGameTeam { db_team_id :: String, -- ^ The home/away team IDs are -- three characters but Postgres -- imposes no performance penalty -- on lengthless text fields, so -- we ignore the probable upper -- bound of three characters. db_abbr :: String, db_team_name :: String } deriving (Eq, Show) -- * OddsGameHomeTeam/OddsGameHomeTeamXml -- | The XML representation of a \, as found in \s. -- data OddsGameHomeTeamXml = OddsGameHomeTeamXml { xml_home_team_id :: String, -- ^ The home/away team IDs -- are three characters but -- Postgres imposes no -- performance penalty on -- lengthless text fields, -- so we ignore the probable -- upper bound of three -- characters. xml_home_rotation_number :: Int, xml_home_abbr :: String, xml_home_team_name :: String, xml_home_casinos :: [OddsGameCasinoXml] } deriving (Eq, Show) instance ToDb OddsGameHomeTeamXml where -- | The database representation of an 'OddsGameHomeTeamXml' is an -- 'OddsGameTeam'. -- type Db OddsGameHomeTeamXml = OddsGameTeam instance FromXml OddsGameHomeTeamXml where -- | We convert from XML to the database by dropping the lines and -- rotation number (which are specific to the games, not the teams -- themselves). -- from_xml OddsGameHomeTeamXml{..} = OddsGameTeam { db_team_id = xml_home_team_id, db_abbr = xml_home_abbr, db_team_name = xml_home_team_name } -- | This allows us to insert the XML representation -- 'OddsGameHomeTeamXml' directly. -- instance XmlImport OddsGameHomeTeamXml where -- * OddsGameAwayTeam/OddsGameAwayTeamXml -- | The XML representation of a \, as found in \s. -- data OddsGameAwayTeamXml = OddsGameAwayTeamXml { xml_away_team_id :: String, -- ^ The home/away team IDs are -- three characters but Postgres -- imposes no performance penalty -- on lengthless text fields, so -- we ignore the probable upper -- bound of three characters xml_away_rotation_number :: Int, xml_away_abbr :: String, xml_away_team_name :: String, xml_away_casinos :: [OddsGameCasinoXml] } deriving (Eq, Show) instance ToDb OddsGameAwayTeamXml where -- | The database representation of an 'OddsGameAwayTeamXml' is an -- 'OddsGameTeam'. -- type Db OddsGameAwayTeamXml = OddsGameTeam instance FromXml OddsGameAwayTeamXml where -- | We convert from XML to the database by dropping the lines and -- rotation number (which are specific to the games, not the teams -- themselves). -- from_xml OddsGameAwayTeamXml{..} = OddsGameTeam xml_away_team_id xml_away_abbr xml_away_team_name -- | This allows us to insert the XML representation -- 'OddsGameAwayTeamXml' directly. -- instance XmlImport OddsGameAwayTeamXml where -- * OddsGame_OddsGameTeam -- | Database mapping between games and their home/away teams. -- data OddsGame_OddsGameTeam = OddsGame_OddsGameTeam { ogogt_odds_games_id :: DefaultKey OddsGame, ogogt_away_team_id :: DefaultKey OddsGameTeam, ogogt_home_team_id :: DefaultKey OddsGameTeam } -- * OddsGameOverUnderXml -- | XML representation of the over/under. A wrapper around a bunch of -- casino elements. -- newtype OddsGameOverUnderXml = OddsGameOverUnderXml { xml_casinos :: [OddsGameCasinoXml] } deriving (Eq, Show) -- * OddsGameLine -- | This database representation of the casino lines can't be -- constructed from the one in the XML. The casinos within -- Game-\>HomeTeam, Game-\>AwayTeam, and Game-\>Over_Under are all more or -- less the same. We don't need a bajillion different tables to -- store that, just one tying the casino/game pair to the three -- lines. -- -- The one small difference between the over/under casinos and the -- home/away ones is that the home/away lines are all 'Double's, but -- the over/under lines appear to be textual. -- data OddsGameLine = OddsGameLine { ogl_odds_games_id :: DefaultKey OddsGame, ogl_odds_casinos_id :: DefaultKey OddsCasino, ogl_over_under :: Maybe String, ogl_away_line :: Maybe Double, ogl_home_line :: Maybe Double } -- * OddsGame/OddsGameXml -- | Database representation of a game. We retain the rotation number -- of the home/away teams, since those are specific to the game and -- not the teams. -- data OddsGame = OddsGame { db_odds_id :: DefaultKey Odds, db_game_id :: Int, db_game_time :: UTCTime, -- ^ Contains both the date and time. db_game_away_team_rotation_number :: Int, db_game_home_team_rotation_number :: Int } -- | XML representation of an 'OddsGame'. -- data OddsGameXml = OddsGameXml { xml_game_id :: Int, xml_game_date :: UTCTime, -- ^ Contains only the date xml_game_time :: UTCTime, -- ^ Contains only the time xml_game_away_team :: OddsGameAwayTeamXml, xml_game_home_team :: OddsGameHomeTeamXml, xml_game_over_under :: OddsGameOverUnderXml } deriving (Eq, Show) -- | Pseudo-field that lets us get the 'OddsGameCasinoXml's out of -- xml_game_over_under. -- xml_game_over_under_casinos :: OddsGameXml -> [OddsGameCasinoXml] xml_game_over_under_casinos = xml_casinos . xml_game_over_under instance ToDb OddsGameXml where -- | The database representation of an 'OddsGameXml' is an -- 'OddsGame'. -- type Db OddsGameXml = OddsGame instance FromXmlFk OddsGameXml where -- | Each 'OddsGameXml' is contained in an 'Odds'. In other words -- the foreign key for 'OddsGame' points to an 'Odds'. -- type Parent OddsGameXml = Odds -- | To convert from the XML representation to the database one, we -- drop the home/away teams and the casino lines, but retain the -- home/away rotation numbers. -- from_xml_fk fk OddsGameXml{..} = OddsGame { db_odds_id = fk, db_game_id = xml_game_id, db_game_time = UTCTime (utctDay xml_game_date) -- Take the day part from one, (utctDayTime xml_game_time), -- the time from the other. db_game_away_team_rotation_number = (xml_away_rotation_number xml_game_away_team), db_game_home_team_rotation_number = (xml_home_rotation_number xml_game_home_team) } -- | This lets us insert the XML representation 'OddsGameXml' directly. -- instance XmlImportFk OddsGameXml -- * OddsGameWithNotes -- | This is our best guess at what occurs in the Odds_XML -- documents. It looks like each consecutive set of games can -- optionally have some notes appear before it. Each \"note\" comes -- as its own \...\ element. -- -- The notes are ignored completely in the database; we only bother -- with them to ensure that we're (un)pickling correctly. -- -- We can't group the notes with a \"set\" of 'OddsGame's, because -- that leads to ambiguity in parsing. Since we're going to ignore -- the notes anyway, we just stick them with an arbitrary -- game. C'est la vie. -- data OddsGameWithNotes = OddsGameWithNotes { notes :: [String], game :: OddsGameXml } deriving (Eq, Show) -- * Odds/Message -- | Database representation of a 'Message'. -- data Odds = Odds { db_xml_file_id :: Int, db_sport :: String, db_title :: String, db_line_time :: String, -- ^ We don't parse these as a 'UTCTime' -- because their timezones are ambiguous -- (and the date is less than useful when -- it might be off by an hour). db_time_stamp :: UTCTime } -- | The XML representation of 'Odds'. -- data Message = Message { xml_xml_file_id :: Int, xml_heading :: String, xml_category :: String, xml_sport :: String, xml_title :: String, xml_line_time :: String, xml_games_with_notes :: [OddsGameWithNotes], xml_time_stamp :: UTCTime } deriving (Eq, Show) -- | Pseudo-field that lets us get the 'OddsGame's out of -- 'xml_games_with_notes'. -- xml_games :: Message -> [OddsGameXml] xml_games m = map game (xml_games_with_notes m) instance ToDb Message where -- | The database representation of a 'Message' is 'Odds'. -- type Db Message = Odds instance FromXml Message where -- | To convert from the XML representation to the database one, we -- just drop a bunch of fields. -- from_xml Message{..} = Odds { db_xml_file_id = xml_xml_file_id, db_sport = xml_sport, db_title = xml_title, db_line_time = xml_line_time, db_time_stamp = xml_time_stamp } -- | This lets us insert the XML representation 'Message' directly. -- instance XmlImport Message -- -- Database code -- -- Groundhog database schema. This must come before the DbImport -- instance definition. Don't know why. mkPersist tsn_codegen_config [groundhog| - entity: Odds constructors: - name: Odds uniques: - name: unique_odds type: constraint # Prevent multiple imports of the same message. fields: [db_xml_file_id] - entity: OddsCasino dbName: odds_casinos constructors: - name: OddsCasino uniques: - name: unique_odds_casino type: constraint fields: [casino_client_id] - entity: OddsGameTeam dbName: odds_games_teams constructors: - name: OddsGameTeam uniques: - name: unique_odds_games_team type: constraint fields: [db_team_id] - entity: OddsGame dbName: odds_games constructors: - name: OddsGame fields: - name: db_odds_id reference: onDelete: cascade - entity: OddsGameLine dbName: odds_games_lines constructors: - name: OddsGameLine fields: - name: ogl_odds_games_id reference: onDelete: cascade - name: ogl_odds_casinos_id reference: onDelete: cascade - entity: OddsGame_OddsGameTeam dbName: odds_games__odds_games_teams constructors: - name: OddsGame_OddsGameTeam fields: - name: ogogt_odds_games_id reference: onDelete: cascade - name: ogogt_away_team_id reference: onDelete: cascade - name: ogogt_home_team_id reference: onDelete: cascade |] instance DbImport Message where dbmigrate _= run_dbmigrate $ do migrate (undefined :: Odds) migrate (undefined :: OddsCasino) migrate (undefined :: OddsGameTeam) migrate (undefined :: OddsGame) migrate (undefined :: OddsGame_OddsGameTeam) migrate (undefined :: OddsGameLine) dbimport m = do -- Insert the root "odds" element and acquire its primary key (id). odds_id <- insert_xml m forM_ (xml_games m) $ \g -> do -- Next, we insert the home and away teams. We do this before -- inserting the game itself because the game has two foreign keys -- pointing to odds_games_teams. -- Next to insert the home and away teams. away_team_id <- insert_xml_or_select (xml_game_away_team g) home_team_id <- insert_xml_or_select (xml_game_home_team g) -- Now insert the game, keyed to the "odds", game_id <- insert_xml_fk odds_id g -- Insert a record into odds_games__odds_games_teams mapping the -- home/away teams to this game. Use the full record syntax -- because the types would let us mix up the home/away teams. insert_ OddsGame_OddsGameTeam { ogogt_odds_games_id = game_id, ogogt_away_team_id = away_team_id, ogogt_home_team_id = home_team_id } -- Finaly, we insert the lines. The over/under entries for this -- game and the lines for the casinos all wind up in the same -- table, odds_games_lines. We can insert the over/under entries -- freely with empty away/home lines: forM_ (xml_game_over_under_casinos g) $ \c -> do -- Start by inderting the casino. ou_casino_id <- insert_xml_or_select c -- Now add the over/under entry with the casino's id. let ogl = OddsGameLine { ogl_odds_games_id = game_id, ogl_odds_casinos_id = ou_casino_id, ogl_over_under = (xml_casino_line c), ogl_away_line = Nothing, ogl_home_line = Nothing } insert_ ogl -- ...but then when we insert the home/away team lines, we -- prefer to update the existing entry rather than overwrite it -- or add a new record. forM_ (xml_away_casinos $ xml_game_away_team g) $ \c -> do -- insert, or more likely retrieve the existing, casino a_casino_id <- insert_xml_or_select c -- Get a Maybe Double instead of the Maybe String that's in there. let away_line = home_away_line c -- Unconditionally update that casino's away team line with ours. update [Ogl_Away_Line =. away_line] $ -- WHERE Ogl_Odds_Casinos_Id ==. a_casino_id -- Repeat all that for the home team. forM_ (xml_home_casinos $ xml_game_home_team g) $ \c ->do h_casino_id <- insert_xml_or_select c let home_line = home_away_line c update [Ogl_Home_Line =. home_line] $ -- WHERE Ogl_Odds_Casinos_Id ==. h_casino_id return game_id return ImportSucceeded -- -- Pickling -- -- | Pickler for an 'OddsGame' optionally preceded by some notes. -- pickle_game_with_notes :: PU OddsGameWithNotes pickle_game_with_notes = xpWrap (from_pair, to_pair) $ xpPair (xpList $ xpElem "Notes" xpText) pickle_game where from_pair = uncurry OddsGameWithNotes to_pair OddsGameWithNotes{..} = (notes, game) -- | Pickler for an 'OddsGameCasinoXml'. -- pickle_casino :: PU OddsGameCasinoXml pickle_casino = xpElem "Casino" $ xpWrap (from_tuple, to_tuple) $ xpTriple (xpAttr "ClientID" xpInt) (xpAttr "Name" xpText) (xpOption xpText) where from_tuple = uncurryN OddsGameCasinoXml -- Use record wildcards to avoid unused field warnings. to_tuple OddsGameCasinoXml{..} = (xml_casino_client_id, xml_casino_name, xml_casino_line) -- | Pickler for an 'OddsGameHomeTeamXml'. -- pickle_home_team :: PU OddsGameHomeTeamXml pickle_home_team = xpElem "HomeTeam" $ xpWrap (from_tuple, to_tuple) $ xp5Tuple (xpElem "HomeTeamID" xpText) (xpElem "HomeRotationNumber" xpInt) (xpElem "HomeAbbr" xpText) (xpElem "HomeTeamName" xpText) (xpList pickle_casino) where from_tuple = uncurryN OddsGameHomeTeamXml -- Use record wildcards to avoid unused field warnings. to_tuple OddsGameHomeTeamXml{..} = (xml_home_team_id, xml_home_rotation_number, xml_home_abbr, xml_home_team_name, xml_home_casinos) -- | Pickler for an 'OddsGameAwayTeamXml'. -- pickle_away_team :: PU OddsGameAwayTeamXml pickle_away_team = xpElem "AwayTeam" $ xpWrap (from_tuple, to_tuple) $ xp5Tuple (xpElem "AwayTeamID" xpText) (xpElem "AwayRotationNumber" xpInt) (xpElem "AwayAbbr" xpText) (xpElem "AwayTeamName" xpText) (xpList pickle_casino) where from_tuple = uncurryN OddsGameAwayTeamXml -- Use record wildcards to avoid unused field warnings. to_tuple OddsGameAwayTeamXml{..} = (xml_away_team_id, xml_away_rotation_number, xml_away_abbr, xml_away_team_name, xml_away_casinos) -- | Pickler for an 'OddsGameOverUnderXml'. -- pickle_over_under :: PU OddsGameOverUnderXml pickle_over_under = xpElem "Over_Under" $ xpWrap (to_newtype, from_newtype) $ xpList pickle_casino where from_newtype (OddsGameOverUnderXml cs) = cs to_newtype = OddsGameOverUnderXml -- | Pickler for an 'OddsGameXml'. -- pickle_game :: PU OddsGameXml pickle_game = xpElem "Game" $ xpWrap (from_tuple, to_tuple) $ xp6Tuple (xpElem "GameID" xpInt) (xpElem "Game_Date" xp_date) (xpElem "Game_Time" xp_time) pickle_away_team pickle_home_team pickle_over_under where from_tuple = uncurryN OddsGameXml -- Use record wildcards to avoid unused field warnings. to_tuple OddsGameXml{..} = (xml_game_id, xml_game_date, xml_game_time, xml_game_away_team, xml_game_home_team, xml_game_over_under) -- | Pickler for the top-level 'Message'. -- pickle_message :: PU Message pickle_message = xpElem "message" $ xpWrap (from_tuple, to_tuple) $ xp8Tuple (xpElem "XML_File_ID" xpInt) (xpElem "heading" xpText) (xpElem "category" xpText) (xpElem "sport" xpText) (xpElem "Title" xpText) (xpElem "Line_Time" xpText) (xpList pickle_game_with_notes) (xpElem "time_stamp" xp_time_stamp) where from_tuple = uncurryN Message to_tuple m = (xml_xml_file_id m, xml_heading m, xml_category m, xml_sport m, xml_title m, xml_line_time m, xml_games_with_notes m, xml_time_stamp m) -- -- Tasty Tests -- -- | A list of all tests for this module. -- odds_tests :: TestTree odds_tests = testGroup "Odds tests" [ test_on_delete_cascade, test_pickle_of_unpickle_is_identity, test_unpickle_succeeds ] -- | If we unpickle something and then pickle it, we should wind up -- with the same thing we started with. WARNING: success of this -- test does not mean that unpickling succeeded. -- test_pickle_of_unpickle_is_identity :: TestTree test_pickle_of_unpickle_is_identity = testGroup "pickle-unpickle tests" [ check "pickle composed with unpickle is the identity" "test/xml/Odds_XML.xml", check "pickle composed with unpickle is the identity (non-int team_id)" "test/xml/Odds_XML-noninteger-team-id.xml", check "pickle composed with unpickle is the identity (positive(+) line)" "test/xml/Odds_XML-positive-line.xml", check "pickle composed with unpickle is the identity (large file)" "test/xml/Odds_XML-largefile.xml" ] where check desc path = testCase desc $ do (expected, actual) <- pickle_unpickle pickle_message path actual @?= expected -- | Make sure we can actually unpickle these things. -- test_unpickle_succeeds :: TestTree test_unpickle_succeeds = testGroup "unpickle tests" [ check "unpickling succeeds" "test/xml/Odds_XML.xml", check "unpickling succeeds (non-int team_id)" "test/xml/Odds_XML-noninteger-team-id.xml", check "unpickling succeeds (positive(+) line)" "test/xml/Odds_XML-positive-line.xml", check "unpickling succeeds (large file)" "test/xml/Odds_XML-largefile.xml" ] where check desc path = testCase desc $ do actual <- unpickleable path pickle_message let expected = True actual @?= expected -- | Make sure everything gets deleted when we delete the top-level -- record. -- test_on_delete_cascade :: TestTree test_on_delete_cascade = testGroup "cascading delete tests" [ check "deleting odds deletes its children" "test/xml/Odds_XML.xml" 13 -- 5 casinos, 8 teams , check "deleting odds deletes its children (non-int team_id)" "test/xml/Odds_XML-noninteger-team-id.xml" 51 -- 5 casinos, 46 teams , check "deleting odds deleted its children (positive(+) line)" "test/xml/Odds_XML-positive-line.xml" 17 -- 5 casinos, 12 teams , check "deleting odds deleted its children (large file)" "test/xml/Odds_XML-largefile.xml" 189 -- 5 casinos, 184 teams ] where check desc path expected = testCase desc $ do odds <- unsafe_unpickle path pickle_message let a = undefined :: Odds let b = undefined :: OddsCasino let c = undefined :: OddsGameTeam let d = undefined :: OddsGame let e = undefined :: OddsGame_OddsGameTeam let f = undefined :: OddsGameLine actual <- withSqliteConn ":memory:" $ runDbConn $ do runMigration silentMigrationLogger $ do migrate a migrate b migrate c migrate d migrate e migrate f _ <- dbimport odds -- No idea how 'delete' works, so do this instead. executeRaw False "DELETE FROM odds;" [] count_a <- countAll a count_b <- countAll b count_c <- countAll c count_d <- countAll d count_e <- countAll e count_f <- countAll f return $ sum [count_a, count_b, count_c, count_d, count_e, count_f ] actual @?= expected