]> gitweb.michael.orlitzky.com - dead/htsn-import.git/blob - src/TSN/XML/JFile.hs
Use Generics.to_tuple in TSN.XML.JFile.
[dead/htsn-import.git] / src / TSN / XML / JFile.hs
1 {-# LANGUAGE DeriveGeneric #-}
2 {-# LANGUAGE FlexibleInstances #-}
3 {-# LANGUAGE GADTs #-}
4 {-# LANGUAGE QuasiQuotes #-}
5 {-# LANGUAGE RecordWildCards #-}
6 {-# LANGUAGE TemplateHaskell #-}
7 {-# LANGUAGE TypeFamilies #-}
8
9 -- | Parse TSN XML for the DTD \"jfilexml.dtd\". There's a top-level
10 -- \<message\>, containing a \<gamelist\>, containing
11 -- \<game\>s. Those games contain a bunch of other stuff. The
12 -- \<gamelist\> is pretty irrelevant; we ignore it and pretend that
13 -- a message contains a bunch of games.
14 --
15 module TSN.XML.JFile (
16 dtd,
17 pickle_message,
18 -- * Tests
19 jfile_tests,
20 -- * WARNING: these are private but exported to silence warnings
21 JFileConstructor(..),
22 JFileGameConstructor(..) )
23 where
24
25 -- System imports
26 import Control.Monad ( forM_ )
27 import Data.List ( intercalate )
28 import Data.String.Utils ( split )
29 import Data.Time ( UTCTime(..) )
30 import Data.Tuple.Curry ( uncurryN )
31 import Database.Groundhog (
32 countAll,
33 deleteAll,
34 migrate,
35 runMigration,
36 silentMigrationLogger )
37 import Database.Groundhog.Core ( DefaultKey )
38 import Database.Groundhog.Generic ( runDbConn )
39 import Database.Groundhog.Sqlite ( withSqliteConn )
40 import Database.Groundhog.TH (
41 groundhog,
42 mkPersist )
43 import qualified GHC.Generics as GHC ( Generic )
44 import Test.Tasty ( TestTree, testGroup )
45 import Test.Tasty.HUnit ( (@?=), testCase )
46 import Text.XML.HXT.Core (
47 PU,
48 xpTriple,
49 xp6Tuple,
50 xp14Tuple,
51 xp19Tuple,
52 xpAttr,
53 xpElem,
54 xpInt,
55 xpList,
56 xpOption,
57 xpPair,
58 xpPrim,
59 xpText,
60 xpText0,
61 xpWrap )
62
63
64 -- Local imports
65 import Generics ( Generic(..), to_tuple )
66 import TSN.Codegen ( tsn_codegen_config )
67 import TSN.Database ( insert_or_select )
68 import TSN.DbImport ( DbImport(..), ImportResult(..), run_dbmigrate )
69 import TSN.Picklers (
70 xp_date,
71 xp_date_padded,
72 xp_datetime,
73 xp_tba_time,
74 xp_time_dots,
75 xp_time_stamp )
76 import TSN.Team (
77 FromXmlFkTeams(..),
78 HTeam(..),
79 Team(..),
80 VTeam(..) )
81 import TSN.XmlImport (
82 XmlImport(..),
83 XmlImportFkTeams(..) )
84 import Xml (
85 Child(..),
86 FromXml(..),
87 ToDb(..),
88 pickle_unpickle,
89 unpickleable,
90 unsafe_unpickle )
91
92
93
94 -- | The DTD to which this module corresponds. Used to invoke dbimport.
95 --
96 dtd :: String
97 dtd = "jfilexml.dtd"
98
99 --
100 -- DB/XML data types
101 --
102
103 -- * JFile/Message
104
105 -- | Database representation of a 'Message'.
106 --
107 data JFile =
108 JFile {
109 db_xml_file_id :: Int,
110 db_heading :: String,
111 db_category :: String,
112 db_sport :: String,
113 db_time_stamp :: UTCTime }
114
115
116
117 -- | XML Representation of an 'JFile'.
118 --
119 data Message =
120 Message {
121 xml_xml_file_id :: Int,
122 xml_heading :: String,
123 xml_category :: String,
124 xml_sport :: String,
125 xml_gamelist :: JFileGameListXml,
126 xml_time_stamp :: UTCTime }
127 deriving (Eq, GHC.Generic, Show)
128
129
130 -- | For 'Generics.to_tuple'.
131 --
132 instance Generic Message
133
134
135 instance ToDb Message where
136 -- | The database analogue of a 'Message' is a 'JFile'.
137 --
138 type Db Message = JFile
139
140
141 -- | The 'FromXml' instance for 'Message' is required for the
142 -- 'XmlImport' instance.
143 --
144 instance FromXml Message where
145 -- | To convert a 'Message' to an 'JFile', we just drop
146 -- the 'xml_gamelist'.
147 --
148 from_xml Message{..} =
149 JFile {
150 db_xml_file_id = xml_xml_file_id,
151 db_heading = xml_heading,
152 db_category = xml_category,
153 db_sport = xml_sport,
154 db_time_stamp = xml_time_stamp }
155
156
157 -- | This allows us to insert the XML representation 'Message'
158 -- directly.
159 --
160 instance XmlImport Message
161
162
163
164 -- * JFileGame/JFileGameXml
165
166 -- | This is an embedded type within each JFileGame. It has its own
167 -- element, \<Odds_Info\>, but there's only one of them per game. So
168 -- essentially all of these fields belong to a 'JFileGame'. Aaaannnd
169 -- most of them are redundant. We'll (un)pickle them for good
170 -- measure, but in the conversion to the database type, we can drop
171 -- all of the redundant information.
172 --
173 -- All of these are optional because TSN does actually leave the
174 -- whole thing empty from time to time.
175 --
176 -- We stick \"info\" on the home/away team ids to avoid a name clash
177 -- with the game itself.
178 --
179 data JFileGameOddsInfo =
180 JFileGameOddsInfo {
181 db_list_date :: Maybe UTCTime,
182 db_info_home_team_id :: Maybe String, -- redundant (Team)
183 db_info_away_team_id :: Maybe String, -- redundant (Team)
184 db_home_abbr :: Maybe String, -- redundant (Team)
185 db_away_abbr :: Maybe String, -- redundant (Team)
186 db_home_team_name :: Maybe String, -- redundant (Team)
187 db_away_team_name :: Maybe String, -- redundant (Team)
188 db_home_starter :: Maybe String,
189 db_away_starter :: Maybe String,
190 db_game_date :: Maybe UTCTime, -- redundant (JFileGame)
191 db_home_game_key :: Maybe Int,
192 db_away_game_key :: Maybe Int,
193 db_current_timestamp :: Maybe UTCTime,
194 db_live :: Maybe Bool,
195 db_notes :: String }
196 deriving (Eq, Show)
197
198
199 -- | Another embedded type within 'JFileGame'. These look like,
200 -- \<status numeral=\"4\"\>FINAL\</status\> within the XML, but
201 -- they're in one-to-one correspondence with the games.
202 --
203 data JFileGameStatus =
204 JFileGameStatus {
205 db_status_numeral :: Int,
206 db_status :: Maybe String }
207 deriving (Eq, Show)
208
209
210
211 -- | Database representation of a \<game\> contained within a
212 -- \<message\>, and, implicitly, a \<gamelist\>.
213 --
214 -- We've left out the game date, opting instead to combine the
215 -- date/time into the 'db_game_time' field.
216 --
217 data JFileGame =
218 JFileGame {
219 db_jfile_id :: DefaultKey JFile,
220 db_away_team_id :: DefaultKey Team,
221 db_home_team_id :: DefaultKey Team,
222 db_game_id :: Int,
223 db_schedule_id :: Int,
224 db_odds_info :: JFileGameOddsInfo,
225 db_season_type :: Maybe String,
226 db_game_time :: Maybe UTCTime,
227 db_vleague :: Maybe String,
228 db_hleague :: Maybe String,
229 db_vscore :: Int,
230 db_hscore :: Int,
231 db_time_remaining :: Maybe String,
232 db_game_status :: JFileGameStatus }
233
234
235 -- | XML representation of a \<game\> contained within a \<message\>,
236 -- and a \<gamelist\>. The Away/Home teams seem to coincide with
237 -- those of 'OddsGame', so we're reusing the DB type via the common
238 -- 'TSN.Team' structure. But the XML types are different, because
239 -- they have different picklers!
240 --
241 data JFileGameXml =
242 JFileGameXml {
243 xml_game_id :: Int,
244 xml_schedule_id :: Int,
245 xml_odds_info :: JFileGameOddsInfo,
246 xml_season_type :: Maybe String,
247 xml_game_date :: UTCTime,
248 xml_game_time :: Maybe UTCTime,
249 xml_vteam :: VTeam,
250 xml_vleague :: Maybe String,
251 xml_hteam :: HTeam,
252 xml_hleague :: Maybe String,
253 xml_vscore :: Int,
254 xml_hscore :: Int,
255 xml_time_remaining :: Maybe String,
256 xml_game_status :: JFileGameStatus }
257 deriving (Eq, GHC.Generic, Show)
258
259
260 -- | For 'Generics.to_tuple'.
261 --
262 instance Generic JFileGameXml
263
264
265 -- * JFileGameListXml
266
267 -- | The XML representation of \<message\> -> \<gamelist\>. This
268 -- element serves only to contain \<game\>s, so we don't store the
269 -- intermediate table in the database.
270 --
271 newtype JFileGameListXml =
272 JFileGameListXml {
273 xml_games ::
274 [JFileGameXml] }
275 deriving (Eq, Show)
276
277
278 instance ToDb JFileGameXml where
279 -- | The database analogue of an 'JFileGameXml' is
280 -- an 'JFileGame'.
281 --
282 type Db JFileGameXml = JFileGame
283
284
285 instance Child JFileGameXml where
286 -- | Each 'JFileGameXml' is contained in (i.e. has a foreign key to)
287 -- a 'JFile'.
288 --
289 type Parent JFileGameXml = JFile
290
291
292 instance FromXmlFkTeams JFileGameXml where
293 -- | To convert an 'JFileGameXml' to an 'JFileGame', we add the
294 -- foreign keys for JFile and the home/away teams. We also mash
295 -- the date/time together into one field.
296 --
297 from_xml_fk_teams fk fk_away fk_home JFileGameXml{..} =
298 JFileGame {
299 db_jfile_id = fk,
300 db_away_team_id = fk_away,
301 db_home_team_id = fk_home,
302 db_game_id = xml_game_id,
303 db_schedule_id = xml_schedule_id,
304 db_odds_info = xml_odds_info,
305 db_season_type = xml_season_type,
306 db_game_time = make_game_time xml_game_date xml_game_time,
307 db_vleague = xml_vleague,
308 db_hleague = xml_hleague,
309 db_vscore = xml_vscore,
310 db_hscore = xml_hscore,
311 db_time_remaining = xml_time_remaining,
312 db_game_status = xml_game_status }
313 where
314 -- | Construct the database game time from the XML \<Game_Date\>
315 -- and \<Game_Time\> elements. The \<Game_Time\> elements
316 -- sometimes have a value of \"TBA\"; in that case, we don't
317 -- want to pretend that we know the time by setting it to
318 -- e.g. midnight, so instead we make the entire date/time
319 -- Nothing.
320 make_game_time :: UTCTime -> Maybe UTCTime -> Maybe UTCTime
321 make_game_time _ Nothing = Nothing
322 make_game_time d (Just t) = Just $ UTCTime (utctDay d) (utctDayTime t)
323
324
325 -- | This allows us to insert the XML representation
326 -- 'JFileGameXml' directly.
327 --
328 instance XmlImportFkTeams JFileGameXml
329
330
331 ---
332 --- Database stuff.
333 ---
334
335 instance DbImport Message where
336 dbmigrate _ =
337 run_dbmigrate $ do
338 migrate (undefined :: Team)
339 migrate (undefined :: JFile)
340 migrate (undefined :: JFileGame)
341
342 dbimport m = do
343 -- Insert the top-level message
344 msg_id <- insert_xml m
345
346 -- Now loop through the message's games
347 forM_ (xml_games $ xml_gamelist m) $ \game -> do
348 -- First we insert the home and away teams.
349 away_team_id <- insert_or_select (vteam $ xml_vteam game)
350 home_team_id <- insert_or_select (hteam $ xml_hteam game)
351
352 -- Now insert the game keyed to the "jfile" and its teams.
353 insert_xml_fk_teams_ msg_id away_team_id home_team_id game
354
355
356 return ImportSucceeded
357
358
359 mkPersist tsn_codegen_config [groundhog|
360 - entity: JFile
361 dbName: jfile
362 constructors:
363 - name: JFile
364 uniques:
365 - name: unique_jfile
366 type: constraint
367 # Prevent multiple imports of the same message.
368 fields: [db_xml_file_id]
369
370 - embedded: JFileGameStatus
371 fields:
372 - name: db_status_numeral
373 dbName: status_numeral
374 - name: db_status
375 dbName: status
376
377 # Many of the JFileGameOddsInfo fields are redundant and have
378 # been left out.
379 - embedded: JFileGameOddsInfo
380 fields:
381 - name: db_list_date
382 dbName: list_date
383 - name: db_home_starter
384 dbName: home_starter
385 - name: db_home_game_key
386 dbName: home_game_key
387 - name: db_away_game_key
388 dbName: away_game_key
389 - name: db_current_timestamp
390 dbName: current_timestamp
391 - name: db_live
392 dbName: live
393 - name: db_notes
394 dbName: notes
395
396 - entity: JFileGame
397 dbName: jfile_games
398 constructors:
399 - name: JFileGame
400 fields:
401 - name: db_jfile_id
402 reference:
403 onDelete: cascade
404 - name: db_away_team_id
405 reference:
406 onDelete: cascade
407 - name: db_home_team_id
408 reference:
409 onDelete: cascade
410 - name: db_odds_info
411 embeddedType:
412 - {name: list_date, dbName: list_date}
413 - {name: home_starter, dbName: home_starter}
414 - {name: away_starter, dbName: away_starter}
415 - {name: home_game_key, dbName: home_game_key}
416 - {name: away_game_key, dbName: away_game_key}
417 - {name: current_timestamp, dbName: current_timestamp}
418 - {name: live, dbName: live}
419 - {name: notes, dbName: notes}
420 - name: db_game_status
421 embeddedType:
422 - {name: status_numeral, dbName: status_numeral}
423 - {name: status, dbName: status}
424
425 |]
426
427
428
429 ---
430 --- Pickling
431 ---
432
433 -- | Pickler for the top-level 'Message'.
434 --
435 pickle_message :: PU Message
436 pickle_message =
437 xpElem "message" $
438 xpWrap (from_tuple, to_tuple) $
439 xp6Tuple (xpElem "XML_File_ID" xpInt)
440 (xpElem "heading" xpText)
441 (xpElem "category" xpText)
442 (xpElem "sport" xpText)
443 pickle_gamelist
444 (xpElem "time_stamp" xp_time_stamp)
445 where
446 from_tuple = uncurryN Message
447
448
449 pickle_gamelist :: PU JFileGameListXml
450 pickle_gamelist =
451 xpElem "gamelist" $
452 xpWrap (to_result, from_result) $ xpList pickle_game
453 where
454 to_result = JFileGameListXml
455 from_result = xml_games
456
457
458
459
460 pickle_game :: PU JFileGameXml
461 pickle_game =
462 xpElem "game" $
463 xpWrap (from_tuple, to_tuple) $
464 xp14Tuple (xpElem "game_id" xpInt)
465 (xpElem "schedule_id" xpInt)
466 pickle_odds_info
467 (xpElem "seasontype" (xpOption xpText))
468 (xpElem "Game_Date" xp_date_padded)
469 (xpElem "Game_Time" xp_tba_time)
470 pickle_away_team
471 (xpOption $ xpElem "vleague" xpText)
472 pickle_home_team
473 (xpOption $ xpElem "hleague" xpText)
474 (xpElem "vscore" xpInt)
475 (xpElem "hscore" xpInt)
476 (xpOption $ xpElem "time_r" xpText)
477 pickle_status
478 where
479 from_tuple = uncurryN JFileGameXml
480
481
482 pickle_odds_info :: PU JFileGameOddsInfo
483 pickle_odds_info =
484 xpElem "Odds_Info" $
485 xpWrap (from_tuple, to_tuple') $
486 xp19Tuple (xpElem "ListDate" (xpOption xp_date))
487 (xpElem "HomeTeamID" (xpOption xpText))
488 (xpElem "AwayTeamID" (xpOption xpText))
489 (xpElem "HomeAbbr" (xpOption xpText))
490 (xpElem "AwayAbbr" (xpOption xpText))
491 (xpElem "HomeTeamName" (xpOption xpText))
492 (xpElem "AwayTeamName" (xpOption xpText))
493 (xpElem "HStarter" (xpOption xpText))
494 (xpElem "AStarter" (xpOption xpText))
495 (xpElem "GameDate" (xpOption xp_datetime))
496 (xpElem "HGameKey" (xpOption xpInt))
497 (xpElem "AGameKey" (xpOption xpInt))
498 (xpElem "CurrentTimeStamp" (xpOption xp_time_dots))
499 (xpElem "Live" (xpOption xpPrim))
500 (xpElem "Notes1" xpText0)
501 (xpElem "Notes2" xpText0)
502 (xpElem "Notes3" xpText0)
503 (xpElem "Notes4" xpText0)
504 (xpElem "Notes5" xpText0)
505 where
506 from_tuple (x1,x2,x3,x4,x5,x6,x7,x8,x9,x10,x11,x12,x13,x14,n1,n2,n3,n4,n5) =
507 JFileGameOddsInfo x1 x2 x3 x4 x5 x6 x7 x8 x9 x10 x11 x12 x13 x14 notes
508 where
509 notes = intercalate "\n" [n1,n2,n3,n4,n5]
510
511 to_tuple' o = (db_list_date o,
512 db_info_home_team_id o,
513 db_info_away_team_id o,
514 db_home_abbr o,
515 db_away_abbr o,
516 db_home_team_name o,
517 db_away_team_name o,
518 db_home_starter o,
519 db_away_starter o,
520 db_game_date o,
521 db_home_game_key o,
522 db_away_game_key o,
523 db_current_timestamp o,
524 db_live o,
525 n1,n2,n3,n4,n5)
526 where
527 note_lines = split "\n" (db_notes o)
528 n1 = case note_lines of
529 (notes1:_) -> notes1
530 _ -> ""
531 n2 = case note_lines of
532 (_:notes2:_) -> notes2
533 _ -> ""
534 n3 = case note_lines of
535 (_:_:notes3:_) -> notes3
536 _ -> ""
537 n4 = case note_lines of
538 (_:_:_:notes4:_) -> notes4
539 _ -> ""
540 n5 = case note_lines of
541 (_:_:_:_:notes5:_) -> notes5
542 _ -> ""
543
544 -- | (Un)pickle a home team to/from the dual XML/DB representation
545 -- 'Team'.
546 --
547 pickle_home_team :: PU HTeam
548 pickle_home_team =
549 xpElem "hteam" $
550 xpWrap (from_tuple, to_tuple') $
551 xpTriple (xpAttr "teamid" xpText) -- Yeah, they're text.
552 (xpAttr "abbr" (xpOption xpText)) -- Some are blank
553 (xpOption xpText) -- Yup, some are nameless
554 where
555 from_tuple = HTeam . (uncurryN Team)
556 to_tuple' (HTeam t) = to_tuple t
557
558
559 -- | (Un)pickle an away team to/from the dual XML/DB representation
560 -- 'Team'.
561 --
562 pickle_away_team :: PU VTeam
563 pickle_away_team =
564 xpElem "vteam" $
565 xpWrap (from_tuple, to_tuple') $
566 xpTriple (xpAttr "teamid" xpText) -- Yeah, they're text.
567 (xpAttr "abbr" (xpOption xpText)) -- Some are blank
568 (xpOption xpText) -- Yup, some are nameless
569 where
570 from_tuple = VTeam . (uncurryN Team)
571 to_tuple' (VTeam t) = to_tuple t
572
573
574 pickle_status :: PU JFileGameStatus
575 pickle_status =
576 xpElem "status" $
577 xpWrap (from_tuple, to_tuple') $
578 xpPair (xpAttr "numeral" xpInt)
579 (xpOption xpText)
580 where
581 from_tuple = uncurry JFileGameStatus
582
583 -- Avoid unused field warnings.
584 to_tuple' JFileGameStatus{..} = (db_status_numeral, db_status)
585
586
587 --
588 -- Tasty Tests
589 --
590
591 -- | A list of all tests for this module.
592 --
593 jfile_tests :: TestTree
594 jfile_tests =
595 testGroup
596 "JFile tests"
597 [ test_on_delete_cascade,
598 test_pickle_of_unpickle_is_identity,
599 test_unpickle_succeeds ]
600
601
602 -- | If we unpickle something and then pickle it, we should wind up
603 -- with the same thing we started with. WARNING: success of this
604 -- test does not mean that unpickling succeeded.
605 --
606 test_pickle_of_unpickle_is_identity :: TestTree
607 test_pickle_of_unpickle_is_identity = testGroup "pickle-unpickle tests"
608 [ check "pickle composed with unpickle is the identity"
609 "test/xml/jfilexml.xml",
610 check "pickle composed with unpickle is the identity (missing fields)"
611 "test/xml/jfilexml-missing-fields.xml",
612
613 check "pickle composed with unpickle is the identity (TBA game time)"
614 "test/xml/jfilexml-tba-game-time.xml"]
615 where
616 check desc path = testCase desc $ do
617 (expected, actual) <- pickle_unpickle pickle_message path
618 actual @?= expected
619
620
621
622 -- | Make sure we can actually unpickle these things.
623 --
624 test_unpickle_succeeds :: TestTree
625 test_unpickle_succeeds = testGroup "unpickle tests"
626 [ check "unpickling succeeds" "test/xml/jfilexml.xml",
627
628 check "unpickling succeeds (missing fields)"
629 "test/xml/jfilexml-missing-fields.xml",
630
631 check "unpickling succeeds (TBA game time)"
632 "test/xml/jfilexml-tba-game-time.xml" ]
633 where
634 check desc path = testCase desc $ do
635 actual <- unpickleable path pickle_message
636
637 let expected = True
638 actual @?= expected
639
640
641
642 -- | Make sure everything gets deleted when we delete the top-level
643 -- record.
644 --
645 test_on_delete_cascade :: TestTree
646 test_on_delete_cascade = testGroup "cascading delete tests"
647 [ check "deleting auto_racing_results deletes its children"
648 "test/xml/jfilexml.xml"
649 20, -- teams
650
651 check "deleting auto_racing_results deletes its children (missing fields)"
652 "test/xml/jfilexml-missing-fields.xml"
653 44,
654
655 check "deleting auto_racing_results deletes its children (TBA game time)"
656 "test/xml/jfilexml-tba-game-time.xml"
657 8 ]
658 where
659 check desc path expected = testCase desc $ do
660 results <- unsafe_unpickle path pickle_message
661 let a = undefined :: Team
662 let b = undefined :: JFile
663 let c = undefined :: JFileGame
664
665 actual <- withSqliteConn ":memory:" $ runDbConn $ do
666 runMigration silentMigrationLogger $ do
667 migrate a
668 migrate b
669 migrate c
670 _ <- dbimport results
671 deleteAll b
672 count_a <- countAll a
673 count_b <- countAll b
674 count_c <- countAll c
675 return $ sum [count_a, count_b, count_c]
676 actual @?= expected