1 {-# LANGUAGE DeriveGeneric #-}
2 {-# LANGUAGE DeriveDataTypeable #-}
3 {-# LANGUAGE FlexibleInstances #-}
5 {-# LANGUAGE QuasiQuotes #-}
6 {-# LANGUAGE RecordWildCards #-}
7 {-# LANGUAGE TemplateHaskell #-}
8 {-# LANGUAGE TypeFamilies #-}
10 -- | Parse TSN XML for the DTD \"newsxml.dtd\". Each document contains
11 -- a root element \<message\> that contains an entire news item.
19 -- * WARNING: these are private but exported to silence warnings
20 News_LocationConstructor(..),
21 News_TeamConstructor(..),
26 import Data.Data ( Data, constrFields, dataTypeConstrs, dataTypeOf )
27 import Data.Time.Clock ( UTCTime )
28 import Data.List.Utils ( join, split )
29 import Data.Tuple.Curry ( uncurryN )
30 import Data.Typeable ( Typeable )
31 import Database.Groundhog (
37 silentMigrationLogger )
38 import Database.Groundhog.Core ( DefaultKey )
39 import Database.Groundhog.Generic ( runDbConn )
40 import Database.Groundhog.Sqlite ( withSqliteConn )
41 import Database.Groundhog.TH (
44 import qualified GHC.Generics as GHC ( Generic )
45 import Test.Tasty ( TestTree, testGroup )
46 import Test.Tasty.HUnit ( (@?=), testCase )
47 import Text.XML.HXT.Core (
70 import Generics ( Generic(..), to_tuple )
73 tsn_db_field_namer ) -- Used in a test
74 import TSN.Database ( insert_or_select )
75 import TSN.DbImport ( DbImport(..), ImportResult(..), run_dbmigrate )
76 import TSN.Location ( Location(..), pickle_location )
77 import TSN.Picklers ( xp_time_stamp )
78 import TSN.Team ( Team(..) )
79 import TSN.XmlImport ( XmlImport(..) )
89 -- | The DTD to which this module corresponds. Used to invoke dbimport.
101 -- | The msg_id child of \<message\> contains an event_id attribute; we
102 -- embed it into the 'News' type. We (pointlessly) use the \"db_\"
103 -- prefix here so that the two names don't collide on \"id\" when
104 -- Groundhog is creating its fields using our field namer.
109 db_event_id :: Maybe Int }
110 deriving (Data, Eq, Show, Typeable)
113 -- | The XML representation of a news item (\<message\>).
117 xml_xml_file_id :: Int,
118 xml_heading :: String,
120 xml_category :: String,
122 xml_url :: Maybe String,
123 xml_teams :: [NewsTeamXml],
124 xml_locations :: [Location],
125 xml_sms :: Maybe String,
126 xml_editor :: Maybe String,
127 xml_text :: Maybe String, -- Text and continue seem to show up in pairs,
128 xml_continue :: Maybe String, -- either both present or both missing.
129 xml_time_stamp :: UTCTime }
130 deriving (Eq, GHC.Generic, Show)
133 -- | For 'Generics.to_tuple'.
135 instance Generic Message
138 -- | The database representation of a news item. We drop several
139 -- uninteresting fields from 'Message', and omit the list fields which
140 -- will be represented as join tables.
144 db_xml_file_id :: Int,
147 db_url :: Maybe String,
148 db_sms :: Maybe String,
149 db_editor :: Maybe String,
150 db_text :: Maybe String,
151 db_continue :: Maybe String,
152 db_time_stamp :: UTCTime }
153 deriving (Data, Eq, Show, Typeable)
157 instance ToDb Message where
158 -- | The database representation of 'Message' is 'News'.
159 type Db Message = News
161 -- | Convert the XML representation 'Message' to the database
162 -- representation 'News'.
164 instance FromXml Message where
165 -- | We use a record wildcard so GHC doesn't complain that we never
166 -- used the field names.
168 from_xml Message{..} = News { db_xml_file_id = xml_xml_file_id,
170 db_sport = xml_sport,
173 db_editor = xml_editor,
175 db_continue = xml_continue,
176 db_time_stamp = xml_time_stamp }
178 -- | This lets us insert the XML representation 'Message' directly.
180 instance XmlImport Message
185 -- | The XML type for teams as they show up in the news. We can't
186 -- reuse the representation from "TSN.Team" because our name doesn't
187 -- appear optional and we have no abbreviation.
190 NewsTeamXml { xml_team_id :: String,
191 xml_team_name :: String }
192 deriving (Eq, GHC.Generic, Show)
195 -- | For 'Generics.to_tuple'.
197 instance Generic NewsTeamXml
200 instance ToDb NewsTeamXml where
201 -- | The database representation of 'NewsTeamXml' is 'Team'.
202 type Db NewsTeamXml = Team
204 -- | Convert the XML representation 'NewsTeamXml' to the database
205 -- representation 'Team'.
207 instance FromXml NewsTeamXml where
208 from_xml NewsTeamXml{..} =
209 Team { team_id = xml_team_id,
210 abbreviation = Nothing,
211 name = Just xml_team_name }
213 -- | Allow us to import 'NewsTeamXml' directly.
215 instance XmlImport NewsTeamXml
220 -- | Mapping between News records and Team records in the database. We
221 -- don't name the fields because we don't use the names explicitly;
222 -- that means we have to give them nice database names via
225 data News_Team = News_Team (DefaultKey News) (DefaultKey Team)
230 -- | Mapping between 'News' records and 'Location' records in the
231 -- database. We don't name the fields because we don't use the names
232 -- explicitly; that means we have to give them nice database names
235 data News_Location = News_Location
237 (DefaultKey Location)
242 -- | Some newsxml documents contain two \<SMS\> elements in a row,
243 -- violating the DTD. The second one has always been empty, but it's
244 -- irrelevant: we can't parse these, and would like to detect them
245 -- in order to report the fact that the busted document is
248 -- This function detects whether two \<SMS\> elements appear in a
251 has_only_single_sms :: XmlTree -> Bool
252 has_only_single_sms xmltree =
257 parse :: XmlTree -> [XmlTree]
258 parse = runLA $ hasName "/"
262 >>> filterAxis (hasName "SMS")
263 >>> followingSiblingAxis
267 elements = parse xmltree
274 -- | Define 'dbmigrate' and 'dbimport' for 'Message's. The import is
275 -- slightly non-generic because of our 'News_Team' and
276 -- 'News_Location' join tables.
278 instance DbImport Message where
281 migrate (undefined :: Location)
282 migrate (undefined :: News)
283 migrate (undefined :: Team)
284 migrate (undefined :: News_Team)
285 migrate (undefined :: News_Location)
287 dbimport message = do
288 -- Insert the message and acquire its primary key (unique ID)
289 news_id <- insert_xml message
291 -- Now insert the teams. We use insert_xml_or_select because we
292 -- know that most teams will already exist, and we want to get
293 -- back the id for the existing team when there's a collision.
294 team_ids <- mapM insert_xml_or_select (xml_teams message)
296 -- Now that the teams have been inserted, create
297 -- news__team records mapping beween the two.
298 let news_teams = map (News_Team news_id) team_ids
299 mapM_ insert_ news_teams
301 -- Do all of that over again for the Locations.
302 loc_ids <- mapM insert_or_select (xml_locations message)
303 let news_news_locations = map (News_Location news_id) loc_ids
304 mapM_ insert_ news_news_locations
306 return ImportSucceeded
310 -- These types have fields with e.g. db_ and xml_ prefixes, so we
311 -- use our own codegen to peel those off before naming the columns.
312 mkPersist tsn_codegen_config [groundhog|
319 # Prevent multiple imports of the same message.
320 fields: [db_xml_file_id]
324 - {name: msg_id, dbName: msg_id}
325 - {name: event_id, dbName: event_id}
339 - name: news_Team0 # Default created by mkNormalFieldName
343 - name: news_Team1 # Default created by mkNormalFieldName
348 - entity: News_Location
349 dbName: news__locations
351 - name: News_Location
353 - name: news_Location0 # Default created by mkNormalFieldName
357 - name: news_Location1 # Default created by mkNormalFieldName
368 -- | Convert a 'NewsTeamXml' to/from XML.
370 pickle_news_team :: PU NewsTeamXml
373 xpWrap (from_tuple, to_tuple) $
374 xpPair (xpAttr "id" xpText)
377 from_tuple = uncurry NewsTeamXml
380 -- | Convert a 'MsgId' to/from XML.
382 pickle_msg_id :: PU MsgId
385 xpWrap (from_tuple, to_tuple') $
386 xpPair xpInt (xpAttr "EventId" (xpOption xpInt))
388 from_tuple = uncurryN MsgId
390 -- Avoid unused field warnings.
391 to_tuple' m = (db_msg_id m, db_event_id m)
395 -- | Convert a 'Message' to/from XML.
397 pickle_message :: PU Message
400 xpWrap (from_tuple, to_tuple) $
401 xp13Tuple (xpElem "XML_File_ID" xpInt)
402 (xpElem "heading" xpText)
404 (xpElem "category" xpText)
405 (xpElem "sport" xpText)
406 (xpElem "url" $ xpOption xpText)
407 (xpList pickle_news_team)
408 (xpList pickle_location)
409 (xpElem "SMS" $ xpOption xpText)
410 (xpOption (xpElem "Editor" xpText))
411 (xpOption (xpElem "text" xpText))
413 (xpElem "time_stamp" xp_time_stamp)
415 from_tuple = uncurryN Message
418 -- | We combine all of the \<continue\> elements into one 'String'
419 -- while unpickling and do the reverse while pickling.
421 pickle_continue :: PU (Maybe String)
424 xpWrap (to_string, from_string) $
426 xpList (xpElem "P" xpText)
428 from_string :: String -> [String]
429 from_string = split "\n"
431 to_string :: [String] -> String
432 to_string = join "\n"
439 -- | A list of all tests for this module.
441 news_tests :: TestTree
445 [ test_news_fields_have_correct_names,
446 test_on_delete_cascade,
447 test_pickle_of_unpickle_is_identity,
448 test_unpickle_succeeds,
449 test_sms_detected_correctly ]
452 -- | Make sure our codegen is producing the correct database names.
454 test_news_fields_have_correct_names :: TestTree
455 test_news_fields_have_correct_names =
456 testCase "news fields get correct database names" $
457 mapM_ check (zip actual expected)
459 -- This is cool, it uses the (derived) Data instance of
460 -- News.News to get its constructor names.
461 field_names :: [String]
463 constrFields . head $ dataTypeConstrs $ dataTypeOf (undefined :: News)
467 map (\x -> tsn_db_field_namer "herp" "derp" 8675309 x 90210) field_names
470 actual = ["xml_file_id",
479 check (x,y) = (x @?= y)
482 -- | If we unpickle something and then pickle it, we should wind up
483 -- with the same thing we started with. WARNING: success of this
484 -- test does not mean that unpickling succeeded.
486 test_pickle_of_unpickle_is_identity :: TestTree
487 test_pickle_of_unpickle_is_identity = testGroup "pickle-unpickle tests"
488 [ check "pickle composed with unpickle is the identity"
489 "test/xml/newsxml.xml",
491 check "pickle composed with unpickle is the identity (with Editor)"
492 "test/xml/newsxml-with-editor.xml",
494 check "pickle composed with unpickle is the identity (empty SMS)"
495 "test/xml/newsxml-empty-sms.xml" ]
497 check desc path = testCase desc $ do
498 (expected, actual) <- pickle_unpickle pickle_message path
502 -- | Make sure we can actually unpickle these things.
504 test_unpickle_succeeds :: TestTree
505 test_unpickle_succeeds = testGroup "unpickle tests"
506 [ check "unpickling succeeds"
507 "test/xml/newsxml.xml",
509 check "unpickling succeeds (with Editor)"
510 "test/xml/newsxml-with-editor.xml",
512 check "unpickling succeeds (empty SMS)"
513 "test/xml/newsxml-empty-sms.xml" ]
515 check desc path = testCase desc $ do
516 actual <- unpickleable path pickle_message
521 -- | Make sure everything gets deleted when we delete the top-level
524 test_on_delete_cascade :: TestTree
525 test_on_delete_cascade = testGroup "cascading delete tests"
526 [ check "deleting news deletes its children"
527 "test/xml/newsxml.xml"
528 4 -- 2 news_teams and 2 news_locations that should remain.
530 check "deleting news deletes its children (empty SMS)"
531 "test/xml/newsxml-empty-sms.xml"
532 4 -- 2 news_teams and 2 news_locations
535 check desc path expected = testCase desc $ do
536 news <- unsafe_unpickle path pickle_message
537 let a = undefined :: Location
538 let b = undefined :: News
539 let c = undefined :: Team
540 let d = undefined :: News_Team
541 let e = undefined :: News_Location
542 actual <- withSqliteConn ":memory:" $ runDbConn $ do
543 runMigration silentMigrationLogger $ do
551 count_a <- countAll a
552 count_b <- countAll b
553 count_c <- countAll c
554 count_d <- countAll d
555 count_e <- countAll e
556 return $ count_a + count_b + count_c + count_d + count_e
560 -- | We want to make sure the single-SMS documents and the multi-SMS
561 -- documents are identified correctly.
563 test_sms_detected_correctly :: TestTree
564 test_sms_detected_correctly =
565 testGroup "newsxml SMS count determined correctly"
566 [ check "test/xml/newsxml.xml"
567 "single SMS detected correctly"
569 check "test/xml/newsxml-multiple-sms.xml"
570 "multiple SMS detected correctly"
573 check path desc expected = testCase desc $ do
574 xmltree <- unsafe_read_document path
575 let actual = has_only_single_sms xmltree