Use parse_opts_novalidate for the problem DTDs weatherxml and newsxml.

[dead/htsn-import.git] / src / Main.hs

diff --git a/src/Main.hs b/src/Main.hs
index 63bde65d78276e6c511f4168217a5f082457b3eb..e36cc789b4d2f2dbfa485e9af9326a2e63da4824 100644 (file)
--- a/src/Main.hs
+++ b/src/Main.hs
@@ -22,6 +22,7 @@ import System.IO.Error ( catchIOError )
 import Text.XML.HXT.Core (
   ArrowXml,
   IOStateArrow,
+  SysConfigList,
   XmlTree,
   (>>>),
   (/>),
@@ -45,6 +46,10 @@ import Network.Services.TSN.Report (
   report_info,
   report_error )
 import TSN.DbImport ( DbImport(..), ImportResult(..) )
+import TSN.Parse ( format_parse_error )
+import qualified TSN.XML.AutoRacingDriverList as AutoRacingDriverList (
+  dtd,
+  pickle_message )
 import qualified TSN.XML.AutoRacingResults as AutoRacingResults (
   dtd,
   pickle_message )
@@ -60,6 +65,9 @@ import qualified TSN.XML.Injuries as Injuries ( dtd, pickle_message )
 import qualified TSN.XML.InjuriesDetail as InjuriesDetail (
   dtd,
   pickle_message )
+import qualified TSN.XML.MLBEarlyLine as MLBEarlyLine (
+  dtd,
+  pickle_message )
 import qualified TSN.XML.JFile as JFile ( dtd, pickle_message )
 import qualified TSN.XML.News as News (
   dtd,
@@ -76,7 +84,7 @@ import qualified TSN.XML.Weather as Weather (
   is_type1,
   pickle_message,
   teams_are_normal )
-import Xml ( DtdName(..), parse_opts )
+import Xml ( DtdName(..), parse_opts, parse_opts_novalidate )
 
 
 -- | This is where most of the work happens. This function is called
@@ -133,9 +141,11 @@ import_file cfg path = do
       -- we couldn't parse the DTD.
       return [ImportFailed errdesc]
 
-    -- | An arrow that reads a document into an 'XmlTree'.
-    readA :: IOStateArrow s a XmlTree
-    readA = readDocument parse_opts path
+    -- | An arrow that reads a document into an 'XmlTree'.  We take a
+    --   SysConfigList so our caller can decide whether or not to
+    --   e.g. validate the document against its DTD.
+    readA :: SysConfigList -> IOStateArrow s a XmlTree
+    readA scl = readDocument scl path
 
     -- | An arrow which parses the doctype "SYSTEM" of an 'XmlTree'.
     --   We use these to determine the parser to use.
@@ -149,11 +159,27 @@ import_file cfg path = do
     --   The result of runX has type IO [IO ImportResult]. We thus use
     --   bind (>>=) and sequence to combine all of the IOs into one
     --   big one outside of the list.
+    --
+    --   Before we actually run the import, we check it against a list
+    --   of problem DTDs. These can produce weird errors, and we have
+    --   checks for them. But with DTD validation enabled, we can't
+    --   even look inside the document to see what's wrong -- parsing
+    --   will fail! So for those special document types, we proceed
+    --   using 'parse_opts_novalidate' instead of the default
+    --   'parse_opts'.
+    --
     parse_and_import :: IO [ImportResult]
-    parse_and_import =
-      runX (readA >>> (dtdnameA &&& returnA) >>> (arr import_with_dtd))
-      >>=
-      sequence
+    parse_and_import = do
+      -- Get the DTD name without validating against it.
+      ((DtdName dtd) : _) <- runX $ (readA parse_opts_novalidate) >>> dtdnameA
+
+      let problem_dtds = [ News.dtd, Weather.dtd ]
+      let opts = if dtd `elem` problem_dtds
+                 then parse_opts_novalidate
+                 else parse_opts
+
+      runX ((readA opts) >>> (dtdnameA &&& returnA) >>> (arr import_with_dtd))
+        >>= sequence
 
     -- | Takes a ('DtdName', 'XmlTree') pair and uses the 'DtdName'
     --   to determine which function to call on the 'XmlTree'.
@@ -191,6 +217,9 @@ import_file cfg path = do
                    (unpickleDoc f xml)
 
           importer
+            | dtd == AutoRacingDriverList.dtd =
+                go AutoRacingDriverList.pickle_message
+
             | dtd == AutoRacingResults.dtd =
                 go AutoRacingResults.pickle_message
 
@@ -207,6 +236,8 @@ import_file cfg path = do
 
             | dtd == JFile.dtd = go JFile.pickle_message
 
+            | dtd == MLBEarlyLine.dtd = go MLBEarlyLine.pickle_message
+
             | dtd == News.dtd =
                 -- Some of the newsxml docs are busted in predictable ways.
                 -- We want them to "succeed" so that they're deleted.
@@ -244,7 +275,7 @@ import_file cfg path = do
                 case either_m of
                   -- This might give us a slightly better error
                   -- message than the default 'errmsg'.
-                  Left err -> return $ ImportFailed err
+                  Left err -> return $ ImportFailed (format_parse_error err)
                   Right m     -> migrate_and_import m
 
             | dtd `elem` SportInfo.dtds = do
@@ -252,7 +283,7 @@ import_file cfg path = do
                 case either_m of
                   -- This might give us a slightly better error
                   -- message than the default 'errmsg'.
-                  Left err -> return $ ImportFailed err
+                  Left err -> return $ ImportFailed (format_parse_error err)
                   Right m     -> migrate_and_import m
 
             | otherwise = do