Right cp ->
let cp_username = get cp "DEFAULT" "username"
cp_password = get cp "DEFAULT" "password"
-
+
cfg_username = either_to_maybe cp_username
cfg_password = either_to_maybe cp_password
in
default_curl_opts =
[ -- The Global cache is not thread-friendly.
CurlDNSUseGlobalCache False,
-
+
-- And we don't want to use a DNS cache anyway.
CurlDNSCacheTimeout 0,
CurlCookieJar cookie_jar,
CurlPost True,
CurlPostFields post_data ]
-
+
curl_opts :: [CurlOption]
curl_opts = default_curl_opts ++ post_opts
remove_comment_links :: (ArrowXml a) => a XmlTree XmlTree
remove_comment_links =
processTopDown $ kill_comments `when` is_link
- where
+ where
is_comment_link =
hasAttrValue "href" (contains "#Comments")
return $
if (isNothing appr) then
fppr
- else
+ else
appr
--
ap_parse :: IOSArrow XmlTree XmlTree -> IO (Maybe Page)
ap_parse xml = do
- arts <- ap_parse_articles xml
+ arts <- ap_parse_articles xml
case arts of
[x] -> return $ Just $ ArticlePage x
_ -> return Nothing
-
+
ap_parse_body :: IOSArrow XmlTree XmlTree -> IO (Maybe String)
ap_parse_body xml = do
let element_filter = xml >>> css "div.ArticleText"
if (isNothing parsed_headline) || (isNothing parsed_body)
then return []
- else do
+ else do
let title' = Title $ fromJust parsed_headline
let byline' = Byline parsed_byline
let body' = BodyHtml $ fromJust parsed_body
-
+
return $ [Article title' byline' body']
let xml = parseHtml $ wrap_in_body_div html
fp_parse_article xml
-
+
-- | In the full page, all of the article titles and bodies are
-- wrapped in one big div.ArticleText.
parse_bodies :: IOSArrow XmlTree XmlTree -> IOSArrow XmlTree XmlTree
changeAttrValue try_make_absolute_url
make_srcs_absolute :: (ArrowXml a) => a XmlTree XmlTree
- make_srcs_absolute =
+ make_srcs_absolute =
processAttrl $ change_src `when` hasName "src"
if (isSuffixOf "/" old_path) || (isSuffixOf "bigpage" old_path) then
-- It already had a trailing slash, or it's a 'bigpage' URL.
-- Trailing slashes after 'bigpage' don't work.
- url
+ url
else
show $ uri { uriPath = old_path ++ "/" }
where
make_absolute_url :: URL -> Maybe URL
-make_absolute_url relative_url =
+make_absolute_url relative_url =
case parse_result of
Nothing -> Nothing
Just relative_uri ->
-- | Try to parse the given article using HXT. We try a few different
-- methods; if none of them work, we return 'Nothing'.
get_xml_from_article :: Cfg -> IO (Maybe (IOStateArrow s b XmlTree))
-get_xml_from_article cfg = do
+get_xml_from_article cfg = do
my_article <- real_article_path (article cfg)
is_file <- doesFileExist my_article
case is_file of