module LWN.Page
where
+import Control.Concurrent.ParallelIO (parallel)
import qualified Data.Map as Map (lookup)
import Data.Time (getCurrentTime)
import qualified Data.ByteString.Lazy as B (ByteString, hPut)
-- These come *before* preprocessing.
download_full_story_urls :: Cfg -> [URL] -> IO StoryMap
download_full_story_urls cfg story_urls = do
- pages <- mapM (page_from_url cfg) story_urls
+ pages <- parallel $ map (page_from_url cfg) story_urls
let pairs = zip story_urls pages
return $ foldl my_insert empty_map pairs
where
[] -> []
-- Put the separator back, it was lost during the split.
let split_articles = map (article_separator ++) split_articles'
- --_ <- mapM print_article split_articles
real_articles <- mapM parse_html_article split_articles
let just_articles = catMaybes real_articles
return just_articles