- def get_page_data(url, headers = {})
+ def get_page_data(url)
# A naive implementation that just grabs the
# data from a page.
uri = URI.parse(url)
response = Net::HTTP.start(uri.host, uri.port) do |http|
- http.get(uri.request_uri, headers)
+ http.get(uri.request_uri, self.headers)
end
return response.body
public;
+ # Additional headers used when requesting data from the website.
+ # These aren't passed as a parameter because the (final)
+ # downloaders need them as well.
+ attr_accessor :headers
+
def initialize(url)
@url = url
+ self.headers = { 'User-Agent' => Configuration::USER_AGENT }
end
-
+
def self.create(url)
# Factory method returning an instance of
# the appropriate subclass.