From 05f6c664ebd77e194656a828855d74d2c4959d85 Mon Sep 17 00:00:00 2001 From: Michael Orlitzky Date: Mon, 24 Jan 2011 21:07:08 -0500 Subject: [PATCH] Allow the downloaders to take advantage of the websites' headers. --- bin/whatever-dl | 1 + src/downloader.rb | 2 +- src/open_uri_downloader.rb | 4 ++-- src/uri_utilities.rb | 13 +++++++------ src/wget_downloader.rb | 8 ++++++-- 5 files changed, 17 insertions(+), 11 deletions(-) diff --git a/bin/whatever-dl b/bin/whatever-dl index 448db85..f042b21 100755 --- a/bin/whatever-dl +++ b/bin/whatever-dl @@ -135,6 +135,7 @@ if (__FILE__ == $0) then begin downloader.download(video_url, site.get_video_filename(), + site.headers(), continue=options[:continue]) rescue Errno::ECONNREFUSED => e puts 'The connection to the server (to download the video file) was refused. Check your connection, and try again later.' diff --git a/src/downloader.rb b/src/downloader.rb index 742e792..b9921e8 100644 --- a/src/downloader.rb +++ b/src/downloader.rb @@ -30,7 +30,7 @@ class Downloader # Abstract - def download(url, outfile, continue=false) + def download(url, outfile, headers = {}, continue = false) raise NotImplementedError end diff --git a/src/open_uri_downloader.rb b/src/open_uri_downloader.rb index 06c9b3d..e55c0fa 100644 --- a/src/open_uri_downloader.rb +++ b/src/open_uri_downloader.rb @@ -20,7 +20,7 @@ require 'src/uri_utilities' class OpenUriDownloader < Downloader - def download(url, outfile, continue=false) + def download(url, outfile, headers = {}, continue = false) if File.exists?(outfile) raise IOError.new("Output file already exists. Please remove #{outfile}, and try again.") end @@ -31,7 +31,7 @@ class OpenUriDownloader < Downloader puts "Fetching #{url}" puts "Saving as #{outfile}." puts '' - uu.download_with_progress_bar(uri, outfile) + uu.download_with_progress_bar(uri, outfile, headers) end end diff --git a/src/uri_utilities.rb b/src/uri_utilities.rb index df09619..dcff2fe 100644 --- a/src/uri_utilities.rb +++ b/src/uri_utilities.rb @@ -28,14 +28,14 @@ class UriUtilities # Download the given URI object to . # Should use the progress_proc parameter to show # a progress bar using the Ruby/ProgressBar library. - def download_with_progress_bar(uri, outfile_name) + def download_with_progress_bar(uri, outfile_name, headers = {}) # We wrap the whole thing in a begin/rescue so that # we can clean up afterwards in case of an error. begin File.open(outfile_name, 'wb') do |outfile| pbar = nil - uri.open(:content_length_proc => lambda {|content_length| - if content_length && (0 < content_length) + uri.open({:content_length_proc => lambda {|content_length| + if content_length && (0 < content_length) pbar = ProgressBar.new("Download", content_length) pbar.instance_eval { @bar_mark = '=' } pbar.file_transfer_mode @@ -43,9 +43,10 @@ class UriUtilities }, :progress_proc => lambda {|size| pbar.set(size) if pbar - }) do |video_file| + } + }.merge!(headers)) do |video_file| outfile.write(video_file.read) - end + end end # Toss out an empty line to get rid of the progress bar. @@ -63,5 +64,5 @@ class UriUtilities raise(e) end end - + end diff --git a/src/wget_downloader.rb b/src/wget_downloader.rb index 825ee12..ce4c760 100644 --- a/src/wget_downloader.rb +++ b/src/wget_downloader.rb @@ -18,7 +18,7 @@ class WgetDownloader < Downloader - def download(url, outfile, continue=false) + def download(url, outfile, headers = {}, continue = false) if (continue == false and File.exists?(outfile)) raise IOError.new("Output file already exists. Please remove #{outfile}, and try again. If this is a partially-downloaded file, you can use the --continue flag to pick up where it left off.") end @@ -26,7 +26,11 @@ class WgetDownloader < Downloader options = '' if continue == true - options += '--continue' + options += '--continue ' + end + + headers.each_key do |key| + options += "--header '#{key}: #{headers[key]}' " end # This one's easy. -- 2.44.2