From: Michael Orlitzky Date: Tue, 2 Sep 2008 03:01:13 +0000 (-0400) Subject: Added independent (non-wget) file downloads via open-uri. X-Git-Url: https://gitweb.michael.orlitzky.com/?a=commitdiff_plain;h=9390083b114048a6782454a37f799733707ee5dc;p=dead%2Fwhatever-dl.git Added independent (non-wget) file downloads via open-uri. Removed a bad vimeo_remote_test require from the remote test suite. Added the UriUtilities class and a bunch of tests for it. Included the ProgressBar class in the vendor directory so that the UriUtilities class can display a progress bar during downloads. Added the ProgressBar tests to the test suite. Added error handling to the main whatever-dl script for newly-possible IO/connection errors. Removed the wget exec() call in favor of the open-uri call. --- diff --git a/bin/whatever-dl b/bin/whatever-dl index c20b8bd..cde9e4d 100755 --- a/bin/whatever-dl +++ b/bin/whatever-dl @@ -19,6 +19,11 @@ # http://www.fsf.org/licensing/licenses/gpl.html # +# We require the UriUtilities class to handle +# the download of the video URL. +require 'src/uri_utilities' + + # All of the website classes are located in one # directory, so we can 'require' them automatically. Dir.glob('src/websites/*.rb').each do |r| @@ -26,6 +31,16 @@ Dir.glob('src/websites/*.rb').each do |r| end +EXIT_SUCCESS = 0 +EXIT_NO_URL = 1 +EXIT_INVALID_URL = 2 +EXIT_COULDNT_GET_VIDEO_URL = 3 +EXIT_OUTPUT_FILE_ALREADY_EXISTS = 4 +EXIT_ERROR_READING_FROM_VIDEO_URL = 5 +EXIT_CONNECTION_REFUSED = 6 +EXIT_HTTP_ERROR = 7 +EXIT_ACCESS_DENIED = 8 + # Only actually do something if this script was called # directly (i.e. not from the tests). if (__FILE__ == $0) then @@ -33,7 +48,7 @@ if (__FILE__ == $0) then # If the user didn't give us a URL, yell # at him or her. puts 'Usage: whatever-dl ' - Kernel.exit(1) + Kernel.exit(EXIT_NO_URL) end # Check the URL against each website's class. @@ -51,16 +66,52 @@ if (__FILE__ == $0) then if site.nil? puts 'Invalid URL.' - exit(1) + exit(EXIT_INVALID_URL) end video_url = site.get_video_url(ARGV[0]) if video_url.nil? puts 'Error retrieving video URL.' - exit(2) + exit(EXIT_COULDNT_GET_VIDEO_URL) + end + + video_uri = URI.parse(video_url) + uu = UriUtilities.new() + + + # Here, we start out with a default file name and + # extension. If UriUtilities can parse a sane filename + # out of the URL, we'll use that. Otherwise, we fall + # back to the default. + outfile_name = 'default.ext' + + if not uu.get_filename(video_uri).nil? + outfile_name = uu.get_filename(video_uri) + else + puts "We couldn't determine the video's filename. Falling back to the default, #{outfile_name}." + end + + + if File.exists?(outfile_name) + puts "Error: output file already exists. Please remove #{outfile_name}, and try again." + Kernel.exit(EXIT_OUTPUT_FILE_ALREADY_EXISTS) + end + + + # Attempt to download the file, and rescue and report + # any (predictable) exceptions. + begin + uu.download_with_progress_bar(video_uri, outfile_name) + rescue Errno::ECONNREFUSED => e + puts 'The connection to the server (to download the video file) was refused. Check your connection, and try again later.' + Kernel.exit(EXIT_CONNECTION_REFUSED) + rescue Errno:EACCES => e + puts "Access denied. Check that you have write permission to the output file/directory. Details: #{e.message}." + rescue OpenURI::HTTPError => e + puts "An HTTP error occurred while downloading the video file: #{e.message}." + Kernel.exit(EXIT_HTTP_ERROR) end - # *classy* - Kernel.exec("wget \"#{video_url}\"") + Kernel.exit(EXIT_SUCCESS) end diff --git a/src/uri_utilities.rb b/src/uri_utilities.rb new file mode 100644 index 0000000..61d57af --- /dev/null +++ b/src/uri_utilities.rb @@ -0,0 +1,74 @@ +# +# Copyright Michael Orlitzky +# +# http://michael.orlitzky.com/ +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# http://www.fsf.org/licensing/licenses/gpl.html +# + +require 'open-uri' +require 'vendor/ruby-progressbar/progressbar' + +# Just a couple of convenience methods for URIs. +# These could be monkey-patched in, but with only +# a few methods, that'd be asking (unnecessarily) +# for trouble. +class UriUtilities + + # Get the filename portion of a given URI. + # Return nil if there is no filename portion. + def get_filename(uri) + return uri.path.split('/').last + end + + + # Download the given URI object to . + # Should use the progress_proc parameter to show + # a progress bar using the Ruby/ProgressBar library. + def download_with_progress_bar(uri, outfile_name) + # We wrap the whole thing in a begin/rescue so that + # we can clean up afterwards in case of an error. + begin + open(outfile_name, 'wb') do |outfile| + pbar = nil + uri.open(:content_length_proc => lambda {|content_length| + if content_length && (0 < content_length) + pbar = ProgressBar.new("Download", content_length) + pbar.instance_eval { @bar_mark = '=' } + pbar.file_transfer_mode + end + }, + :progress_proc => lambda {|size| + pbar.set(size) if pbar + }) do |video_file| + outfile.write(video_file.read) + end + end + + # Toss out an empty line to get rid of the progress bar. + # Normally, it would remain on the shell's "current" line. + puts '' + + rescue Errno::EACCES => e + # Don't delete the file if it's unwritable. + raise(e) + rescue => e + # Here we get rid of the output file if there was an error. + # We test File.exists? first since the first line, the open() + # call, could theoretically fail. + File.delete(outfile_name) if File.exists?(outfile_name) + raise(e) + end + end + +end diff --git a/src/website.rb b/src/website.rb index dad2264..1239712 100644 --- a/src/website.rb +++ b/src/website.rb @@ -25,6 +25,8 @@ class Website superclass.inherited(subclass) end + # Every time we're subclassed, add the new + # subclass to our list of subclasses. @subclasses ||= [] @subclasses << subclass end diff --git a/test/remote_test_suite.rb b/test/remote_test_suite.rb index 4a20e62..46c75a5 100644 --- a/test/remote_test_suite.rb +++ b/test/remote_test_suite.rb @@ -17,5 +17,5 @@ # require 'test/infoq_remote_test' -require 'test/vimeo_remote_test' +require 'test/uri_utilities_remote_test' require 'test/youporn_remote_test' diff --git a/test/test_suite.rb b/test/test_suite.rb index 70fec3a..6efa044 100644 --- a/test/test_suite.rb +++ b/test/test_suite.rb @@ -20,6 +20,10 @@ require 'test/howcast_test' require 'test/infoq_test' require 'test/redtube_test' require 'test/veoh_test' +require 'test/uri_utilities_test' require 'test/vimeo_test' require 'test/website_test' require 'test/youporn_test' + +# Vendor tests +#require 'vendor/ruby-progressbar/test' diff --git a/test/uri_utilities_remote_test.rb b/test/uri_utilities_remote_test.rb new file mode 100644 index 0000000..f8c6b8d --- /dev/null +++ b/test/uri_utilities_remote_test.rb @@ -0,0 +1,40 @@ +# +# Copyright Michael Orlitzky +# +# http://michael.orlitzky.com/ +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# http://www.fsf.org/licensing/licenses/gpl.html +# + +require 'test/unit' +require 'src/uri_utilities.rb' + +class UriUtilitiesTest < Test::Unit::TestCase + + # Test that we get a 404 error for a nonexistent page. + def test_404_on_nonexistent_page + uu = UriUtilities.new() + + # *Hopefully* this connection is refused. + example_uri = 'http://www.example.com/test-nonexistent-page.html' + uri = URI.parse(example_uri) + + assert_raise OpenURI::HTTPError do + uu.download_with_progress_bar(uri, 'dummy.tmp') + end + + # This should be gone since there was an HTTP error. + assert(!File.exists?('dummy.tmp')) + end + +end diff --git a/test/uri_utilities_test.rb b/test/uri_utilities_test.rb new file mode 100644 index 0000000..ad4d4f3 --- /dev/null +++ b/test/uri_utilities_test.rb @@ -0,0 +1,91 @@ +# +# Copyright Michael Orlitzky +# +# http://michael.orlitzky.com/ +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# http://www.fsf.org/licensing/licenses/gpl.html +# + +require 'test/unit' +require 'src/uri_utilities.rb' + +class UriUtilitiesTest < Test::Unit::TestCase + + def test_filename + uu = UriUtilities.new() + example_uri = 'http://www.example.com/whatever.avi' + uri = URI.parse(example_uri) + assert_equal('whatever.avi', uu.get_filename(uri)) + end + + + def test_no_filename_results_in_nil + uu = UriUtilities.new() + example_uri = 'http://www.example.com' + uri = URI.parse(example_uri) + assert(uu.get_filename(uri).nil?) + end + + + def test_no_filename_with_trailing_slash_results_in_nil + uu = UriUtilities.new() + example_uri = 'http://www.example.com/' + uri = URI.parse(example_uri) + assert(uu.get_filename(uri).nil?) + end + + + def test_system_call_exception_on_connection_refused + uu = UriUtilities.new() + + # *Hopefully* this connection is refused. + example_uri = 'http://localhost:1/' + uri = URI.parse(example_uri) + + assert_raise Errno::ECONNREFUSED do + uu.download_with_progress_bar(uri, 'dummy.tmp') + end + + # This should be gone since there was an error. + assert(!File.exists?('dummy.tmp')) + end + + + def test_access_denied_on_unwritable_outfile + # Get rid of the dummy file if it already exists. + File.delete('dummy.tmp') if File.exists?('dummy.tmp') + + File.new('dummy.tmp', 'w') + + # Make the new file unwritable + File.chmod(0400, 'dummy.tmp') + + uu = UriUtilities.new() + + # *Hopefully* this connection is refused. + example_uri = 'http://localhost:1/' + uri = URI.parse(example_uri) + + assert_raise Errno::EACCES do + uu.download_with_progress_bar(uri, 'dummy.tmp') + end + + # This should *not* be deleted, since it was unwritable, + # and we got an IO error instead of an HTTP or connection + # error. + assert(File.exists?('dummy.tmp')) + + # Ok, now we delete it anyway since it will mess up the other tests. + File.delete('dummy.tmp') + end +end diff --git a/vendor/ruby-progressbar/progressbar.rb b/vendor/ruby-progressbar/progressbar.rb new file mode 100644 index 0000000..507082d --- /dev/null +++ b/vendor/ruby-progressbar/progressbar.rb @@ -0,0 +1,236 @@ +# +# Ruby/ProgressBar - a text progress bar library +# +# Copyright (C) 2001-2005 Satoru Takabayashi +# All rights reserved. +# This is free software with ABSOLUTELY NO WARRANTY. +# +# You can redistribute it and/or modify it under the terms +# of Ruby's license. +# + +class ProgressBar + VERSION = "0.9" + + def initialize (title, total, out = STDERR) + @title = title + @total = total + @out = out + @terminal_width = 80 + @bar_mark = "o" + @current = 0 + @previous = 0 + @finished_p = false + @start_time = Time.now + @previous_time = @start_time + @title_width = 14 + @format = "%-#{@title_width}s %3d%% %s %s" + @format_arguments = [:title, :percentage, :bar, :stat] + clear + show + end + attr_reader :title + attr_reader :current + attr_reader :total + attr_accessor :start_time + + private + def fmt_bar + bar_width = do_percentage * @terminal_width / 100 + sprintf("|%s%s|", + @bar_mark * bar_width, + " " * (@terminal_width - bar_width)) + end + + def fmt_percentage + do_percentage + end + + def fmt_stat + if @finished_p then elapsed else eta end + end + + def fmt_stat_for_file_transfer + if @finished_p then + sprintf("%s %s %s", bytes, transfer_rate, elapsed) + else + sprintf("%s %s %s", bytes, transfer_rate, eta) + end + end + + def fmt_title + @title[0,(@title_width - 1)] + ":" + end + + def convert_bytes (bytes) + if bytes < 1024 + sprintf("%6dB", bytes) + elsif bytes < 1024 * 1000 # 1000kb + sprintf("%5.1fKB", bytes.to_f / 1024) + elsif bytes < 1024 * 1024 * 1000 # 1000mb + sprintf("%5.1fMB", bytes.to_f / 1024 / 1024) + else + sprintf("%5.1fGB", bytes.to_f / 1024 / 1024 / 1024) + end + end + + def transfer_rate + bytes_per_second = @current.to_f / (Time.now - @start_time) + sprintf("%s/s", convert_bytes(bytes_per_second)) + end + + def bytes + convert_bytes(@current) + end + + def format_time (t) + t = t.to_i + sec = t % 60 + min = (t / 60) % 60 + hour = t / 3600 + sprintf("%02d:%02d:%02d", hour, min, sec); + end + + # ETA stands for Estimated Time of Arrival. + def eta + if @current == 0 + "ETA: --:--:--" + else + elapsed = Time.now - @start_time + eta = elapsed * @total / @current - elapsed; + sprintf("ETA: %s", format_time(eta)) + end + end + + def elapsed + elapsed = Time.now - @start_time + sprintf("Time: %s", format_time(elapsed)) + end + + def eol + if @finished_p then "\n" else "\r" end + end + + def do_percentage + if @total.zero? + 100 + else + @current * 100 / @total + end + end + + def get_width + # FIXME: I don't know how portable it is. + default_width = 80 + begin + tiocgwinsz = 0x5413 + data = [0, 0, 0, 0].pack("SSSS") + if @out.ioctl(tiocgwinsz, data) >= 0 then + rows, cols, xpixels, ypixels = data.unpack("SSSS") + if cols >= 0 then cols else default_width end + else + default_width + end + rescue Exception + default_width + end + end + + def show + arguments = @format_arguments.map {|method| + method = sprintf("fmt_%s", method) + send(method) + } + line = sprintf(@format, *arguments) + + width = get_width + if line.length == width - 1 + @out.print(line + eol) + @out.flush + elsif line.length >= width + @terminal_width = [@terminal_width - (line.length - width + 1), 0].max + if @terminal_width == 0 then @out.print(line + eol) else show end + else # line.length < width - 1 + @terminal_width += width - line.length + 1 + show + end + @previous_time = Time.now + end + + def show_if_needed + if @total.zero? + cur_percentage = 100 + prev_percentage = 0 + else + cur_percentage = (@current * 100 / @total).to_i + prev_percentage = (@previous * 100 / @total).to_i + end + + # Use "!=" instead of ">" to support negative changes + if cur_percentage != prev_percentage || + Time.now - @previous_time >= 1 || @finished_p + show + end + end + + public + def clear + @out.print "\r" + @out.print(" " * (get_width - 1)) + @out.print "\r" + end + + def finish + @current = @total + @finished_p = true + show + end + + def finished? + @finished_p + end + + def file_transfer_mode + @format_arguments = [:title, :percentage, :bar, :stat_for_file_transfer] + end + + def format= (format) + @format = format + end + + def format_arguments= (arguments) + @format_arguments = arguments + end + + def halt + @finished_p = true + show + end + + def inc (step = 1) + @current += step + @current = @total if @current > @total + show_if_needed + @previous = @current + end + + def set (count) + if count < 0 || count > @total + raise "invalid count: #{count} (total: #{@total})" + end + @current = count + show_if_needed + @previous = @current + end + + def inspect + "#" + end +end + +class ReversedProgressBar < ProgressBar + def do_percentage + 100 - super + end +end + diff --git a/vendor/ruby-progressbar/test.rb b/vendor/ruby-progressbar/test.rb new file mode 100644 index 0000000..06cd571 --- /dev/null +++ b/vendor/ruby-progressbar/test.rb @@ -0,0 +1,110 @@ +require 'test/unit' + +# Modified a little to run from the whatever-dl test suite. +require (File.dirname(__FILE__) + '/progressbar') + +class ProgressBarTest < Test::Unit::TestCase + SleepUnit = 0.01 + + def do_make_progress_bar (title, total) + ProgressBar.new(title, total) + end + + def test_bytes + total = 1024 * 1024 + pbar = do_make_progress_bar("test(bytes)", total) + pbar.file_transfer_mode + 0.step(total, 2**14) {|x| + pbar.set(x) + sleep(SleepUnit) + } + pbar.finish + end + + def test_clear + total = 100 + pbar = do_make_progress_bar("test(clear)", total) + total.times { + sleep(SleepUnit) + pbar.inc + } + pbar.clear + puts + end + + def test_halt + total = 100 + pbar = do_make_progress_bar("test(halt)", total) + (total / 2).times { + sleep(SleepUnit) + pbar.inc + } + pbar.halt + end + + def test_inc + total = 100 + pbar = do_make_progress_bar("test(inc)", total) + total.times { + sleep(SleepUnit) + pbar.inc + } + pbar.finish + end + + def test_inc_x + # Modified a little to run from the whatever-dl test suite. + pbar_file_path = (File.dirname(__FILE__) + '/progressbar.rb') + + total = File.size(pbar_file_path) + pbar = do_make_progress_bar("test(inc(x))", total) + File.new(pbar_file_path).each {|line| + sleep(SleepUnit) + pbar.inc(line.length) + } + pbar.finish + end + + def test_invalid_set + total = 100 + pbar = do_make_progress_bar("test(invalid set)", total) + begin + pbar.set(200) + rescue RuntimeError => e + puts e.message + end + end + + def test_set + total = 1000 + pbar = do_make_progress_bar("test(set)", total) + (1..total).find_all {|x| x % 10 == 0}.each {|x| + sleep(SleepUnit) + pbar.set(x) + } + pbar.finish + end + + def test_slow + total = 100000 + pbar = do_make_progress_bar("test(slow)", total) + 0.step(500, 1) {|x| + pbar.set(x) + sleep(SleepUnit) + } + pbar.halt + end + + def test_total_zero + total = 0 + pbar = do_make_progress_bar("test(total=0)", total) + pbar.finish + end +end + +class ReversedProgressBarTest < ProgressBarTest + def do_make_progress_bar (title, total) + ReversedProgressBar.new(title, total) + end +end +