From af614c64b3d5998471af5e54b3d8f36d3e00cc63 Mon Sep 17 00:00:00 2001 From: mjo Date: Tue, 2 Sep 2008 18:13:40 -0400 Subject: [PATCH] Added the ability to download Youtube videos. Fixed a typo in the main executable which was causing a crash. Added the progress bar tests back to the suite. Added a Youtube class and the accompanying unit tests/fixtures. --- bin/whatever-dl | 2 +- src/websites/youtube.rb | 107 ++ test/fixtures/youtube/SudixyugiX4.html | 1916 ++++++++++++++++++++++++ test/test_suite.rb | 3 +- test/youtube_test.rb | 64 + 5 files changed, 2090 insertions(+), 2 deletions(-) create mode 100644 src/websites/youtube.rb create mode 100644 test/fixtures/youtube/SudixyugiX4.html create mode 100644 test/youtube_test.rb diff --git a/bin/whatever-dl b/bin/whatever-dl index cde9e4d..970e0e0 100755 --- a/bin/whatever-dl +++ b/bin/whatever-dl @@ -106,7 +106,7 @@ if (__FILE__ == $0) then rescue Errno::ECONNREFUSED => e puts 'The connection to the server (to download the video file) was refused. Check your connection, and try again later.' Kernel.exit(EXIT_CONNECTION_REFUSED) - rescue Errno:EACCES => e + rescue Errno::EACCES => e puts "Access denied. Check that you have write permission to the output file/directory. Details: #{e.message}." rescue OpenURI::HTTPError => e puts "An HTTP error occurred while downloading the video file: #{e.message}." diff --git a/src/websites/youtube.rb b/src/websites/youtube.rb new file mode 100644 index 0000000..6766af2 --- /dev/null +++ b/src/websites/youtube.rb @@ -0,0 +1,107 @@ +# +# Copyright Michael Orlitzky +# +# http://michael.orlitzky.com/ +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# http://www.fsf.org/licensing/licenses/gpl.html +# + +require 'src/website' + +# Needed to download the page, which is in turn +# needed because it contains the video URL. +require 'net/http' +require 'uri' + + +class Youtube < Website + + VALID_YOUTUBE_URL_REGEX = /^(http:\/\/)?(www\.)?youtube\.com\/((watch\?v=)|(v\/))[[:alnum:]]+(\&.*)?$/ + + def self.owns_url?(url) + return url =~ VALID_YOUTUBE_URL_REGEX + end + + + def get_video_url(url) + video_id = self.parse_video_id(url) + + # The video's URL (the "page data" URL) may be different from the + # URL that was passed to the program. We support the /v/video_id + # URL format, but that is *not* the main video page where we can + # retrieve the "t" parameter. We can only get that from the + # /watch?v=video_id form. + page_data_url = "http://www.youtube.com/watch?v=#{video_id}" + page_data = self.get_page_data(page_data_url) + + # Magic. + t_parameter = self.parse_t_parameter(page_data) + + video_url = "http://www.youtube.com/get_video?video_id=#{video_id}&t=#{t_parameter}" + + return video_url + end + + + protected; + + # Get the video id from the URL. Should be relatively easy, + # unless Youtube supports some URL formats of which I'm unaware. + def parse_video_id(url) + # Return nil if we get no matches below. + video_id = nil + + # Both URLs are fairly easy to parse if you handle + # them one at a time. The only tricky situation is when + # parameters like "&hl=en" are tacked on to the end. + # We'll call /watch?v=video_id the "first form." + first_form_video_id_regex = /v=([[:alnum:]]+)$/ + first_form_matches = first_form_video_id_regex.match(url) + return first_form_matches[1] if not (first_form_matches.nil? || + first_form_matches.length < 2) + + # First form didn't work? Try the second. + second_form_video_id_regex = /\/v\/([[:alnum:]]+)/ + second_form_matches = second_form_video_id_regex.match(url) + video_id = second_form_matches[1] if not (second_form_matches.nil? || + second_form_matches.length < 2) + + return video_id + end + + + # Parse out the "t" parameter from the video's page. I'm not sure + # what "t" stands for, but it's located in some JSON, and is required + # for the final video URL to work. + def parse_t_parameter(page_data) + t_parameter = nil + + t_parameter_regex = /\"t\"\:[[:space:]]\"([[:alnum:]]+)\"/ + matches = t_parameter_regex.match(page_data) + t_parameter = matches[1] if not (matches.nil? || matches.length < 2) + + return t_parameter + end + + + def get_page_data(url) + uri = URI.parse(url) + + response = Net::HTTP.start(uri.host, uri.port) do |http| + http.get(uri.request_uri) + end + + return response.body + end + +end diff --git a/test/fixtures/youtube/SudixyugiX4.html b/test/fixtures/youtube/SudixyugiX4.html new file mode 100644 index 0000000..80688bc --- /dev/null +++ b/test/fixtures/youtube/SudixyugiX4.html @@ -0,0 +1,1916 @@ + + + + + + + + + + YouTube - Domino Domino Logic + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ +
+ +
+ + +
+ +
+ +
+ + + Upload + + +
+ + + +
+ +
+ + advanced +
+
+ +
+ +
+ +
+
Loading...
+
+ +
+
+
+ + + + + + + + +
+
Domino Domino Logic
+
+ +
+
+
+ + +
+
+ + + + + +
+
+
Rate:
+ + + + +
4.5
557 ratings
Sign in to rate
+ +
+ +
+
+ Views: + 141,905 +
+ + + +
+
+ +
+
+ +
+ + + + + + + +
+ + Share + Share + + + + Favorite + Favorite + + + + Add to Playlists + Playlists + + + + Report video as inappropriate + Flag + +
+
+
+
+ +
This video will appear on your blog shortly.
+
Loading...
+
Loading...
+
Loading...
+
Loading...
+
Thank you for sharing this video!
+ +
+
+
+
+ + + + +
+
+
Saving...
+
+
+ + + + +
+
+
+
+
(close)
+
 
+ This video has been added to your favorites.

+
+
+
(close)
+
 
+ This video has been removed from your favorites.

+
+
+ +
+
+
+ Want to add to Favorites?
+ Sign in to YouTube now! +
+ + Sign in with your Google Account! + + +
+ +
+
+
+
Loading...
+
The video has been added to your playlist.
+
+
+
+ Want to add to Playlists?
+ Sign in to YouTube now! +
+ + Sign in with your Google Account! + + +
+ +
+
+
+
Loading...
+
+
+
+
+ Want to flag a video?
+ Sign in to YouTube now! +
+ + Sign in with your Google Account! + + +
+ +
+
+
+
+ + + + + +
+ + + + + +
+ Commentary + + Statistics & Data +
+
+
+
+ Video Responses: 0 + Text Comments: 180 +
+ +
+
+ Sign in to post a Video Response +
+ Video Responses (0) +
+ This video has no Responses. Be the first to Post a Video Response. +
+
+ +
+
+ Sign in to post a Comment +
+ Text Comments (180) + +
+
+
+
+
+ Show: + + + Help + + Change this to see only comments above a certain value.
Change the value of a comment by clicking on a thumb. + + +
+
+ +
+ +
+ +
+ + + + + + + +
+ + +
+
+ cyborgtroy + (1 week ago) + Show + Hide +
+
+ + 0 + + Poor comment + Good comment + +
+ + + Marked as spam +
+ Reply + + + + + + +
+ +
+
+
+
+ I think I tried to do this sort of thing once. AND gates rather befuddled me.

And how would NOT work? +
+
+
+ + +
+ + + +
+ + +
+
+ jefersonrod + (2 weeks ago) + Show + Hide +
+
+ + 0 + + Poor comment + Good comment + +
+ + + Marked as spam +
+ Reply + + + + + + +
+ +
+
+
+
+ awesome +
+
+
+ + +
+ + + +
+ + +
+
+ Tehw00tz + (2 weeks ago) + Show + Hide +
+
+ + 0 + + Poor comment + Good comment + +
+ + + Marked as spam +
+ Reply + + + + + + +
+ +
+
+
+
+ Jesus you must have a lot of patience to be able to knock those dominos down and put them back up just to knock them down again +
+
+
+ + +
+ + + +
+ + +
+
+ iamneuron + (3 weeks ago) + Show + Hide +
+
+ +  -2 + + Poor comment + Good comment + +
+ + + Marked as spam +
+ Reply + + + + + + +
+ +
+
+
+
+ Where is NAND? +
+
+
+ + +
+ + + +
+ +
+ +
+
+ TiCLer + (2 weeks ago) + Show + Hide +
+
+ + +2 + + Poor comment + Good comment + +
+ + + Marked as spam +
+ Reply + + + + + + +
+ +
+
+
+
+ Lurk moar! +
+
+
+ +
+ +
+ + + +
+ + +
+
+ smasher32 + (3 weeks ago) + Show + Hide +
+
+ +  -3 + + Poor comment + Good comment + +
+ + + Marked as spam +
+ Reply + + + + + + +
+ +
+
+
+
+ dont quite understand the logic behind it, but seems interesting. +
+
+
+ + +
+ + + +
+ + +
+
+ MBieroegel + (3 weeks ago) + Show + Hide +
+
+ + 0 + + Poor comment + Good comment + +
+ + + Marked as spam +
+ Reply + + + + + + +
+ +
+
+
+
+ Yeah cool... now i know, how it work's ;-) +
+
+
+ + +
+ + + +
+ + +
+
+ AnotherGlenn + (3 weeks ago) + Show + Hide +
+
+ + +3 + + Poor comment + Good comment + +
+ + + Marked as spam +
+ Reply + + + + + + +
+ +
+
+
+
+ LOL, nerdier than me :) +
+
+
+ + +
+ + + +
+ + +
+
+ thailsoff + (3 weeks ago) + Show + Hide +
+
+ + +3 + + Poor comment + Good comment + +
+ + + Marked as spam +
+ Reply + + + + + + +
+ +
+
+
+
+ O my f***ing GOD ! the Xor is PERFECT ! lol ! very very good movie. +
+
+
+ + +
+ + + +
+ + +
+
+ smartinp + (3 weeks ago) + Show + Hide +
+
+ + 0 + + Poor comment + Good comment + +
+ + + Marked as spam +
+ Reply + + + + + + +
+ +
+
+
+
+ WOW xD +
+
+
+ + +
+ + + +
+
+ +Next +
+ +
+ Pages: + 1 + 2 + 3 +  ...  + +
+
+
+ + +
+ + + +
+

Would you like to comment?

+
+ Join YouTube for a free account, or + sign in if you are already a member. +
+
+ +
+
+
+ + +
+
+ + + + + + + + + + + +
Added: June 29, 2008Views: 141,905Ratings: 557
Responses: 0Comments: 180 + Favorited: 575 times +
+ + + + + + +
+ Honors for this video (0) +
+
Loading...
+
+
+ + +
+ Sites linking to this video (5) +
+ +
+
+
Clicks
+
URL
+
+
+ +
7,590
+ +
+
5,955
+ +
+
5,763
+ +
+
4,691
+ +
+
3,874
+ +
+
+ +
+
+
+ + + +
+
+
+ +
+ +
+ + +
+
+
+ Channel Icon +
+ + +
+ From: Odo987
+ Joined: 2 years ago
+ Videos: 5 +
+
+
+
+ + + +
+
+ +
+
+ Added: + June 29, 2008 + (More info) +
+ +
+ Create OR, AND and XOR gates out of dominoes. ... +
+
+ +
+
+ Added: + June 29, 2008 + (Less info) +
+ +
+ Create OR, AND and XOR gates out of dominoes. Replacement for TTL and CMOS.

http://neil.fraser.name/ +
+ +
+ Category:  + Science & Technology +
+ +
+
+ Tags:  +
+
+ Domino  + Boolean  + Logic  + And  + Or  + Xor  +
+
+
+ + +
+
+ + +
+
+
+ + +
+
+ +
+
+   +
+
+
+
+ +
+
+
+
+ +
Loading...
+ +
+ + + +
+ + + +
+ + + +
+ + + + + +
+
+ More From: Odo987 +
+
Loading...
+
+
+ +
+ + +
+
+
+ + Play All + + + Stop Autoplaying + + | + + Play Next + +
+ + + QuickList (0) + + +
+
+ + + +
+ +
+ 1 +
+
+ +
+
+
+ +
+
+ +
+
+ +
+ + + +
+
+ + + + + + +
+
+ +
+ + Clear | + Save + +
+ +
+ +
+ + + + +
+ Promoted Videos +
+
+ + + +
+ +
Happy Tree Friends - Ka-Pow! Coming Soon
+ + + + + 00:36 + + +
+ +
+
+ + + +
+ +
FANalysts Intro Video - Fantasy Football Advice and Analysis
+ + + + + 01:54 + +
+ FANALYSTS
+
+
+ +
+
+ + + +
+ +
Inside the Tent: Pedicab Confessions
+ + + + + 02:26 + + +
+ +
+
+ + + +
+ +
The House Bunny CosmoGIRL Shoot
+ + + + + 01:58 + +
+ CosmoGirl
+
+
+ +
+
+
+
+ + + +
+ + + + +
+ + + + +
+ + + + + + + + \ No newline at end of file diff --git a/test/test_suite.rb b/test/test_suite.rb index 6efa044..26c595d 100644 --- a/test/test_suite.rb +++ b/test/test_suite.rb @@ -24,6 +24,7 @@ require 'test/uri_utilities_test' require 'test/vimeo_test' require 'test/website_test' require 'test/youporn_test' +require 'test/youtube_test' # Vendor tests -#require 'vendor/ruby-progressbar/test' +require 'vendor/ruby-progressbar/test' diff --git a/test/youtube_test.rb b/test/youtube_test.rb new file mode 100644 index 0000000..08e8030 --- /dev/null +++ b/test/youtube_test.rb @@ -0,0 +1,64 @@ +# +# Copyright Michael Orlitzky +# +# http://michael.orlitzky.com/ +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# http://www.fsf.org/licensing/licenses/gpl.html +# + +require 'test/unit' +require 'src/websites/youtube' + +class YoutubeTest < Test::Unit::TestCase + + def test_owns_youtube_urls + assert(Youtube.owns_url?('http://www.youtube.com/watch?v=SudixyugiX4')) + assert(Youtube.owns_url?('http://www.youtube.com/watch?v=SudixyugiX4&hl=en')) + assert(Youtube.owns_url?('http://youtube.com/watch?v=SudixyugiX4&hl=en')) + assert(Youtube.owns_url?('http://www.youtube.com/v/SudixyugiX4')) + assert(Youtube.owns_url?('http://www.youtube.com/v/SudixyugiX4&hl=en')) + assert(Youtube.owns_url?('http://youtube.com/v/SudixyugiX4&hl=en')) + end + + + def test_doesnt_own_redtube_urls + assert(!Youtube.owns_url?('http://www.redtube.com/6807')) + assert(!Youtube.owns_url?('www.redtube.com/6807')) + assert(!Youtube.owns_url?('http://redtube.com/6807')) + assert(!Youtube.owns_url?('redtube.com/6807')) + end + + + def test_parse_video_id + yt = Youtube.new() + expected_result = 'SudixyugiX4' + actual_result = yt.send('parse_video_id', 'http://www.youtube.com/watch?v=SudixyugiX4') + assert_equal(expected_result, actual_result) + end + + + def test_parse_t_parameter + yt = Youtube.new() + + page_data = nil + + File.open('test/fixtures/youtube/SudixyugiX4.html') do |f| + page_data = f.read + end + + expected_result = 'OEgsToPDskLQUAntWWpzhEMhBMlgqHdo' + actual_result = yt.send('parse_t_parameter', page_data) + assert_equal(expected_result, actual_result) + end + +end -- 2.43.2