From: mjo Date: Fri, 29 Aug 2008 21:37:30 +0000 (-0400) Subject: Added the ability to download Vimeo videos. X-Git-Url: http://gitweb.michael.orlitzky.com/?p=dead%2Fwhatever-dl.git;a=commitdiff_plain;h=00f5e6f475d07afa862c8f33854d8ed52185ea81 Added the ability to download Vimeo videos. Created a Vimeo class. Created a bunch of tests for the new Vimeo class. Added a fixture for the Vimeo tests: an XML file containing the video data. Added a GPLv3 notice to the top of any file which did not contain one. --- diff --git a/src/websites/vimeo.rb b/src/websites/vimeo.rb new file mode 100644 index 0000000..3836df1 --- /dev/null +++ b/src/websites/vimeo.rb @@ -0,0 +1,125 @@ +# +# Copyright Michael Orlitzky +# +# http://michael.orlitzky.com/ +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# http://www.fsf.org/licensing/licenses/gpl.html +# + +require 'src/website' + +# Needed to download the video XML file, which is in turn +# needed because it contains the junk we need to construct +# the video URL. +require 'net/http' +require 'uri' + + +class Vimeo < Website + + VALID_VIMEO_URL_REGEX = /^(http:\/\/)?(www\.)?vimeo\.com\/(moogaloop\.swf\?clip_id=)?(\d+)/ + + def self.owns_url?(url) + return url =~ VALID_VIMEO_URL_REGEX + end + + + def get_video_url(url) + # First, figure out the video id from the URL. + # Then, use the video id to construct the video details URL. + # Get the video details page, and parse the resulting XML for + # the junk we need to construct the video URL. Note that the file + # URL given in the XML is *not* valid. + video_id = parse_video_id(url) + details_url = "http://www.vimeo.com/moogaloop/load/clip:#{video_id}/local" + details_data = get_page_data(details_url) + + + request_signature = parse_request_signature(details_data) + request_signature_expires = parse_request_signature_expires(details_data) + quality = parse_quality(details_data) + + # Being slightly explicit about what we're doing here... + video_url = "http://www.vimeo.com/moogaloop/play/clip:#{video_id}/#{request_signature}/#{request_signature_expires}/?q=#{quality}" + + return video_url + end + + + protected; + + def parse_video_id(url) + video_id = nil + + # First, try to get the video id from the end of the URL. + video_id_regex = /\/(\d+)$/ + matches = video_id_regex.match(url) + + if matches.nil? + # If that didn't work, the URL must be of the clip_id= form. + video_id_regex = /clip_id\=(\d+)/ + matches = video_id_regex.match(url) + video_id = matches[1] if not (matches.nil? || matches.length < 1) + else + video_id = matches[1] if not (matches.nil? || matches.length < 1) + end + + return video_id + end + + + def parse_request_signature(page_data) + # It's XML. + rs_regex = /(.*?)<\/request_signature>/ + matches = rs_regex.match(page_data) + request_signature = matches[1] if not (matches.nil? || matches.length < 1) + + return request_signature + end + + + def parse_request_signature_expires(page_data) + rse_regex = /(.*?)<\/request_signature_expires>/ + matches = rse_regex.match(page_data) + request_signature_expires = matches[1] if not (matches.nil? || matches.length < 1) + + return request_signature_expires + end + + + def parse_quality(page_data) + quality_regex = /([01])<\isHD>/ + matches = quality_regex.match(page_data) + is_hd = matches[1] if not (matches.nil? || matches.length < 1) + + if is_hd == '1' then + # High-definition + return "hd" + else + # Standard-definition + return "sd" + end + end + + + def get_page_data(url) + uri = URI.parse(url) + + response = Net::HTTP.start(uri.host, uri.port) do |http| + http.get(uri.path) + end + + return response.body + end + +end diff --git a/test/fixtures/vimeo/details_data-1561578.xml b/test/fixtures/vimeo/details_data-1561578.xml new file mode 100644 index 0000000..eeef6a4 --- /dev/null +++ b/test/fixtures/vimeo/details_data-1561578.xml @@ -0,0 +1,53 @@ + + + 1220036679 + 0 + + + cache + + + www.vimeo.com + 1 + 1 + 0 + 01AAEA + 1 + 0 + 0 + default + + + 1220036679 + 0 + db66311d431bedb4ba997a7b1d1946e2 + 1220068800 + diff --git a/test/howcast_test.rb b/test/howcast_test.rb index 5aa0e62..a76c725 100644 --- a/test/howcast_test.rb +++ b/test/howcast_test.rb @@ -1,11 +1,29 @@ +# +# Copyright Michael Orlitzky +# +# http://michael.orlitzky.com/ +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# http://www.fsf.org/licensing/licenses/gpl.html +# + +require 'test/unit' +require 'src/websites/howcast' + # Unit tests for the Howcast class. # This class is easy because get_video_url is # just a string interpolation (we don't have to # test that). -require 'test/unit' -require 'src/websites/howcast' - class HowcastTest < Test::Unit::TestCase def test_owns_howcast_urls diff --git a/test/infoq_remote_test.rb b/test/infoq_remote_test.rb index 7700d6b..35ecc15 100644 --- a/test/infoq_remote_test.rb +++ b/test/infoq_remote_test.rb @@ -1,9 +1,27 @@ -# Remote Infoq tests. Actually hit their website -# and attempt to parse the data returned. +# +# Copyright Michael Orlitzky +# +# http://michael.orlitzky.com/ +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# http://www.fsf.org/licensing/licenses/gpl.html +# require 'test/unit' require 'src/websites/infoq' +# Remote Infoq tests. Actually hit their website +# and attempt to parse the data returned. + class InfoqRemoteTest < Test::Unit::TestCase def test_get_page_data diff --git a/test/infoq_test.rb b/test/infoq_test.rb index 51d1a3f..e34f501 100644 --- a/test/infoq_test.rb +++ b/test/infoq_test.rb @@ -1,9 +1,27 @@ -# Unit tests for the Infoq class. Basically just checking -# the results of parse_video_url for now. +# +# Copyright Michael Orlitzky +# +# http://michael.orlitzky.com/ +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# http://www.fsf.org/licensing/licenses/gpl.html +# require 'test/unit' require 'src/websites/infoq' +# Unit tests for the Infoq class. Basically just checking +# the results of parse_video_url for now. + class InfoqTest < Test::Unit::TestCase def test_owns_infoq_urls diff --git a/test/redtube_test.rb b/test/redtube_test.rb index a929f61..b6cae0f 100644 --- a/test/redtube_test.rb +++ b/test/redtube_test.rb @@ -1,9 +1,27 @@ -# Unit tests for the Redtube class. Basically just checking -# the results of get_video_url for known ids. +# +# Copyright Michael Orlitzky +# +# http://michael.orlitzky.com/ +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# http://www.fsf.org/licensing/licenses/gpl.html +# require 'test/unit' require 'src/websites/redtube' +# Unit tests for the Redtube class. Basically just checking +# the results of get_video_url for known ids. + class RedtubeTest < Test::Unit::TestCase def test_owns_redtube_urls diff --git a/test/remote_test_suite.rb b/test/remote_test_suite.rb index d4085d9..4a20e62 100644 --- a/test/remote_test_suite.rb +++ b/test/remote_test_suite.rb @@ -1,2 +1,21 @@ +# +# Copyright Michael Orlitzky +# +# http://michael.orlitzky.com/ +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# http://www.fsf.org/licensing/licenses/gpl.html +# + require 'test/infoq_remote_test' +require 'test/vimeo_remote_test' require 'test/youporn_remote_test' diff --git a/test/test_suite.rb b/test/test_suite.rb index 6150e08..70fec3a 100644 --- a/test/test_suite.rb +++ b/test/test_suite.rb @@ -1,6 +1,25 @@ +# +# Copyright Michael Orlitzky +# +# http://michael.orlitzky.com/ +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# http://www.fsf.org/licensing/licenses/gpl.html +# + require 'test/howcast_test' require 'test/infoq_test' require 'test/redtube_test' require 'test/veoh_test' +require 'test/vimeo_test' require 'test/website_test' require 'test/youporn_test' diff --git a/test/veoh_test.rb b/test/veoh_test.rb index f104d3a..1a53e43 100644 --- a/test/veoh_test.rb +++ b/test/veoh_test.rb @@ -1,3 +1,21 @@ +# +# Copyright Michael Orlitzky +# +# http://michael.orlitzky.com/ +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# http://www.fsf.org/licensing/licenses/gpl.html +# + require 'test/unit' require 'src/websites/veoh' diff --git a/test/vimeo_test.rb b/test/vimeo_test.rb new file mode 100644 index 0000000..6578b8c --- /dev/null +++ b/test/vimeo_test.rb @@ -0,0 +1,123 @@ +# +# Copyright Michael Orlitzky +# +# http://michael.orlitzky.com/ +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# http://www.fsf.org/licensing/licenses/gpl.html +# + +require 'test/unit' +require 'src/websites/vimeo' + +# There are no remote tests for Vimeo because they use +# an expiration timestamp to change certain values in the +# URL. Obviously, these tests wouldn't pass after the +# timestamp expired. +class VimeoTest < Test::Unit::TestCase + + def test_parse_standard_video_id + v = Vimeo.new + + # First form, with the id at the end. + video_id = v.send('parse_video_id', 'http://www.vimeo.com/1561578') + assert_equal('1561578', video_id) + end + + + def test_parse_swf_video_id + v = Vimeo.new + + # Second, clip_id= form. + video_id = v.send('parse_video_id', 'http://www.vimeo.com/moogaloop.swf?clip_id=1561578&server=www.vimeo.com&show_title=1&show_byline=1&show_portrait=0&color=&fullscreen=1') + assert_equal('1561578', video_id) + end + + + def test_parse_request_signature + v = Vimeo.new + + details_data = nil + + File.open('test/fixtures/vimeo/details_data-1561578.xml') do |f| + details_data = f.read + end + + actual_result = v.send('parse_request_signature', details_data) + expected_result = 'db66311d431bedb4ba997a7b1d1946e2' + + assert_equal(expected_result, actual_result) + end + + + def test_parse_request_signature_expires + v = Vimeo.new + + details_data = nil + + File.open('test/fixtures/vimeo/details_data-1561578.xml') do |f| + details_data = f.read + end + + actual_result = v.send('parse_request_signature_expires', details_data) + expected_result = '1220068800' + + assert_equal(expected_result, actual_result) + end + + + def test_parse_quality + v = Vimeo.new + + details_data = nil + + File.open('test/fixtures/vimeo/details_data-1561578.xml') do |f| + details_data = f.read + end + + actual_result = v.send('parse_quality', details_data) + expected_result = 'sd' + + assert_equal(expected_result, actual_result) + end + + + def test_owns_standard_vimeo_url + assert(Vimeo.owns_url?('http://www.vimeo.com/1561578')) + end + + + def test_owns_swf_video_url + assert(Vimeo.owns_url?('http://www.vimeo.com/moogaloop.swf?clip_id=1561578&server=www.vimeo.com&show_title=1&show_byline=1&show_portrait=0&color=&fullscreen=1')) + end + + + def test_doesnt_own_howcast_urls + assert(!Vimeo.owns_url?('http://www.howcast.com/videos/6807-2twr')) + assert(!Vimeo.owns_url?('www.howcast.com/videos/6807-2dgfdg')) + assert(!Vimeo.owns_url?('http://howcast.com/videos/6807-cse')) + assert(!Vimeo.owns_url?('howcast.com/videos/6807-asdgasd')) + end + + + def test_doesnt_own_redtube_urls + assert(!Vimeo.owns_url?('http://www.redtube.com/6807')) + assert(!Vimeo.owns_url?('www.redtube.com/6807')) + assert(!Vimeo.owns_url?('http://redtube.com/6807')) + assert(!Vimeo.owns_url?('redtube.com/6807')) + end + + def test_doesnt_own_misc_urls + assert(!Vimeo.owns_url?('http://www.howcast.com/abc')) + end + +end diff --git a/test/website_test.rb b/test/website_test.rb index e523b1f..e140676 100644 --- a/test/website_test.rb +++ b/test/website_test.rb @@ -1,6 +1,25 @@ -# Tests common to all websites. +# +# Copyright Michael Orlitzky +# +# http://michael.orlitzky.com/ +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# http://www.fsf.org/licensing/licenses/gpl.html +# + require 'test/unit' +# Tests common to all websites. + # All of the website classes are located in one # directory, so we can 'require' them automatically. Dir.glob('src/websites/*.rb').each do |r| diff --git a/test/youporn_remote_test.rb b/test/youporn_remote_test.rb index 1a8dda0..6dd26d7 100644 --- a/test/youporn_remote_test.rb +++ b/test/youporn_remote_test.rb @@ -1,9 +1,27 @@ -# Remote Youporn tests. Actually hit their website -# and attempt to parse the data returned. +# +# Copyright Michael Orlitzky +# +# http://michael.orlitzky.com/ +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# http://www.fsf.org/licensing/licenses/gpl.html +# require 'test/unit' require 'src/websites/youporn' +# Remote Youporn tests. Actually hit their website +# and attempt to parse the data returned. + class YoupornRemoteTest < Test::Unit::TestCase def test_get_page_data diff --git a/test/youporn_test.rb b/test/youporn_test.rb index 59c14d5..d06743e 100644 --- a/test/youporn_test.rb +++ b/test/youporn_test.rb @@ -1,9 +1,27 @@ -# Unit tests for the Youporn class. Basically just checking -# the results of parse_video_url for now. +# +# Copyright Michael Orlitzky +# +# http://michael.orlitzky.com/ +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# http://www.fsf.org/licensing/licenses/gpl.html +# require 'test/unit' require 'src/websites/youporn' +# Unit tests for the Youporn class. Basically just checking +# the results of parse_video_url for now. + class YoupornTest < Test::Unit::TestCase def test_parse_video_url