release 2014.01.07.2

On Windows and other systems, other encodings would break when trying to encode non-ASCII characters.
Simply use UTF-8, like every sane system.

README.md

@@ -193,7 +193,9 @@ which means you can modify it, redistribute it or use it however you like.
                                processed files are overwritten by default
     --embed-subs               embed subtitles in the video (only for mp4
                                videos)
-    --add-metadata             add metadata to the files
+    --add-metadata             write metadata to the video file
+    --xattrs                   write metadata to the video file's xattrs (using
+                               dublin core and xdg standards)
 
 # CONFIGURATION
 

devscripts/bash-completion.in

@@ -6,7 +6,7 @@ __youtube_dl()
     prev="${COMP_WORDS[COMP_CWORD-1]}"
     opts="{{flags}}"
     keywords=":ytfavorites :ytrecommended :ytsubscriptions :ytwatchlater :ythistory"
-    fileopts="-a|--batch-file|--download-archive|--cookies"
+    fileopts="-a|--batch-file|--download-archive|--cookies|--load-info"
     diropts="--cache-dir"
 
     if [[ ${prev} =~ ${fileopts} ]]; then

setup.py

@@ -71,7 +71,10 @@ setup(
     author_email='ytdl@yt-dl.org',
     maintainer='Philipp Hagemeister',
     maintainer_email='phihag@phihag.de',
-    packages=['youtube_dl', 'youtube_dl.extractor', 'youtube_dl.downloader'],
+    packages=[
+        'youtube_dl',
+        'youtube_dl.extractor', 'youtube_dl.downloader',
+        'youtube_dl.postprocessor'],
 
     # Provokes warning on most systems (why?!)
     # test_suite = 'nose.collector',

test/test_all_urls.py

@@ -113,6 +113,8 @@ class TestAllURLsMatching(unittest.TestCase):
     def test_vimeo_matching(self):
         self.assertMatch('http://vimeo.com/channels/tributes', ['vimeo:channel'])
         self.assertMatch('http://vimeo.com/user7108434', ['vimeo:user'])
+        self.assertMatch('http://vimeo.com/user7108434/videos', ['vimeo:user'])
+        self.assertMatch('https://vimeo.com/user21297594/review/75524534/3c257a1b5d', ['vimeo:review'])
 
     # https://github.com/rg3/youtube-dl/issues/1930
     def test_soundcloud_not_matching_sets(self):

youtube_dl/YoutubeDL.py

@@ -55,7 +55,7 @@ from .utils import (
 )
 from .extractor import get_info_extractor, gen_extractors
 from .downloader import get_suitable_downloader
-from .PostProcessor import FFmpegMergerPP
+from .postprocessor import FFmpegMergerPP
 from .version import __version__
 
 
@@ -1062,7 +1062,9 @@ class YoutubeDL(object):
                 res += '%4dk ' % fdict['tbr']
             if (fdict.get('vcodec') is not None and
                     fdict.get('vcodec') != 'none'):
-                res += '%-5s@' % fdict['vcodec']
+                res += '%-5s' % fdict['vcodec']
+                if fdict.get('vbr') is not None:
+                    res += '@'
             elif fdict.get('vbr') is not None and fdict.get('abr') is not None:
                 res += 'video@'
             if fdict.get('vbr') is not None:

youtube_dl/__init__.py

@@ -38,6 +38,7 @@ __authors__  = (
     'Takuya Tsuchida',
     'Sergey M.',
     'Michael Orlitzky',
+    'Chris Gahan',
 )
 
 __license__ = 'Public Domain'
@@ -74,11 +75,12 @@ from .FileDownloader import (
 from .extractor import gen_extractors
 from .version import __version__
 from .YoutubeDL import YoutubeDL
-from .PostProcessor import (
+from .postprocessor import (
     FFmpegMetadataPP,
     FFmpegVideoConvertor,
     FFmpegExtractAudioPP,
     FFmpegEmbedSubtitlePP,
+    XAttrMetadataPP,
 )
 
 
@@ -415,7 +417,9 @@ def parseOpts(overrideArguments=None):
     postproc.add_option('--embed-subs', action='store_true', dest='embedsubtitles', default=False,
             help='embed subtitles in the video (only for mp4 videos)')
     postproc.add_option('--add-metadata', action='store_true', dest='addmetadata', default=False,
-            help='add metadata to the files')
+            help='write metadata to the video file')
+    postproc.add_option('--xattrs', action='store_true', dest='xattrs', default=False,
+            help='write metadata to the video file\'s xattrs (using dublin core and xdg standards)')
 
 
     parser.add_option_group(general)
@@ -523,7 +527,7 @@ def _real_main(argv=None):
     all_urls = batchurls + args
     all_urls = [url.strip() for url in all_urls]
     _enc = preferredencoding()
-    all_urls = [url.decode(_enc, 'ignore') if isinstance(url, bytes) else url]
+    all_urls = [url.decode(_enc, 'ignore') if isinstance(url, bytes) else url for url in all_urls]
 
     extractors = gen_extractors()
 
@@ -717,6 +721,8 @@ def _real_main(argv=None):
             ydl.add_post_processor(FFmpegVideoConvertor(preferedformat=opts.recodevideo))
         if opts.embedsubtitles:
             ydl.add_post_processor(FFmpegEmbedSubtitlePP(subtitlesformat=opts.subtitlesformat))
+        if opts.xattrs:
+            ydl.add_post_processor(XAttrMetadataPP())
 
         # Update version
         if opts.update_self:

youtube_dl/extractor/__init__.py

@@ -112,6 +112,7 @@ from .metacafe import MetacafeIE
 from .metacritic import MetacriticIE
 from .mit import TechTVMITIE, MITIE
 from .mixcloud import MixcloudIE
+from .mpora import MporaIE
 from .mofosex import MofosexIE
 from .mtv import MTVIE
 from .muzu import MuzuTVIE
@@ -199,6 +200,7 @@ from .vimeo import (
     VimeoUserIE,
     VimeoAlbumIE,
     VimeoGroupsIE,
+    VimeoReviewIE,
 )
 from .vine import VineIE
 from .viki import VikiIE

youtube_dl/extractor/bliptv.py

@@ -8,10 +8,8 @@ import socket
 from .common import InfoExtractor
 from ..utils import (
     compat_http_client,
-    compat_parse_qs,
     compat_str,
     compat_urllib_error,
-    compat_urllib_parse_urlparse,
     compat_urllib_request,
 
     ExtractorError,

youtube_dl/extractor/brightcove.py

@@ -1,4 +1,5 @@
 # encoding: utf-8
+from __future__ import unicode_literals
 
 import re
 import json
@@ -13,6 +14,7 @@ from ..utils import (
     compat_urllib_request,
 
     ExtractorError,
+    unsmuggle_url,
 )
 
 
@@ -24,47 +26,47 @@ class BrightcoveIE(InfoExtractor):
     _TESTS = [
         {
             # From http://www.8tv.cat/8aldia/videos/xavier-sala-i-martin-aquesta-tarda-a-8-al-dia/
-            u'url': u'http://c.brightcove.com/services/viewer/htmlFederated?playerID=1654948606001&flashID=myExperience&%40videoPlayer=2371591881001',
-            u'file': u'2371591881001.mp4',
-            u'md5': u'5423e113865d26e40624dce2e4b45d95',
-            u'note': u'Test Brightcove downloads and detection in GenericIE',
-            u'info_dict': {
-                u'title': u'Xavier Sala i Martín: “Un banc que no presta és un banc zombi que no serveix per a res”',
-                u'uploader': u'8TV',
-                u'description': u'md5:a950cc4285c43e44d763d036710cd9cd',
+            'url': 'http://c.brightcove.com/services/viewer/htmlFederated?playerID=1654948606001&flashID=myExperience&%40videoPlayer=2371591881001',
+            'file': '2371591881001.mp4',
+            'md5': '5423e113865d26e40624dce2e4b45d95',
+            'note': 'Test Brightcove downloads and detection in GenericIE',
+            'info_dict': {
+                'title': 'Xavier Sala i Martín: “Un banc que no presta és un banc zombi que no serveix per a res”',
+                'uploader': '8TV',
+                'description': 'md5:a950cc4285c43e44d763d036710cd9cd',
             }
         },
         {
             # From http://medianetwork.oracle.com/video/player/1785452137001
-            u'url': u'http://c.brightcove.com/services/viewer/htmlFederated?playerID=1217746023001&flashID=myPlayer&%40videoPlayer=1785452137001',
-            u'file': u'1785452137001.flv',
-            u'info_dict': {
-                u'title': u'JVMLS 2012: Arrays 2.0 - Opportunities and Challenges',
-                u'description': u'John Rose speaks at the JVM Language Summit, August 1, 2012.',
-                u'uploader': u'Oracle',
+            'url': 'http://c.brightcove.com/services/viewer/htmlFederated?playerID=1217746023001&flashID=myPlayer&%40videoPlayer=1785452137001',
+            'file': '1785452137001.flv',
+            'info_dict': {
+                'title': 'JVMLS 2012: Arrays 2.0 - Opportunities and Challenges',
+                'description': 'John Rose speaks at the JVM Language Summit, August 1, 2012.',
+                'uploader': 'Oracle',
             },
         },
         {
             # From http://mashable.com/2013/10/26/thermoelectric-bracelet-lets-you-control-your-body-temperature/
-            u'url': u'http://c.brightcove.com/services/viewer/federated_f9?&playerID=1265504713001&publisherID=AQ%7E%7E%2CAAABBzUwv1E%7E%2CxP-xFHVUstiMFlNYfvF4G9yFnNaqCw_9&videoID=2750934548001',
-            u'info_dict': {
-                u'id': u'2750934548001',
-                u'ext': u'mp4',
-                u'title': u'This Bracelet Acts as a Personal Thermostat',
-                u'description': u'md5:547b78c64f4112766ccf4e151c20b6a0',
-                u'uploader': u'Mashable',
+            'url': 'http://c.brightcove.com/services/viewer/federated_f9?&playerID=1265504713001&publisherID=AQ%7E%7E%2CAAABBzUwv1E%7E%2CxP-xFHVUstiMFlNYfvF4G9yFnNaqCw_9&videoID=2750934548001',
+            'info_dict': {
+                'id': '2750934548001',
+                'ext': 'mp4',
+                'title': 'This Bracelet Acts as a Personal Thermostat',
+                'description': 'md5:547b78c64f4112766ccf4e151c20b6a0',
+                'uploader': 'Mashable',
             },
         },
         {
             # test that the default referer works
             # from http://national.ballet.ca/interact/video/Lost_in_Motion_II/
-            u'url': u'http://link.brightcove.com/services/player/bcpid756015033001?bckey=AQ~~,AAAApYJi_Ck~,GxhXCegT1Dp39ilhXuxMJxasUhVNZiil&bctid=2878862109001',
-            u'info_dict': {
-                u'id': u'2878862109001',
-                u'ext': u'mp4',
-                u'title': u'Lost in Motion II',
-                u'description': u'md5:363109c02998fee92ec02211bd8000df',
-                u'uploader': u'National Ballet of Canada',
+            'url': 'http://link.brightcove.com/services/player/bcpid756015033001?bckey=AQ~~,AAAApYJi_Ck~,GxhXCegT1Dp39ilhXuxMJxasUhVNZiil&bctid=2878862109001',
+            'info_dict': {
+                'id': '2878862109001',
+                'ext': 'mp4',
+                'title': 'Lost in Motion II',
+                'description': 'md5:363109c02998fee92ec02211bd8000df',
+                'uploader': 'National Ballet of Canada',
             },
         },
     ]
@@ -80,10 +82,10 @@ class BrightcoveIE(InfoExtractor):
         object_str = re.sub(r'(<param name="[^"]+" value="[^"]+")>',
                             lambda m: m.group(1) + '/>', object_str)
         # Fix up some stupid XML, see https://github.com/rg3/youtube-dl/issues/1608
-        object_str = object_str.replace(u'<--', u'<!--')
+        object_str = object_str.replace('<--', '<!--')
 
         object_doc = xml.etree.ElementTree.fromstring(object_str)
-        assert u'BrightcoveExperience' in object_doc.attrib['class']
+        assert 'BrightcoveExperience' in object_doc.attrib['class']
         params = {'flashID': object_doc.attrib['id'],
                   'playerID': find_xpath_attr(object_doc, './param', 'name', 'playerID').attrib['value'],
                   }
@@ -120,6 +122,8 @@ class BrightcoveIE(InfoExtractor):
             return None
 
     def _real_extract(self, url):
+        url, smuggled_data = unsmuggle_url(url, {})
+
         # Change the 'videoId' and others field to '@videoPlayer'
         url = re.sub(r'(?<=[?&])(videoI(d|D)|bctid)', '%40videoPlayer', url)
         # Change bckey (used by bcove.me urls) to playerKey
@@ -130,9 +134,10 @@ class BrightcoveIE(InfoExtractor):
 
         videoPlayer = query.get('@videoPlayer')
         if videoPlayer:
-            return self._get_video_info(videoPlayer[0], query_str, query,
-                # We set the original url as the default 'Referer' header
-                referer=url)
+            # We set the original url as the default 'Referer' header
+            referer = smuggled_data.get('Referer', url)
+            return self._get_video_info(
+                videoPlayer[0], query_str, query, referer=referer)
         else:
             player_key = query['playerKey']
             return self._get_playlist_info(player_key[0])
@@ -156,11 +161,11 @@ class BrightcoveIE(InfoExtractor):
 
     def _get_playlist_info(self, player_key):
         playlist_info = self._download_webpage(self._PLAYLIST_URL_TEMPLATE % player_key,
-                                               player_key, u'Downloading playlist information')
+                                               player_key, 'Downloading playlist information')
 
         json_data = json.loads(playlist_info)
         if 'videoList' not in json_data:
-            raise ExtractorError(u'Empty playlist')
+            raise ExtractorError('Empty playlist')
         playlist_info = json_data['videoList']
         videos = [self._extract_video_info(video_info) for video_info in playlist_info['mediaCollectionDTO']['videoDTOs']]
 
@@ -189,5 +194,5 @@ class BrightcoveIE(InfoExtractor):
                 'url': video_info['FLVFullLengthURL'],
             })
         else:
-            raise ExtractorError(u'Unable to extract video url for %s' % info['id'])
+            raise ExtractorError('Unable to extract video url for %s' % info['id'])
         return info

youtube_dl/extractor/common.py

@@ -73,6 +73,10 @@ class InfoExtractor(object):
                                  by this field.
                                  -1 for default (order by other properties),
                                  -2 or smaller for less than default.
+                    * quality    Order number of the video quality of this
+                                 format, irrespective of the file format.
+                                 -1 for default (order by other properties),
+                                 -2 or smaller for less than default.
     url:            Final video URL.
     ext:            Video filename extension.
     format:         The video format, defaults to ext (used for --get-format)
@@ -483,6 +487,7 @@ class InfoExtractor(object):
 
             return (
                 preference,
+                f.get('quality') if f.get('quality') is not None else -1,
                 f.get('height') if f.get('height') is not None else -1,
                 f.get('width') if f.get('width') is not None else -1,
                 ext_preference,

youtube_dl/extractor/generic.py

@@ -1,9 +1,12 @@
 # encoding: utf-8
 
+from __future__ import unicode_literals
+
 import os
 import re
 
 from .common import InfoExtractor
+from .youtube import YoutubeIE
 from ..utils import (
     compat_urllib_error,
     compat_urllib_parse,
@@ -22,78 +25,78 @@ from .ooyala import OoyalaIE
 
 
 class GenericIE(InfoExtractor):
-    IE_DESC = u'Generic downloader that works on some sites'
+    IE_DESC = 'Generic downloader that works on some sites'
     _VALID_URL = r'.*'
-    IE_NAME = u'generic'
+    IE_NAME = 'generic'
     _TESTS = [
         {
-            u'url': u'http://www.hodiho.fr/2013/02/regis-plante-sa-jeep.html',
-            u'file': u'13601338388002.mp4',
-            u'md5': u'6e15c93721d7ec9e9ca3fdbf07982cfd',
-            u'info_dict': {
-                u"uploader": u"www.hodiho.fr",
-                u"title": u"R\u00e9gis plante sa Jeep"
+            'url': 'http://www.hodiho.fr/2013/02/regis-plante-sa-jeep.html',
+            'file': '13601338388002.mp4',
+            'md5': '6e15c93721d7ec9e9ca3fdbf07982cfd',
+            'info_dict': {
+                'uploader': 'www.hodiho.fr',
+                'title': 'R\u00e9gis plante sa Jeep',
             }
         },
         # embedded vimeo video
         {
-            u'add_ie': ['Vimeo'],
-            u'url': u'http://skillsmatter.com/podcast/home/move-semanticsperfect-forwarding-and-rvalue-references',
-            u'file': u'22444065.mp4',
-            u'md5': u'2903896e23df39722c33f015af0666e2',
-            u'info_dict': {
-                u'title': u'ACCU 2011: Move Semantics,Perfect Forwarding, and Rvalue references- Scott Meyers- 13/04/2011',
-                u"uploader_id": u"skillsmatter",
-                u"uploader": u"Skills Matter",
+            'add_ie': ['Vimeo'],
+            'url': 'http://skillsmatter.com/podcast/home/move-semanticsperfect-forwarding-and-rvalue-references',
+            'file': '22444065.mp4',
+            'md5': '2903896e23df39722c33f015af0666e2',
+            'info_dict': {
+                'title': 'ACCU 2011: Move Semantics,Perfect Forwarding, and Rvalue references- Scott Meyers- 13/04/2011',
+                'uploader_id': 'skillsmatter',
+                'uploader': 'Skills Matter',
             }
         },
         # bandcamp page with custom domain
         {
-            u'add_ie': ['Bandcamp'],
-            u'url': u'http://bronyrock.com/track/the-pony-mash',
-            u'file': u'3235767654.mp3',
-            u'info_dict': {
-                u'title': u'The Pony Mash',
-                u'uploader': u'M_Pallante',
+            'add_ie': ['Bandcamp'],
+            'url': 'http://bronyrock.com/track/the-pony-mash',
+            'file': '3235767654.mp3',
+            'info_dict': {
+                'title': 'The Pony Mash',
+                'uploader': 'M_Pallante',
             },
-            u'skip': u'There is a limit of 200 free downloads / month for the test song',
+            'skip': 'There is a limit of 200 free downloads / month for the test song',
         },
         # embedded brightcove video
         # it also tests brightcove videos that need to set the 'Referer' in the
         # http requests
         {
-            u'add_ie': ['Brightcove'],
-            u'url': u'http://www.bfmtv.com/video/bfmbusiness/cours-bourse/cours-bourse-l-analyse-technique-154522/',
-            u'info_dict': {
-                u'id': u'2765128793001',
-                u'ext': u'mp4',
-                u'title': u'Le cours de bourse : l’analyse technique',
-                u'description': u'md5:7e9ad046e968cb2d1114004aba466fd9',
-                u'uploader': u'BFM BUSINESS',
+            'add_ie': ['Brightcove'],
+            'url': 'http://www.bfmtv.com/video/bfmbusiness/cours-bourse/cours-bourse-l-analyse-technique-154522/',
+            'info_dict': {
+                'id': '2765128793001',
+                'ext': 'mp4',
+                'title': 'Le cours de bourse : l’analyse technique',
+                'description': 'md5:7e9ad046e968cb2d1114004aba466fd9',
+                'uploader': 'BFM BUSINESS',
             },
-            u'params': {
-                u'skip_download': True,
+            'params': {
+                'skip_download': True,
             },
         },
         # Direct link to a video
         {
-            u'url': u'http://media.w3.org/2010/05/sintel/trailer.mp4',
-            u'file': u'trailer.mp4',
-            u'md5': u'67d406c2bcb6af27fa886f31aa934bbe',
-            u'info_dict': {
-                u'id': u'trailer',
-                u'title': u'trailer',
-                u'upload_date': u'20100513',
+            'url': 'http://media.w3.org/2010/05/sintel/trailer.mp4',
+            'file': 'trailer.mp4',
+            'md5': '67d406c2bcb6af27fa886f31aa934bbe',
+            'info_dict': {
+                'id': 'trailer',
+                'title': 'trailer',
+                'upload_date': '20100513',
             }
         },
         # ooyala video
         {
-            u'url': u'http://www.rollingstone.com/music/videos/norwegian-dj-cashmere-cat-goes-spartan-on-with-me-premiere-20131219',
-            u'md5': u'5644c6ca5d5782c1d0d350dad9bd840c',
-            u'info_dict': {
-                u'id': u'BwY2RxaTrTkslxOfcan0UCf0YqyvWysJ',
-                u'ext': u'mp4',
-                u'title': u'2cc213299525360.mov', #that's what we get
+            'url': 'http://www.rollingstone.com/music/videos/norwegian-dj-cashmere-cat-goes-spartan-on-with-me-premiere-20131219',
+            'md5': '5644c6ca5d5782c1d0d350dad9bd840c',
+            'info_dict': {
+                'id': 'BwY2RxaTrTkslxOfcan0UCf0YqyvWysJ',
+                'ext': 'mp4',
+                'title': '2cc213299525360.mov', #that's what we get
             },
         },
     ]
@@ -101,12 +104,12 @@ class GenericIE(InfoExtractor):
     def report_download_webpage(self, video_id):
         """Report webpage download."""
         if not self._downloader.params.get('test', False):
-            self._downloader.report_warning(u'Falling back on generic information extractor.')
+            self._downloader.report_warning('Falling back on generic information extractor.')
         super(GenericIE, self).report_download_webpage(video_id)
 
     def report_following_redirect(self, new_url):
         """Report information extraction."""
-        self._downloader.to_screen(u'[redirect] Following redirect to %s' % new_url)
+        self._downloader.to_screen('[redirect] Following redirect to %s' % new_url)
 
     def _send_head(self, url):
         """Check if it is a redirect, like url shorteners, in case return the new url."""
@@ -152,7 +155,7 @@ class GenericIE(InfoExtractor):
 
         response = opener.open(HEADRequest(url))
         if response is None:
-            raise ExtractorError(u'Invalid URL protocol')
+            raise ExtractorError('Invalid URL protocol')
         return response
 
     def _real_extract(self, url):
@@ -162,7 +165,7 @@ class GenericIE(InfoExtractor):
             return self.url_result('http://' + url)
         video_id = os.path.splitext(url.split('/')[-1])[0]
 
-        self.to_screen(u'%s: Requesting header' % video_id)
+        self.to_screen('%s: Requesting header' % video_id)
 
         try:
             response = self._send_head(url)
@@ -186,7 +189,7 @@ class GenericIE(InfoExtractor):
                     'formats': [{
                         'format_id': m.group('format_id'),
                         'url': url,
-                        'vcodec': u'none' if m.group('type') == 'audio' else None
+                        'vcodec': 'none' if m.group('type') == 'audio' else None
                     }],
                     'upload_date': upload_date,
                 }
@@ -200,7 +203,7 @@ class GenericIE(InfoExtractor):
         except ValueError:
             # since this is the last-resort InfoExtractor, if
             # this error is thrown, it'll be thrown here
-            raise ExtractorError(u'Failed to download URL: %s' % url)
+            raise ExtractorError('Failed to download URL: %s' % url)
 
         self.report_extraction(video_id)
 
@@ -211,18 +214,19 @@ class GenericIE(InfoExtractor):
         #   Video Title - Tagline | Site Name
         # and so on and so forth; it's just not practical
         video_title = self._html_search_regex(
-            r'(?s)<title>(.*?)</title>', webpage, u'video title',
-            default=u'video')
+            r'(?s)<title>(.*?)</title>', webpage, 'video title',
+            default='video')
 
         # video uploader is domain name
         video_uploader = self._search_regex(
-            r'^(?:https?://)?([^/]*)/.*', url, u'video uploader')
+            r'^(?:https?://)?([^/]*)/.*', url, 'video uploader')
 
         # Look for BrightCove:
         bc_url = BrightcoveIE._extract_brightcove_url(webpage)
         if bc_url is not None:
-            self.to_screen(u'Brightcove video detected.')
-            return self.url_result(bc_url, 'Brightcove')
+            self.to_screen('Brightcove video detected.')
+            surl = smuggle_url(bc_url, {'Referer': url})
+            return self.url_result(surl, 'Brightcove')
 
         # Look for embedded (iframe) Vimeo player
         mobj = re.search(
@@ -303,6 +307,11 @@ class GenericIE(InfoExtractor):
         if mobj is not None:
             return self.url_result(mobj.group(1), 'Aparat')
 
+        # Look for MPORA videos
+        mobj = re.search(r'<iframe .*?src="(http://mpora.com/videos/[^"]+)"', webpage)
+        if mobj is not None:
+            return self.url_result(mobj.group(1), 'Mpora')
+
         # Start with something easy: JW Player in SWFObject
         mobj = re.search(r'flashvars: [\'"](?:.*&)?file=(http[^\'"&]*)', webpage)
         if mobj is None:
@@ -328,23 +337,27 @@ class GenericIE(InfoExtractor):
             # HTML5 video
             mobj = re.search(r'<video[^<]*(?:>.*?<source.*?)? src="([^"]+)"', webpage, flags=re.DOTALL)
         if mobj is None:
-            raise ExtractorError(u'Unsupported URL: %s' % url)
+            raise ExtractorError('Unsupported URL: %s' % url)
 
         # It's possible that one of the regexes
         # matched, but returned an empty group:
         if mobj.group(1) is None:
-            raise ExtractorError(u'Did not find a valid video URL at %s' % url)
+            raise ExtractorError('Did not find a valid video URL at %s' % url)
 
         video_url = mobj.group(1)
         video_url = compat_urlparse.urljoin(url, video_url)
         video_id = compat_urllib_parse.unquote(os.path.basename(video_url))
 
+        # Sometimes, jwplayer extraction will result in a YouTube URL
+        if YoutubeIE.suitable(video_url):
+            return self.url_result(video_url, 'Youtube')
+
         # here's a fun little line of code for you:
         video_id = os.path.splitext(video_id)[0]
 
         return {
-            'id':       video_id,
-            'url':      video_url,
+            'id': video_id,
+            'url': video_url,
             'uploader': video_uploader,
-            'title':    video_title,
+            'title': video_title,
         }

youtube_dl/extractor/lynda.py

@@ -3,11 +3,12 @@ from __future__ import unicode_literals
 import re
 import json
 
+from .subtitles import SubtitlesInfoExtractor
 from .common import InfoExtractor
 from ..utils import ExtractorError
 
 
-class LyndaIE(InfoExtractor):
+class LyndaIE(SubtitlesInfoExtractor):
     IE_NAME = 'lynda'
     IE_DESC = 'lynda.com videos'
     _VALID_URL = r'https?://www\.lynda\.com/[^/]+/[^/]+/\d+/(\d+)-\d\.html'
@@ -16,12 +17,12 @@ class LyndaIE(InfoExtractor):
         'url': 'http://www.lynda.com/Bootstrap-tutorials/Using-exercise-files/110885/114408-4.html',
         'file': '114408.mp4',
         'md5': 'ecfc6862da89489161fb9cd5f5a6fac1',
-        u"info_dict": {
+        'info_dict': {
             'title': 'Using the exercise files',
             'duration': 68
         }
     }
-
+    
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group(1)
@@ -45,17 +46,59 @@ class LyndaIE(InfoExtractor):
                     'width': fmt['Width'],
                     'height': fmt['Height'],
                     'filesize': fmt['FileSize'],
-                    'format_id': fmt['Resolution']
+                    'format_id': str(fmt['Resolution'])
                     } for fmt in video_json['Formats']]
 
         self._sort_formats(formats)
-
+        
+        if self._downloader.params.get('listsubtitles', False):
+            self._list_available_subtitles(video_id, page)
+            return
+        
+        subtitles = self._fix_subtitles(self.extract_subtitles(video_id, page))
+        
         return {
             'id': video_id,
             'title': title,
             'duration': duration,
+            'subtitles': subtitles,
             'formats': formats
         }
+        
+    _TIMECODE_REGEX = r'\[(?P<timecode>\d+:\d+:\d+[\.,]\d+)\]'
+
+    def _fix_subtitles(self, subtitles):
+        if subtitles is None:
+            return subtitles  # subtitles not requested
+
+        fixed_subtitles = {}
+        for k, v in subtitles.items():
+            subs = json.loads(v)
+            if len(subs) == 0:
+                continue
+            srt = ''
+            for pos in range(0, len(subs) - 1):
+                seq_current = subs[pos]
+                m_current = re.match(self._TIMECODE_REGEX, seq_current['Timecode'])
+                if m_current is None:
+                    continue
+                seq_next = subs[pos + 1]
+                m_next = re.match(self._TIMECODE_REGEX, seq_next['Timecode'])
+                if m_next is None:
+                    continue
+                appear_time = m_current.group('timecode')
+                disappear_time = m_next.group('timecode')
+                text = seq_current['Caption']
+                srt += '%s\r\n%s --> %s\r\n%s' % (str(pos), appear_time, disappear_time, text)
+            if srt:
+                fixed_subtitles[k] = srt
+        return fixed_subtitles
+        
+    def _get_available_subtitles(self, video_id, webpage):
+        url = 'http://www.lynda.com/ajax/player?videoId=%s&type=transcript' % video_id
+        sub = self._download_webpage(url, None, note=False)
+        sub_json = json.loads(sub)
+        return {'en': url} if len(sub_json) > 0 else {}
 
 
 class LyndaCourseIE(InfoExtractor):

youtube_dl/extractor/mpora.py

@@ -0,0 +1,66 @@
+from __future__ import unicode_literals
+
+import json
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+)
+
+
+class MporaIE(InfoExtractor):
+    _VALID_URL = r'^https?://(www\.)?mpora\.(?:com|de)/videos/(?P<id>[^?#/]+)'
+    _IE_NAME = 'MPORA'
+
+    _TEST = {
+        'url': 'http://mpora.de/videos/AAdo8okx4wiz/embed?locale=de',
+        'file': 'AAdo8okx4wiz.mp4',
+        'md5': 'a7a228473eedd3be741397cf452932eb',
+        'info_dict': {
+            'title': 'Katy Curd -  Winter in the Forest',
+            'duration': 416,
+            'uploader': 'petenewman',
+        },
+    }
+
+    def _real_extract(self, url):
+        m = re.match(self._VALID_URL, url)
+        video_id = m.group('id')
+
+        webpage = self._download_webpage(url, video_id)
+        data_json = self._search_regex(
+            r"new FM\.Player\('[^']+',\s*(\{.*?)\);\n", webpage, 'json')
+
+        data = json.loads(data_json)
+
+        uploader = data['info_overlay']['name']
+        duration = data['video']['duration'] // 1000
+        thumbnail = data['video']['encodings']['sd']['poster']
+        title = data['info_overlay']['title']
+
+        formats = []
+        for encoding_id, edata in data['video']['encodings'].items():
+            for src in edata['sources']:
+                width_str = self._search_regex(
+                    r'_([0-9]+)\.[a-zA-Z0-9]+$', src['src'],
+                    False, default=None)
+                vcodec = src['type'].partition('/')[2]
+                
+                formats.append({
+                    'format_id': encoding_id + '-' + vcodec,
+                    'url': src['src'],
+                    'vcodec': vcodec,
+                    'width': int_or_none(width_str),
+                })
+
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'formats': formats,
+            'uploader': uploader,
+            'duration': duration,
+            'thumbnail': thumbnail,
+        }

youtube_dl/extractor/orf.py

@@ -1,54 +1,98 @@
 # coding: utf-8
+from __future__ import unicode_literals
 
-import re
-import xml.etree.ElementTree
 import json
+import re
 
 from .common import InfoExtractor
 from ..utils import (
-    compat_urlparse,
-    ExtractorError,
-    find_xpath_attr,
+    HEADRequest,
+    unified_strdate,
 )
 
+
 class ORFIE(InfoExtractor):
-    _VALID_URL = r'https?://tvthek\.orf\.at/(programs/.+?/episodes|topics/.+?)/(?P<id>\d+)'
+    _VALID_URL = r'https?://tvthek\.orf\.at/(?:programs/.+?/episodes|topics/.+?|program/[^/]+)/(?P<id>\d+)'
+
+    _TEST = {
+        'url': 'http://tvthek.orf.at/program/matinee-Was-Sie-schon-immer-ueber-Klassik-wissen-wollten/7317210/Was-Sie-schon-immer-ueber-Klassik-wissen-wollten/7319746/Was-Sie-schon-immer-ueber-Klassik-wissen-wollten/7319747',
+        'file': '7319747.mp4',
+        'md5': 'bd803c5d8c32d3c64a0ea4b4eeddf375',
+        'info_dict': {
+            'title': 'Was Sie schon immer über Klassik wissen wollten',
+            'description': 'md5:0ddf0d5f0060bd53f744edaa5c2e04a4',
+            'duration': 3508,
+            'upload_date': '20140105',
+        },
+        'skip': 'Blocked outside of Austria',
+    }
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         playlist_id = mobj.group('id')
         webpage = self._download_webpage(url, playlist_id)
 
-        flash_xml = self._search_regex('ORF.flashXML = \'(.+?)\'', webpage, u'flash xml')
-        flash_xml = compat_urlparse.parse_qs('xml='+flash_xml)['xml'][0]
-        flash_config = xml.etree.ElementTree.fromstring(flash_xml.encode('utf-8'))
-        playlist_json = self._search_regex(r'playlist\': \'(\[.*?\])\'', webpage, u'playlist').replace(r'\"','"')
-        playlist = json.loads(playlist_json)
+        data_json = self._search_regex(
+            r'initializeAdworx\((.+?)\);\n', webpage, 'video info')
+        all_data = json.loads(data_json)
+        sdata = all_data[0]['values']['segments']
 
-        videos = []
-        ns = '{http://tempuri.org/XMLSchema.xsd}'
-        xpath = '%(ns)sPlaylist/%(ns)sItems/%(ns)sItem' % {'ns': ns}
-        webpage_description = self._og_search_description(webpage)
-        for (i, (item, info)) in enumerate(zip(flash_config.findall(xpath), playlist), 1):
-            # Get best quality url
-            rtmp_url = None
-            for q in ['Q6A', 'Q4A', 'Q1A']:
-                video_url = find_xpath_attr(item, '%sVideoUrl' % ns, 'quality', q)
-                if video_url is not None:
-                    rtmp_url = video_url.text
-                    break
-            if rtmp_url is None:
-                raise ExtractorError(u'Couldn\'t get video url: %s' % info['id'])
-            description = self._html_search_regex(
-                r'id="playlist_entry_%s".*?<p>(.*?)</p>' % i, webpage,
-                u'description', default=webpage_description, flags=re.DOTALL)
-            videos.append({
+        def quality_to_int(s):
+            m = re.search('([0-9]+)', s)
+            if m is None:
+                return -1
+            return int(m.group(1))
+
+        entries = []
+        for sd in sdata:
+            video_id = sd['id']
+            formats = [{
+                'preference': -10 if fd['delivery'] == 'hls' else None,
+                'format_id': '%s-%s-%s' % (
+                    fd['delivery'], fd['quality'], fd['quality_string']),
+                'url': fd['src'],
+                'protocol': fd['protocol'],
+                'quality': quality_to_int(fd['quality']),
+            } for fd in sd['playlist_item_array']['sources']]
+
+            # Check for geoblocking.
+            # There is a property is_geoprotection, but that's always false
+            geo_str = sd.get('geoprotection_string')
+            if geo_str:
+                try:
+                    http_url = next(
+                        f['url']
+                        for f in formats
+                        if re.match(r'^https?://.*\.mp4$', f['url']))
+                except StopIteration:
+                    pass
+                else:
+                    req = HEADRequest(http_url)
+                    self._request_webpage(
+                        req, video_id,
+                        note='Testing for geoblocking',
+                        errnote=((
+                            'This video seems to be blocked outside of %s. '
+                            'You may want to try the streaming-* formats.')
+                            % geo_str),
+                        fatal=False)
+
+            self._sort_formats(formats)
+
+            upload_date = unified_strdate(sd['created_date'])
+            entries.append({
                 '_type': 'video',
-                'id': info['id'],
-                'title': info['title'],
-                'url': rtmp_url,
-                'ext': 'flv',
-                'description': description,
-                })
+                'id': video_id,
+                'title': sd['header'],
+                'formats': formats,
+                'description': sd.get('description'),
+                'duration': int(sd['duration_in_seconds']),
+                'upload_date': upload_date,
+                'thumbnail': sd.get('image_full_url'),
+            })
 
-        return videos
+        return {
+            '_type': 'playlist',
+            'entries': entries,
+            'id': playlist_id,
+        }

youtube_dl/extractor/veehd.py

@@ -1,3 +1,5 @@
+from __future__ import unicode_literals
+
 import re
 import json
 
@@ -8,16 +10,17 @@ from ..utils import (
     clean_html,
 )
 
+
 class VeeHDIE(InfoExtractor):
     _VALID_URL = r'https?://veehd\.com/video/(?P<id>\d+)'
 
     _TEST = {
-        u'url': u'http://veehd.com/video/4686958',
-        u'file': u'4686958.mp4',
-        u'info_dict': {
-            u'title': u'Time Lapse View from Space ( ISS)',
-            u'uploader_id': u'spotted',
-            u'description': u'md5:f0094c4cf3a72e22bc4e4239ef767ad7',
+        'url': 'http://veehd.com/video/4686958',
+        'file': '4686958.mp4',
+        'info_dict': {
+            'title': 'Time Lapse View from Space ( ISS)',
+            'uploader_id': 'spotted',
+            'description': 'md5:f0094c4cf3a72e22bc4e4239ef767ad7',
         },
     }
 
@@ -25,24 +28,30 @@ class VeeHDIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
 
+        # VeeHD seems to send garbage on the first request.
+        # See https://github.com/rg3/youtube-dl/issues/2102
+        self._download_webpage(url, video_id, 'Requesting webpage')
         webpage = self._download_webpage(url, video_id)
-        player_path = self._search_regex(r'\$\("#playeriframe"\).attr\({src : "(.+?)"',
-            webpage, u'player path')
+        player_path = self._search_regex(
+            r'\$\("#playeriframe"\).attr\({src : "(.+?)"',
+            webpage, 'player path')
         player_url = compat_urlparse.urljoin(url, player_path)
-        player_page = self._download_webpage(player_url, video_id,
-            u'Downloading player page')
-        config_json = self._search_regex(r'value=\'config=({.+?})\'',
-            player_page, u'config json')
+
+        self._download_webpage(player_url, video_id, 'Requesting player page')
+        player_page = self._download_webpage(
+            player_url, video_id, 'Downloading player page')
+        config_json = self._search_regex(
+            r'value=\'config=({.+?})\'', player_page, 'config json')
         config = json.loads(config_json)
 
         video_url = compat_urlparse.unquote(config['clip']['url'])
         title = clean_html(get_element_by_id('videoName', webpage).rpartition('|')[0])
         uploader_id = self._html_search_regex(r'<a href="/profile/\d+">(.+?)</a>',
-            webpage, u'uploader')
+            webpage, 'uploader')
         thumbnail = self._search_regex(r'<img id="veehdpreview" src="(.+?)"',
-            webpage, u'thumbnail')
+            webpage, 'thumbnail')
         description = self._html_search_regex(r'<td class="infodropdown".*?<div>(.*?)<ul',
-            webpage, u'description', flags=re.DOTALL)
+            webpage, 'description', flags=re.DOTALL)
 
         return {
             '_type': 'video',

youtube_dl/extractor/veoh.py

@@ -1,22 +1,22 @@
+from __future__ import unicode_literals
+
 import re
 import json
 
 from .common import InfoExtractor
-from ..utils import (
-    determine_ext,
-)
+
 
 class VeohIE(InfoExtractor):
-    _VALID_URL = r'http://www\.veoh\.com/watch/v(?P<id>\d*)'
+    _VALID_URL = r'http://(?:www\.)?veoh\.com/(?:watch|iphone/#_Watch)/v(?P<id>\d*)'
 
     _TEST = {
-        u'url': u'http://www.veoh.com/watch/v56314296nk7Zdmz3',
-        u'file': u'56314296.mp4',
-        u'md5': u'620e68e6a3cff80086df3348426c9ca3',
-        u'info_dict': {
-            u'title': u'Straight Backs Are Stronger',
-            u'uploader': u'LUMOback',
-            u'description': u'At LUMOback, we believe straight backs are stronger.  The LUMOback Posture & Movement Sensor:  It gently vibrates when you slouch, inspiring improved posture and mobility.  Use the app to track your data and improve your posture over time. ',
+        'url': 'http://www.veoh.com/watch/v56314296nk7Zdmz3',
+        'file': '56314296.mp4',
+        'md5': '620e68e6a3cff80086df3348426c9ca3',
+        'info_dict': {
+            'title': 'Straight Backs Are Stronger',
+            'uploader': 'LUMOback',
+            'description': 'At LUMOback, we believe straight backs are stronger.  The LUMOback Posture & Movement Sensor:  It gently vibrates when you slouch, inspiring improved posture and mobility.  Use the app to track your data and improve your posture over time. ',
         }
     }
 
@@ -28,20 +28,20 @@ class VeohIE(InfoExtractor):
         m_youtube = re.search(r'http://www\.youtube\.com/v/(.*?)(\&|")', webpage)
         if m_youtube is not None:
             youtube_id = m_youtube.group(1)
-            self.to_screen(u'%s: detected Youtube video.' % video_id)
+            self.to_screen('%s: detected Youtube video.' % video_id)
             return self.url_result(youtube_id, 'Youtube')
 
         self.report_extraction(video_id)
         info = self._search_regex(r'videoDetailsJSON = \'({.*?})\';', webpage, 'info')
         info = json.loads(info)
-        video_url =  info.get('fullPreviewHashHighPath') or info.get('fullPreviewHashLowPath')
+        video_url = info.get('fullPreviewHashHighPath') or info.get('fullPreviewHashLowPath')
 
-        return {'id': info['videoId'], 
-                'title': info['title'],
-                'ext': determine_ext(video_url),
-                'url': video_url,
-                'uploader': info['username'],
-                'thumbnail': info.get('highResImage') or info.get('medResImage'),
-                'description': info['description'],
-                'view_count': info['views'],
-                }
+        return {
+            'id': info['videoId'],
+            'title': info['title'],
+            'url': video_url,
+            'uploader': info['username'],
+            'thumbnail': info.get('highResImage') or info.get('medResImage'),
+            'description': info['description'],
+            'view_count': info['views'],
+        }

youtube_dl/extractor/vimeo.py

@@ -1,4 +1,6 @@
 # encoding: utf-8
+from __future__ import unicode_literals
+
 import json
 import re
 import itertools
@@ -31,54 +33,55 @@ class VimeoIE(InfoExtractor):
         (?P<id>[0-9]+)
         /?(?:[?&].*)?(?:[#].*)?$'''
     _NETRC_MACHINE = 'vimeo'
-    IE_NAME = u'vimeo'
+    IE_NAME = 'vimeo'
     _TESTS = [
         {
-            u'url': u'http://vimeo.com/56015672#at=0',
-            u'file': u'56015672.mp4',
-            u'md5': u'8879b6cc097e987f02484baf890129e5',
-            u'info_dict': {
-                u"upload_date": u"20121220", 
-                u"description": u"This is a test case for youtube-dl.\nFor more information, see github.com/rg3/youtube-dl\nTest chars: \u2605 \" ' \u5e78 / \\ \u00e4 \u21ad \U0001d550", 
-                u"uploader_id": u"user7108434", 
-                u"uploader": u"Filippo Valsorda", 
-                u"title": u"youtube-dl test video - \u2605 \" ' \u5e78 / \\ \u00e4 \u21ad \U0001d550",
+            'url': 'http://vimeo.com/56015672#at=0',
+            'file': '56015672.mp4',
+            'md5': '8879b6cc097e987f02484baf890129e5',
+            'info_dict': {
+                "upload_date": "20121220", 
+                "description": "This is a test case for youtube-dl.\nFor more information, see github.com/rg3/youtube-dl\nTest chars: \u2605 \" ' \u5e78 / \\ \u00e4 \u21ad \U0001d550", 
+                "uploader_id": "user7108434", 
+                "uploader": "Filippo Valsorda", 
+                "title": "youtube-dl test video - \u2605 \" ' \u5e78 / \\ \u00e4 \u21ad \U0001d550",
             },
         },
         {
-            u'url': u'http://vimeopro.com/openstreetmapus/state-of-the-map-us-2013/video/68093876',
-            u'file': u'68093876.mp4',
-            u'md5': u'3b5ca6aa22b60dfeeadf50b72e44ed82',
-            u'note': u'Vimeo Pro video (#1197)',
-            u'info_dict': {
-                u'uploader_id': u'openstreetmapus', 
-                u'uploader': u'OpenStreetMap US', 
-                u'title': u'Andy Allan - Putting the Carto into OpenStreetMap Cartography',
+            'url': 'http://vimeopro.com/openstreetmapus/state-of-the-map-us-2013/video/68093876',
+            'file': '68093876.mp4',
+            'md5': '3b5ca6aa22b60dfeeadf50b72e44ed82',
+            'note': 'Vimeo Pro video (#1197)',
+            'info_dict': {
+                'uploader_id': 'openstreetmapus',
+                'uploader': 'OpenStreetMap US',
+                'title': 'Andy Allan - Putting the Carto into OpenStreetMap Cartography',
             },
         },
         {
-            u'url': u'http://player.vimeo.com/video/54469442',
-            u'file': u'54469442.mp4',
-            u'md5': u'619b811a4417aa4abe78dc653becf511',
-            u'note': u'Videos that embed the url in the player page',
-            u'info_dict': {
-                u'title': u'Kathy Sierra: Building the minimum Badass User, Business of Software',
-                u'uploader': u'The BLN & Business of Software',
+            'url': 'http://player.vimeo.com/video/54469442',
+            'file': '54469442.mp4',
+            'md5': '619b811a4417aa4abe78dc653becf511',
+            'note': 'Videos that embed the url in the player page',
+            'info_dict': {
+                'title': 'Kathy Sierra: Building the minimum Badass User, Business of Software',
+                'uploader': 'The BLN & Business of Software',
+                'uploader_id': 'theblnbusinessofsoftware',
             },
         },
         {
-            u'url': u'http://vimeo.com/68375962',
-            u'file': u'68375962.mp4',
-            u'md5': u'aaf896bdb7ddd6476df50007a0ac0ae7',
-            u'note': u'Video protected with password',
-            u'info_dict': {
-                u'title': u'youtube-dl password protected test video',
-                u'upload_date': u'20130614',
-                u'uploader_id': u'user18948128',
-                u'uploader': u'Jaime Marquínez Ferrándiz',
+            'url': 'http://vimeo.com/68375962',
+            'file': '68375962.mp4',
+            'md5': 'aaf896bdb7ddd6476df50007a0ac0ae7',
+            'note': 'Video protected with password',
+            'info_dict': {
+                'title': 'youtube-dl password protected test video',
+                'upload_date': '20130614',
+                'uploader_id': 'user18948128',
+                'uploader': 'Jaime Marquínez Ferrándiz',
             },
-            u'params': {
-                u'videopassword': u'youtube-dl',
+            'params': {
+                'videopassword': 'youtube-dl',
             },
         },
     ]
@@ -90,7 +93,7 @@ class VimeoIE(InfoExtractor):
         self.report_login()
         login_url = 'https://vimeo.com/log_in'
         webpage = self._download_webpage(login_url, None, False)
-        token = re.search(r'xsrft: \'(.*?)\'', webpage).group(1)
+        token = self._search_regex(r'xsrft: \'(.*?)\'', webpage, 'login token')
         data = compat_urllib_parse.urlencode({'email': username,
                                               'password': password,
                                               'action': 'login',
@@ -100,13 +103,13 @@ class VimeoIE(InfoExtractor):
         login_request = compat_urllib_request.Request(login_url, data)
         login_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
         login_request.add_header('Cookie', 'xsrft=%s' % token)
-        self._download_webpage(login_request, None, False, u'Wrong login info')
+        self._download_webpage(login_request, None, False, 'Wrong login info')
 
     def _verify_video_password(self, url, video_id, webpage):
         password = self._downloader.params.get('videopassword', None)
         if password is None:
-            raise ExtractorError(u'This video is protected by a password, use the --video-password option')
-        token = re.search(r'xsrft: \'(.*?)\'', webpage).group(1)
+            raise ExtractorError('This video is protected by a password, use the --video-password option')
+        token = self._search_regex(r'xsrft: \'(.*?)\'', webpage, 'login token')
         data = compat_urllib_parse.urlencode({'password': password,
                                               'token': token})
         # I didn't manage to use the password with https
@@ -118,8 +121,8 @@ class VimeoIE(InfoExtractor):
         password_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
         password_request.add_header('Cookie', 'xsrft=%s' % token)
         self._download_webpage(password_request, video_id,
-                               u'Verifying the password',
-                               u'Wrong password')
+                               'Verifying the password',
+                               'Wrong password')
 
     def _real_initialize(self):
         self._login()
@@ -134,7 +137,7 @@ class VimeoIE(InfoExtractor):
         # Extract ID from URL
         mobj = re.match(self._VALID_URL, url)
         if mobj is None:
-            raise ExtractorError(u'Invalid URL: %s' % url)
+            raise ExtractorError('Invalid URL: %s' % url)
 
         video_id = mobj.group('id')
         if mobj.group('pro') or mobj.group('player'):
@@ -155,7 +158,7 @@ class VimeoIE(InfoExtractor):
         try:
             try:
                 config_url = self._html_search_regex(
-                    r' data-config-url="(.+?)"', webpage, u'config URL')
+                    r' data-config-url="(.+?)"', webpage, 'config URL')
                 config_json = self._download_webpage(config_url, video_id)
                 config = json.loads(config_json)
             except RegexNotFoundError:
@@ -166,19 +169,23 @@ class VimeoIE(InfoExtractor):
                     config_re = r'%s=({.+?});' % re.escape(m_variable_name.group(1))
                 else:
                     config_re = [r' = {config:({.+?}),assets:', r'(?:[abc])=({.+?});']
-                config = self._search_regex(config_re, webpage, u'info section',
+                config = self._search_regex(config_re, webpage, 'info section',
                     flags=re.DOTALL)
                 config = json.loads(config)
         except Exception as e:
             if re.search('The creator of this video has not given you permission to embed it on this domain.', webpage):
-                raise ExtractorError(u'The author has restricted the access to this video, try with the "--referer" option')
+                raise ExtractorError('The author has restricted the access to this video, try with the "--referer" option')
 
             if re.search('<form[^>]+?id="pw_form"', webpage) is not None:
                 self._verify_video_password(url, video_id, webpage)
                 return self._real_extract(url)
             else:
-                raise ExtractorError(u'Unable to extract info section',
+                raise ExtractorError('Unable to extract info section',
                                      cause=e)
+        else:
+            if config.get('view') == 4:
+                self._verify_video_password(url, video_id, webpage)
+                return self._real_extract(url)
 
         # Extract title
         video_title = config["video"]["title"]
@@ -212,9 +219,9 @@ class VimeoIE(InfoExtractor):
             video_upload_date = mobj.group(1) + mobj.group(2) + mobj.group(3)
 
         try:
-            view_count = int(self._search_regex(r'UserPlays:(\d+)', webpage, u'view count'))
-            like_count = int(self._search_regex(r'UserLikes:(\d+)', webpage, u'like count'))
-            comment_count = int(self._search_regex(r'UserComments:(\d+)', webpage, u'comment count'))
+            view_count = int(self._search_regex(r'UserPlays:(\d+)', webpage, 'view count'))
+            like_count = int(self._search_regex(r'UserLikes:(\d+)', webpage, 'like count'))
+            comment_count = int(self._search_regex(r'UserComments:(\d+)', webpage, 'comment count'))
         except RegexNotFoundError:
             # This info is only available in vimeo.com/{id} urls
             view_count = None
@@ -255,7 +262,7 @@ class VimeoIE(InfoExtractor):
         for key in ('other', 'sd', 'hd'):
             formats += files[key]
         if len(formats) == 0:
-            raise ExtractorError(u'No known codec found')
+            raise ExtractorError('No known codec found')
 
         return {
             'id':       video_id,
@@ -274,7 +281,7 @@ class VimeoIE(InfoExtractor):
 
 
 class VimeoChannelIE(InfoExtractor):
-    IE_NAME = u'vimeo:channel'
+    IE_NAME = 'vimeo:channel'
     _VALID_URL = r'(?:https?://)?vimeo.\com/channels/(?P<id>[^/]+)'
     _MORE_PAGES_INDICATOR = r'<a.+?rel="next"'
     _TITLE_RE = r'<link rel="alternate"[^>]+?title="(.*?)"'
@@ -283,14 +290,14 @@ class VimeoChannelIE(InfoExtractor):
         return '%s/videos/page:%d/' % (base_url, pagenum)
 
     def _extract_list_title(self, webpage):
-        return self._html_search_regex(self._TITLE_RE, webpage, u'list title')
+        return self._html_search_regex(self._TITLE_RE, webpage, 'list title')
 
     def _extract_videos(self, list_id, base_url):
         video_ids = []
         for pagenum in itertools.count(1):
             webpage = self._download_webpage(
                 self._page_url(base_url, pagenum) ,list_id,
-                u'Downloading page %s' % pagenum)
+                'Downloading page %s' % pagenum)
             video_ids.extend(re.findall(r'id="clip_(\d+?)"', webpage))
             if re.search(self._MORE_PAGES_INDICATOR, webpage, re.DOTALL) is None:
                 break
@@ -310,8 +317,8 @@ class VimeoChannelIE(InfoExtractor):
 
 
 class VimeoUserIE(VimeoChannelIE):
-    IE_NAME = u'vimeo:user'
-    _VALID_URL = r'(?:https?://)?vimeo.\com/(?P<name>[^/]+)'
+    IE_NAME = 'vimeo:user'
+    _VALID_URL = r'(?:https?://)?vimeo.\com/(?P<name>[^/]+)(?:/videos|[#?]|$)'
     _TITLE_RE = r'<a[^>]+?class="user">([^<>]+?)</a>'
 
     @classmethod
@@ -327,7 +334,7 @@ class VimeoUserIE(VimeoChannelIE):
 
 
 class VimeoAlbumIE(VimeoChannelIE):
-    IE_NAME = u'vimeo:album'
+    IE_NAME = 'vimeo:album'
     _VALID_URL = r'(?:https?://)?vimeo.\com/album/(?P<id>\d+)'
     _TITLE_RE = r'<header id="page_header">\n\s*<h1>(.*?)</h1>'
 
@@ -336,12 +343,12 @@ class VimeoAlbumIE(VimeoChannelIE):
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        album_id =  mobj.group('id')
+        album_id = mobj.group('id')
         return self._extract_videos(album_id, 'http://vimeo.com/album/%s' % album_id)
 
 
 class VimeoGroupsIE(VimeoAlbumIE):
-    IE_NAME = u'vimeo:group'
+    IE_NAME = 'vimeo:group'
     _VALID_URL = r'(?:https?://)?vimeo.\com/groups/(?P<name>[^/]+)'
 
     def _extract_list_title(self, webpage):
@@ -351,3 +358,24 @@ class VimeoGroupsIE(VimeoAlbumIE):
         mobj = re.match(self._VALID_URL, url)
         name = mobj.group('name')
         return self._extract_videos(name, 'http://vimeo.com/groups/%s' % name)
+
+
+class VimeoReviewIE(InfoExtractor):
+    IE_NAME = 'vimeo:review'
+    IE_DESC = 'Review pages on vimeo'
+    _VALID_URL = r'(?:https?://)?vimeo.\com/[^/]+/review/(?P<id>[^/]+)'
+    _TEST = {
+        'url': 'https://vimeo.com/user21297594/review/75524534/3c257a1b5d',
+        'file': '75524534.mp4',
+        'md5': 'c507a72f780cacc12b2248bb4006d253',
+        'info_dict': {
+            'title': "DICK HARDWICK 'Comedian'",
+            'uploader': 'Richard Hardwick',
+        }
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        player_url = 'https://player.vimeo.com/player/' + video_id
+        return self.url_result(player_url, 'Vimeo', video_id)

youtube_dl/postprocessor/__init__.py

@@ -0,0 +1,18 @@
+
+from .ffmpeg import (
+    FFmpegMergerPP,
+    FFmpegMetadataPP,
+    FFmpegVideoConvertor,
+    FFmpegExtractAudioPP,
+    FFmpegEmbedSubtitlePP,
+)
+from .xattrpp import XAttrMetadataPP
+
+__all__ = [
+    'FFmpegMergerPP',
+    'FFmpegMetadataPP',
+    'FFmpegVideoConvertor',
+    'FFmpegExtractAudioPP',
+    'FFmpegEmbedSubtitlePP',
+    'XAttrMetadataPP',
+]

youtube_dl/postprocessor/common.py

@@ -0,0 +1,49 @@
+from ..utils import PostProcessingError
+
+
+class PostProcessor(object):
+    """Post Processor class.
+
+    PostProcessor objects can be added to downloaders with their
+    add_post_processor() method. When the downloader has finished a
+    successful download, it will take its internal chain of PostProcessors
+    and start calling the run() method on each one of them, first with
+    an initial argument and then with the returned value of the previous
+    PostProcessor.
+
+    The chain will be stopped if one of them ever returns None or the end
+    of the chain is reached.
+
+    PostProcessor objects follow a "mutual registration" process similar
+    to InfoExtractor objects.
+    """
+
+    _downloader = None
+
+    def __init__(self, downloader=None):
+        self._downloader = downloader
+
+    def set_downloader(self, downloader):
+        """Sets the downloader for this PP."""
+        self._downloader = downloader
+
+    def run(self, information):
+        """Run the PostProcessor.
+
+        The "information" argument is a dictionary like the ones
+        composed by InfoExtractors. The only difference is that this
+        one has an extra field called "filepath" that points to the
+        downloaded file.
+
+        This method returns a tuple, the first element of which describes
+        whether the original file should be kept (i.e. not deleted - None for
+        no preference), and the second of which is the updated information.
+
+        In addition, this method may raise a PostProcessingError
+        exception if post processing fails.
+        """
+        return None, information  # by default, keep file and do nothing
+
+
+class AudioConversionError(PostProcessingError):
+    pass

youtube_dl/postprocessor/ffmpeg.py

@@ -4,65 +4,23 @@ import sys
 import time
 
 
-from .utils import (
+from .common import AudioConversionError, PostProcessor
+
+from ..utils import (
+    check_executable,
     compat_subprocess_get_DEVNULL,
     encodeFilename,
     PostProcessingError,
+    prepend_extension,
     shell_quote,
     subtitles_filename,
-    prepend_extension,
 )
 
 
-class PostProcessor(object):
-    """Post Processor class.
-
-    PostProcessor objects can be added to downloaders with their
-    add_post_processor() method. When the downloader has finished a
-    successful download, it will take its internal chain of PostProcessors
-    and start calling the run() method on each one of them, first with
-    an initial argument and then with the returned value of the previous
-    PostProcessor.
-
-    The chain will be stopped if one of them ever returns None or the end
-    of the chain is reached.
-
-    PostProcessor objects follow a "mutual registration" process similar
-    to InfoExtractor objects.
-    """
-
-    _downloader = None
-
-    def __init__(self, downloader=None):
-        self._downloader = downloader
-
-    def set_downloader(self, downloader):
-        """Sets the downloader for this PP."""
-        self._downloader = downloader
-
-    def run(self, information):
-        """Run the PostProcessor.
-
-        The "information" argument is a dictionary like the ones
-        composed by InfoExtractors. The only difference is that this
-        one has an extra field called "filepath" that points to the
-        downloaded file.
-
-        This method returns a tuple, the first element of which describes
-        whether the original file should be kept (i.e. not deleted - None for
-        no preference), and the second of which is the updated information.
-
-        In addition, this method may raise a PostProcessingError
-        exception if post processing fails.
-        """
-        return None, information # by default, keep file and do nothing
 
 class FFmpegPostProcessorError(PostProcessingError):
     pass
 
-class AudioConversionError(PostProcessingError):
-    pass
-
 class FFmpegPostProcessor(PostProcessor):
     def __init__(self,downloader=None):
         PostProcessor.__init__(self, downloader)
@@ -70,14 +28,8 @@ class FFmpegPostProcessor(PostProcessor):
 
     @staticmethod
     def detect_executables():
-        def executable(exe):
-            try:
-                subprocess.Popen([exe, '-version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
-            except OSError:
-                return False
-            return exe
         programs = ['avprobe', 'avconv', 'ffmpeg', 'ffprobe']
-        return dict((program, executable(program)) for program in programs)
+        return dict((program, check_executable(program, ['-version'])) for program in programs)
 
     def run_ffmpeg_multiple_files(self, input_paths, out_path, opts):
         if not self._exes['ffmpeg'] and not self._exes['avconv']:
@@ -108,6 +60,7 @@ class FFmpegPostProcessor(PostProcessor):
             return u'./' + fn
         return fn
 
+
 class FFmpegExtractAudioPP(FFmpegPostProcessor):
     def __init__(self, downloader=None, preferredcodec=None, preferredquality=None, nopostoverwrites=False):
         FFmpegPostProcessor.__init__(self, downloader)
@@ -236,6 +189,7 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
         information['filepath'] = new_path
         return self._nopostoverwrites,information
 
+
 class FFmpegVideoConvertor(FFmpegPostProcessor):
     def __init__(self, downloader=None,preferedformat=None):
         super(FFmpegVideoConvertor, self).__init__(downloader)
@@ -519,3 +473,4 @@ class FFmpegMergerPP(FFmpegPostProcessor):
         args = ['-c', 'copy']
         self.run_ffmpeg_multiple_files(info['__files_to_merge'], filename, args)
         return True, info
+

youtube_dl/postprocessor/xattrpp.py

@@ -0,0 +1,109 @@
+import os
+import subprocess
+import sys
+
+from .common import PostProcessor
+from ..utils import (
+    check_executable,
+    hyphenate_date,
+    preferredencoding,
+)
+
+
+class XAttrMetadataPP(PostProcessor):
+
+    #
+    # More info about extended attributes for media:
+    #   http://freedesktop.org/wiki/CommonExtendedAttributes/
+    #   http://www.freedesktop.org/wiki/PhreedomDraft/
+    #   http://dublincore.org/documents/usageguide/elements.shtml
+    #
+    # TODO:
+    #  * capture youtube keywords and put them in 'user.dublincore.subject' (comma-separated)
+    #  * figure out which xattrs can be used for 'duration', 'thumbnail', 'resolution'
+    #
+
+    def run(self, info):
+        """ Set extended attributes on downloaded file (if xattr support is found). """
+
+        # This mess below finds the best xattr tool for the job and creates a
+        # "write_xattr" function.
+        try:
+            # try the pyxattr module...
+            import xattr
+
+            def write_xattr(path, key, value):
+                return xattr.setxattr(path, key, value)
+
+        except ImportError:
+            if os.name == 'nt':
+                # Write xattrs to NTFS Alternate Data Streams:
+                # http://en.wikipedia.org/wiki/NTFS#Alternate_data_streams_.28ADS.29
+                def write_xattr(path, key, value):
+                    assert ':' not in key
+                    assert os.path.exists(path)
+
+                    ads_fn = path + ":" + key
+                    with open(ads_fn, "wb") as f:
+                        f.write(value)
+            else:
+                user_has_setfattr = check_executable("setfattr", ['--version'])
+                user_has_xattr = check_executable("xattr", ['-h'])
+
+                if user_has_setfattr or user_has_xattr:
+
+                    def write_xattr(path, key, value):
+                        if user_has_setfattr:
+                            cmd = ['setfattr', '-n', key, '-v', value, path]
+                        elif user_has_xattr:
+                            cmd = ['xattr', '-w', key, value, path]
+
+                        subprocess.check_output(cmd)
+
+                else:
+                    # On Unix, and can't find pyxattr, setfattr, or xattr.
+                    if sys.platform.startswith('linux'):
+                        self._downloader.report_error(
+                            "Couldn't find a tool to set the xattrs. "
+                            "Install either the python 'pyxattr' or 'xattr' "
+                            "modules, or the GNU 'attr' package "
+                            "(which contains the 'setfattr' tool).")
+                    else:
+                        self._downloader.report_error(
+                            "Couldn't find a tool to set the xattrs. "
+                            "Install either the python 'xattr' module, "
+                            "or the 'xattr' binary.")
+
+        # Write the metadata to the file's xattrs
+        self._downloader.to_screen('[metadata] Writing metadata to file\'s xattrs')
+
+        filename = info['filepath']
+
+        try:
+            xattr_mapping = {
+                'user.xdg.referrer.url': 'webpage_url',
+                # 'user.xdg.comment':            'description',
+                'user.dublincore.title': 'title',
+                'user.dublincore.date': 'upload_date',
+                'user.dublincore.description': 'description',
+                'user.dublincore.contributor': 'uploader',
+                'user.dublincore.format': 'format',
+            }
+
+            for xattrname, infoname in xattr_mapping.items():
+
+                value = info.get(infoname)
+
+                if value:
+                    if infoname == "upload_date":
+                        value = hyphenate_date(value)
+
+                    byte_value = value.encode('utf-8')
+                    write_xattr(filename, xattrname, byte_value)
+
+            return True, info
+
+        except (subprocess.CalledProcessError, OSError):
+            self._downloader.report_error("This filesystem doesn't support extended attributes. (You may have to enable them in your /etc/fstab)")
+            return False, info
+

youtube_dl/utils.py

@@ -764,6 +764,7 @@ def unified_strdate(date_str):
         '%Y-%m-%d',
         '%d/%m/%Y',
         '%Y/%m/%d %H:%M:%S',
+        '%Y-%m-%d %H:%M:%S',
         '%d.%m.%Y %H:%M',
         '%Y-%m-%dT%H:%M:%SZ',
         '%Y-%m-%dT%H:%M:%S.%fZ',
@@ -817,6 +818,15 @@ def date_from_str(date_str):
         return today + delta
     return datetime.datetime.strptime(date_str, "%Y%m%d").date()
     
+def hyphenate_date(date_str):
+    """
+    Convert a date in 'YYYYMMDD' format to 'YYYY-MM-DD' format"""
+    match = re.match(r'^(\d\d\d\d)(\d\d)(\d\d)$', date_str)
+    if match is not None:
+        return '-'.join(match.groups())
+    else:
+        return date_str
+
 class DateRange(object):
     """Represents a time interval between two dates"""
     def __init__(self, start=None, end=None):
@@ -1026,9 +1036,9 @@ def smuggle_url(url, data):
     return url + u'#' + sdata
 
 
-def unsmuggle_url(smug_url):
+def unsmuggle_url(smug_url, default=None):
     if not '#__youtubedl_smuggle' in smug_url:
-        return smug_url, None
+        return smug_url, default
     url, _, sdata = smug_url.rpartition(u'#')
     jsond = compat_parse_qs(sdata)[u'__youtubedl_smuggle'][0]
     data = json.loads(jsond)
@@ -1141,3 +1151,13 @@ def parse_duration(s):
 def prepend_extension(filename, ext):
     name, real_ext = os.path.splitext(filename) 
     return u'{0}.{1}{2}'.format(name, ext, real_ext)
+
+
+def check_executable(exe, args=[]):
+    """ Checks if the given binary is installed somewhere in PATH, and returns its name.
+    args can be a list of arguments for a short output (like -version) """
+    try:
+        subprocess.Popen([exe] + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
+    except OSError:
+        return False
+    return exe

youtube_dl/version.py

@@ -1,2 +1,2 @@
 
-__version__ = '2014.01.05.4'
+__version__ = '2014.01.07.2'