Windows build for 2012.11.28

Also enable album URLs on Vimeo.

.travis.yml

@@ -0,0 +1,9 @@
+language: python
+#specify the python version
+python:
+  - "2.6"
+  - "2.7"
+#command to install the setup
+install:
+# command to run tests
+script: nosetests test --nocapture

LATEST_VERSION

@@ -1 +1 @@
-2012.09.27
+2012.11.28

Makefile

@@ -1,26 +1,57 @@
-default: update
+all: youtube-dl README.md youtube-dl.1 youtube-dl.bash-completion LATEST_VERSION
+# TODO: re-add youtube-dl.exe, and make sure it's 1. safe and 2. doesn't need sudo
 
-update: compile update-readme update-latest
+clean:
+	rm -f youtube-dl youtube-dl.exe youtube-dl.1 LATEST_VERSION
 
-update-latest:
-	./youtube-dl.dev --version > LATEST_VERSION
+PREFIX=/usr/local
+BINDIR=$(PREFIX)/bin
+MANDIR=$(PREFIX)/man
+SYSCONFDIR=/etc
 
-update-readme:
-	@options=$$(COLUMNS=80 ./youtube-dl.dev --help | sed -e '1,/.*General Options.*/ d' -e 's/^\W\{2\}\(\w\)/### \1/') && \
-		header=$$(sed -e '/.*## OPTIONS/,$$ d' README.md) && \
-		footer=$$(sed -e '1,/.*## FAQ/ d' README.md) && \
-		echo "$${header}" > README.md && \
-		echo >> README.md && \
-		echo '## OPTIONS' >> README.md && \
-		echo "$${options}" >> README.md&& \
-		echo >> README.md && \
-		echo '## FAQ' >> README.md && \
-		echo "$${footer}" >> README.md
+install: youtube-dl youtube-dl.1 youtube-dl.bash-completion
+	install -d $(DESTDIR)$(BINDIR)
+	install -m 755 youtube-dl $(DESTDIR)$(BINDIR)
+	install -d $(DESTDIR)$(MANDIR)/man1
+	install -m 644 youtube-dl.1 $(DESTDIR)$(MANDIR)/man1
+	install -d $(DESTDIR)$(SYSCONFDIR)/bash_completion.d
+	install -m 644 youtube-dl.bash-completion $(DESTDIR)$(SYSCONFDIR)/bash_completion.d/youtube-dl
 
-compile:
+test:
+	nosetests2 --nocapture test
+
+.PHONY: all clean install test README.md youtube-dl.bash-completion
+# TODO un-phony README.md and youtube-dl.bash_completion by reading from .in files and generating from them
+
+youtube-dl: youtube_dl/*.py
 	zip --quiet --junk-paths youtube-dl youtube_dl/*.py
 	echo '#!/usr/bin/env python' > youtube-dl
 	cat youtube-dl.zip >> youtube-dl
 	rm youtube-dl.zip
+	chmod a+x youtube-dl
 
-.PHONY: default compile update update-latest update-readme
+youtube-dl.exe: youtube_dl/*.py
+	bash devscripts/wine-py2exe.sh build_exe.py
+
+README.md: youtube_dl/*.py
+	@options=$$(COLUMNS=80 python -m youtube_dl --help | sed -e '1,/.*General Options.*/ d' -e 's/^\W\{2\}\(\w\)/## \1/') && \
+		header=$$(sed -e '/.*# OPTIONS/,$$ d' README.md) && \
+		footer=$$(sed -e '1,/.*# CONFIGURATION/ d' README.md) && \
+		echo "$${header}" > README.md && \
+		echo >> README.md && \
+		echo '# OPTIONS' >> README.md && \
+		echo "$${options}" >> README.md&& \
+		echo >> README.md && \
+		echo '# CONFIGURATION' >> README.md && \
+		echo "$${footer}" >> README.md
+
+youtube-dl.1: README.md
+	pandoc -s -w man README.md -o youtube-dl.1
+
+youtube-dl.bash-completion: README.md
+	@options=`egrep -o '(--[a-z-]+) ' README.md | sort -u | xargs echo` && \
+		content=`sed "s/opts=\"[^\"]*\"/opts=\"$${options}\"/g" youtube-dl.bash-completion` && \
+		echo "$${content}" > youtube-dl.bash-completion
+
+LATEST_VERSION: youtube_dl/__init__.py
+	python -m youtube_dl --version > LATEST_VERSION

README.md

@@ -1,16 +1,19 @@
-# youtube-dl
+% youtube-dl(1)
 
-## USAGE
-youtube-dl [options] url [url...]
+# NAME
+youtube-dl
 
-## DESCRIPTION
+# SYNOPSIS
+**youtube-dl** [OPTIONS] URL [URL...]
+
+# DESCRIPTION
 **youtube-dl** is a small command-line program to download videos from
 YouTube.com and a few more sites. It requires the Python interpreter, version
 2.x (x being at least 6), and it is not platform specific. It should work in
 your Unix box, in Windows or in Mac OS X. It is released to the public domain,
 which means you can modify it, redistribute it or use it however you like.
 
-## OPTIONS
+# OPTIONS
     -h, --help               print this help text and exit
     --version                print program version and exit
     -U, --update             update this program to latest version
@@ -18,10 +21,11 @@ which means you can modify it, redistribute it or use it however you like.
     -r, --rate-limit LIMIT   download rate limit (e.g. 50k or 44.6m)
     -R, --retries RETRIES    number of retries (default is 10)
     --dump-user-agent        display the current browser identification
+    --user-agent UA          specify a custom user agent
     --list-extractors        List all supported extractors and the URLs they
                              would handle
 
-### Video Selection:
+## Video Selection:
     --playlist-start NUMBER  playlist video to start at (default is 1)
     --playlist-end NUMBER    playlist video to end at (default is last)
     --match-title REGEX      download only matching titles (regex or caseless
@@ -30,17 +34,21 @@ which means you can modify it, redistribute it or use it however you like.
                              caseless sub-string)
     --max-downloads NUMBER   Abort after downloading NUMBER files
 
-### Filesystem Options:
+## Filesystem Options:
     -t, --title              use title in file name
-    -l, --literal            use literal title in file name
+    --id                     use video ID in file name
+    -l, --literal            [deprecated] alias of --title
     -A, --auto-number        number downloaded files starting from 00000
-    -o, --output TEMPLATE    output filename template. Use %(stitle)s to get the
+    -o, --output TEMPLATE    output filename template. Use %(title)s to get the
                              title, %(uploader)s for the uploader name,
                              %(autonumber)s to get an automatically incremented
                              number, %(ext)s for the filename extension,
-                             %(upload_date)s for the upload date (YYYYMMDD), and
-                             %% for a literal percent. Use - to output to
-                             stdout.
+                             %(upload_date)s for the upload date (YYYYMMDD),
+                             %(extractor)s for the provider (youtube, metacafe,
+                             etc), %(id)s for the video id and %% for a literal
+                             percent. Use - to output to stdout.
+    --restrict-filenames     Restrict filenames to only ASCII characters, and
+                             avoid "&" and spaces in filenames
     -a, --batch-file FILE    file containing URLs to download ('-' for stdin)
     -w, --no-overwrites      do not overwrite files
     -c, --continue           resume partially downloaded files
@@ -53,7 +61,7 @@ which means you can modify it, redistribute it or use it however you like.
     --write-description      write video description to a .description file
     --write-info-json        write video metadata to a .info.json file
 
-### Verbosity / Simulation Options:
+## Verbosity / Simulation Options:
     -q, --quiet              activates quiet mode
     -s, --simulate           do not download the video and do not write anything
                              to disk
@@ -68,7 +76,7 @@ which means you can modify it, redistribute it or use it however you like.
     --console-title          display progress in console titlebar
     -v, --verbose            print various debugging information
 
-### Video Format Options:
+## Video Format Options:
     -f, --format FORMAT      video format code
     --all-formats            download all available video formats
     --prefer-free-formats    prefer free video formats unless a specific one is
@@ -80,22 +88,27 @@ which means you can modify it, redistribute it or use it however you like.
     --srt-lang LANG          language of the closed captions to download
                              (optional) use IETF language tags like 'en'
 
-### Authentication Options:
+## Authentication Options:
     -u, --username USERNAME  account username
     -p, --password PASSWORD  account password
     -n, --netrc              use .netrc authentication data
 
-### Post-processing Options:
-    --extract-audio          convert video files to audio-only files (requires
+## Post-processing Options:
+    -x, --extract-audio      convert video files to audio-only files (requires
                              ffmpeg or avconv and ffprobe or avprobe)
     --audio-format FORMAT    "best", "aac", "vorbis", "mp3", "m4a", or "wav";
                              best by default
-    --audio-quality QUALITY  ffmpeg/avconv audio bitrate specification, 128k by
-                             default
+    --audio-quality QUALITY  ffmpeg/avconv audio quality specification, insert a
+                             value between 0 (better) and 9 (worse) for VBR or a
+                             specific bitrate like 128K (default 5)
     -k, --keep-video         keeps the video file on disk after the post-
                              processing; the video is erased by default
 
-## FAQ
+# CONFIGURATION
+
+You can configure youtube-dl by placing default arguments (such as `--extract-audio --no-mtime` to always extract the audio and not copy the mtime) into `/etc/youtube-dl.conf` and/or `~/.local/config/youtube-dl.conf`.
+
+# FAQ
 
 ### Can you please put the -b option back?
 
@@ -117,13 +130,42 @@ The URLs youtube-dl outputs require the downloader to have the correct cookies.
 
 youtube has switched to a new video info format in July 2011 which is not supported by old versions of youtube-dl. You can update youtube-dl with `sudo youtube-dl --update`.
 
-## COPYRIGHT
+### ERROR: unable to download video ###
+
+youtube requires an additional signature since September 2012 which is not supported by old versions of youtube-dl. You can update youtube-dl with `sudo youtube-dl --update`.
+
+### SyntaxError: Non-ASCII character ###
+
+The error
+
+    File "youtube-dl", line 2
+    SyntaxError: Non-ASCII character '\x93' ...
+
+means you're using an outdated version of Python. Please update to Python 2.6 or 2.7.
+
+To run youtube-dl under Python 2.5, you'll have to manually check it out like this:
+
+	git clone git://github.com/rg3/youtube-dl.git
+	cd youtube-dl
+	python -m youtube_dl --help
+
+Please note that Python 2.5 is not supported anymore.
+
+### What is this binary file? Where has the code gone?
+
+Since June 2012 (#342) youtube-dl is packed as an executable zipfile, simply unzip it (might need renaming to `youtube-dl.zip` first on some systems) or clone the git repository, as laid out above. If you modify the code, you can run it by executing the `__main__.py` file. To recompile the executable, run `make youtube-dl`.
+
+### The exe throws a *Runtime error from Visual C++*
+
+To run the exe you need to install first the [Microsoft Visual C++ 2008 Redistributable Package](http://www.microsoft.com/en-us/download/details.aspx?id=29).
+
+# COPYRIGHT
 
 youtube-dl is released into the public domain by the copyright holders.
 
 This README file was originally written by Daniel Bolton (<https://github.com/dbbolton>) and is likewise released into the public domain.
 
-## BUGS
+# BUGS
 
 Bugs and suggestions should be reported at: <https://github.com/rg3/youtube-dl/issues>
 

devscripts/release.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+if [ -z "$1" ]; then echo "ERROR: specify version number like this: $0 1994.09.06"; exit 1; fi
+version="$1"
+if [ ! -z "`git tag | grep "$version"`" ]; then echo 'ERROR: version already present'; exit 1; fi
+if [ ! -z "`git status --porcelain`" ]; then echo 'ERROR: the working directory is not clean; commit or stash changes'; exit 1; fi
+sed -i "s/__version__ = '.*'/__version__ = '$version'/" youtube_dl/__init__.py
+make all
+git add -A
+git commit -m "release $version"
+git tag -m "Release $version" "$version"

test/parameters.json

@@ -0,0 +1 @@
+{"username": null, "listformats": null, "skip_download": false, "usenetrc": false, "max_downloads": null, "noprogress": false, "forcethumbnail": false, "forceformat": false, "format_limit": null, "ratelimit": null, "nooverwrites": false, "forceurl": false, "writeinfojson": false, "simulate": false, "playliststart": 1, "continuedl": true, "password": null, "prefer_free_formats": false, "nopart": false, "retries": 10, "updatetime": true, "consoletitle": false, "verbose": true, "forcefilename": false, "ignoreerrors": false, "logtostderr": false, "format": null, "subtitleslang": null, "quiet": false, "outtmpl": "%(id)s.%(ext)s", "rejecttitle": null, "playlistend": -1, "writedescription": false, "forcetitle": false, "forcedescription": false, "writesubtitles": false, "matchtitle": null}

test/test_div.py

@@ -1,29 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Various small unit tests
-
-import os,sys
-sys.path.append(os.path.dirname(os.path.dirname(__file__)))
-
-import youtube_dl
-
-def test_simplify_title():
-	assert youtube_dl._simplify_title(u'abc') == u'abc'
-	assert youtube_dl._simplify_title(u'abc_d-e') == u'abc_d-e'
-
-	assert youtube_dl._simplify_title(u'123') == u'123'
-
-	assert u'/' not in youtube_dl._simplify_title(u'abc/de')
-	assert u'abc' in youtube_dl._simplify_title(u'abc/de')
-	assert u'de' in youtube_dl._simplify_title(u'abc/de')
-	assert u'/' not in youtube_dl._simplify_title(u'abc/de///')
-
-	assert u'\\' not in youtube_dl._simplify_title(u'abc\\de')
-	assert u'abc' in youtube_dl._simplify_title(u'abc\\de')
-	assert u'de' in youtube_dl._simplify_title(u'abc\\de')
-
-	assert youtube_dl._simplify_title(u'ä') == u'ä'
-	assert youtube_dl._simplify_title(u'кириллица') == u'кириллица'
-
-	# Strip underlines
-	assert youtube_dl._simplify_title(u'\'a_') == u'a'

test/test_download.py

@@ -0,0 +1,198 @@
+#!/usr/bin/env python2
+import unittest
+import hashlib
+import os
+import json
+
+from youtube_dl.FileDownloader import FileDownloader
+from youtube_dl.InfoExtractors  import YoutubeIE, DailymotionIE
+from youtube_dl.InfoExtractors import  MetacafeIE, BlipTVIE
+from youtube_dl.InfoExtractors import  XVideosIE, VimeoIE
+from youtube_dl.InfoExtractors import  SoundcloudIE, StanfordOpenClassroomIE
+from youtube_dl.InfoExtractors import  CollegeHumorIE, XNXXIE
+
+
+class DownloadTest(unittest.TestCase):
+	PARAMETERS_FILE = "test/parameters.json"
+	#calculated with md5sum:
+	#md5sum (GNU coreutils) 8.19
+
+	YOUTUBE_SIZE = 1993883
+	YOUTUBE_URL = "http://www.youtube.com/watch?v=BaW_jenozKc"
+	YOUTUBE_FILE = "BaW_jenozKc.mp4"
+
+	DAILYMOTION_MD5 = "d363a50e9eb4f22ce90d08d15695bb47"
+	DAILYMOTION_URL = "http://www.dailymotion.com/video/x33vw9_tutoriel-de-youtubeur-dl-des-video_tech"
+	DAILYMOTION_FILE = "x33vw9.mp4"
+
+	METACAFE_SIZE = 5754305
+	METACAFE_URL = "http://www.metacafe.com/watch/yt-_aUehQsCQtM/the_electric_company_short_i_pbs_kids_go/"
+	METACAFE_FILE = "_aUehQsCQtM.flv"
+
+	BLIP_MD5 = "93c24d2f4e0782af13b8a7606ea97ba7"
+	BLIP_URL = "http://blip.tv/cbr/cbr-exclusive-gotham-city-imposters-bats-vs-jokerz-short-3-5796352"
+	BLIP_FILE = "5779306.m4v"
+
+	XVIDEO_MD5 = "1ab4dedc01f771cb2a65e91caa801aaf"
+	XVIDEO_URL = "http://www.xvideos.com/video939581/funny_porns_by_s_-1"
+	XVIDEO_FILE = "939581.flv"
+
+	VIMEO_MD5 = "1ab4dedc01f771cb2a65e91caa801aaf"
+	VIMEO_URL = "http://vimeo.com/14160053"
+	VIMEO_FILE = ""
+
+	VIMEO2_MD5 = ""
+	VIMEO2_URL = "http://player.vimeo.com/video/47019590"
+	VIMEO2_FILE = ""
+
+	SOUNDCLOUD_MD5 = "ce3775768ebb6432fa8495d446a078ed"
+	SOUNDCLOUD_URL = "http://soundcloud.com/ethmusic/lostin-powers-she-so-heavy"
+	SOUNDCLOUD_FILE = "n6FLbx6ZzMiu.mp3"
+
+	STANDFORD_MD5 = "22c8206291368c4e2c9c1a307f0ea0f4"
+	STANDFORD_URL = "http://openclassroom.stanford.edu/MainFolder/VideoPage.php?course=PracticalUnix&video=intro-environment&speed=100"
+	STANDFORD_FILE = "PracticalUnix_intro-environment.mp4"
+
+	COLLEGEHUMOR_MD5 = ""
+	COLLEGEHUMOR_URL = "http://www.collegehumor.com/video/6830834/mitt-romney-style-gangnam-style-parody"
+	COLLEGEHUMOR_FILE = ""
+
+	XNXX_MD5 = "5f0469c8d1dfd1bc38c8e6deb5e0a21d"
+	XNXX_URL = "http://video.xnxx.com/video1135332/lida_naked_funny_actress_5_"
+	XNXX_FILE = "1135332.flv"
+
+	def test_youtube(self):
+		#let's download a file from youtube
+		with open(DownloadTest.PARAMETERS_FILE) as f:
+			fd = FileDownloader(json.load(f))
+		fd.add_info_extractor(YoutubeIE())
+		fd.download([DownloadTest.YOUTUBE_URL])
+		self.assertTrue(os.path.exists(DownloadTest.YOUTUBE_FILE))
+		self.assertEqual(os.path.getsize(DownloadTest.YOUTUBE_FILE), DownloadTest.YOUTUBE_SIZE)
+
+	def test_dailymotion(self):
+		with open(DownloadTest.PARAMETERS_FILE) as f:
+			fd = FileDownloader(json.load(f))
+		fd.add_info_extractor(DailymotionIE())
+		fd.download([DownloadTest.DAILYMOTION_URL])
+		self.assertTrue(os.path.exists(DownloadTest.DAILYMOTION_FILE))
+		md5_down_file = md5_for_file(DownloadTest.DAILYMOTION_FILE)
+		self.assertEqual(md5_down_file, DownloadTest.DAILYMOTION_MD5)
+
+	def test_metacafe(self):
+		#this emulate a skip,to be 2.6 compatible
+		with open(DownloadTest.PARAMETERS_FILE) as f:
+			fd = FileDownloader(json.load(f))
+		fd.add_info_extractor(MetacafeIE())
+		fd.add_info_extractor(YoutubeIE())
+		fd.download([DownloadTest.METACAFE_URL])
+		self.assertTrue(os.path.exists(DownloadTest.METACAFE_FILE))
+		self.assertEqual(os.path.getsize(DownloadTest.METACAFE_FILE), DownloadTest.METACAFE_SIZE)
+
+	def test_blip(self):
+		with open(DownloadTest.PARAMETERS_FILE) as f:
+			fd = FileDownloader(json.load(f))
+		fd.add_info_extractor(BlipTVIE())
+		fd.download([DownloadTest.BLIP_URL])
+		self.assertTrue(os.path.exists(DownloadTest.BLIP_FILE))
+		md5_down_file = md5_for_file(DownloadTest.BLIP_FILE)
+		self.assertEqual(md5_down_file, DownloadTest.BLIP_MD5)
+
+	def test_xvideo(self):
+		with open(DownloadTest.PARAMETERS_FILE) as f:
+			fd = FileDownloader(json.load(f))
+		fd.add_info_extractor(XVideosIE())
+		fd.download([DownloadTest.XVIDEO_URL])
+		self.assertTrue(os.path.exists(DownloadTest.XVIDEO_FILE))
+		md5_down_file = md5_for_file(DownloadTest.XVIDEO_FILE)
+		self.assertEqual(md5_down_file, DownloadTest.XVIDEO_MD5)
+
+	def test_vimeo(self):
+		#skipped for the moment produce an error
+		return
+		with open(DownloadTest.PARAMETERS_FILE) as f:
+			fd = FileDownloader(json.load(f))
+		fd.add_info_extractor(VimeoIE())
+		fd.download([DownloadTest.VIMEO_URL])
+		self.assertTrue(os.path.exists(DownloadTest.VIMEO_FILE))
+		md5_down_file = md5_for_file(DownloadTest.VIMEO_FILE)
+		self.assertEqual(md5_down_file, DownloadTest.VIMEO_MD5)
+
+	def test_vimeo2(self):
+		#skipped for the moment produce an error
+		return
+		with open(DownloadTest.PARAMETERS_FILE) as f:
+			fd = FileDownloader(json.load(f))
+		fd.add_info_extractor(VimeoIE())
+		fd.download([DownloadTest.VIMEO2_URL])
+		self.assertTrue(os.path.exists(DownloadTest.VIMEO2_FILE))
+		md5_down_file = md5_for_file(DownloadTest.VIMEO2_FILE)
+		self.assertEqual(md5_down_file, DownloadTest.VIMEO2_MD5)
+
+	def test_soundcloud(self):
+		with open(DownloadTest.PARAMETERS_FILE) as f:
+			fd = FileDownloader(json.load(f))
+		fd.add_info_extractor(SoundcloudIE())
+		fd.download([DownloadTest.SOUNDCLOUD_URL])
+		self.assertTrue(os.path.exists(DownloadTest.SOUNDCLOUD_FILE))
+		md5_down_file = md5_for_file(DownloadTest.SOUNDCLOUD_FILE)
+		self.assertEqual(md5_down_file, DownloadTest.SOUNDCLOUD_MD5)
+
+	def test_standford(self):
+		with open(DownloadTest.PARAMETERS_FILE) as f:
+			fd = FileDownloader(json.load(f))
+		fd.add_info_extractor(StanfordOpenClassroomIE())
+		fd.download([DownloadTest.STANDFORD_URL])
+		self.assertTrue(os.path.exists(DownloadTest.STANDFORD_FILE))
+		md5_down_file = md5_for_file(DownloadTest.STANDFORD_FILE)
+		self.assertEqual(md5_down_file, DownloadTest.STANDFORD_MD5)
+
+	def test_collegehumor(self):
+		with open(DownloadTest.PARAMETERS_FILE) as f:
+			fd = FileDownloader(json.load(f))
+		fd.add_info_extractor(CollegeHumorIE())
+		fd.download([DownloadTest.COLLEGEHUMOR_URL])
+		self.assertTrue(os.path.exists(DownloadTest.COLLEGEHUMOR_FILE))
+		md5_down_file = md5_for_file(DownloadTest.COLLEGEHUMOR_FILE)
+		self.assertEqual(md5_down_file, DownloadTest.COLLEGEHUMOR_MD5)
+
+	def test_xnxx(self):
+		with open(DownloadTest.PARAMETERS_FILE) as f:
+			fd = FileDownloader(json.load(f))
+		fd.add_info_extractor(XNXXIE())
+		fd.download([DownloadTest.XNXX_URL])
+		self.assertTrue(os.path.exists(DownloadTest.XNXX_FILE))
+		md5_down_file = md5_for_file(DownloadTest.XNXX_FILE)
+		self.assertEqual(md5_down_file, DownloadTest.XNXX_MD5)
+
+	def tearDown(self):
+		if os.path.exists(DownloadTest.YOUTUBE_FILE):
+			os.remove(DownloadTest.YOUTUBE_FILE)
+		if os.path.exists(DownloadTest.DAILYMOTION_FILE):
+			os.remove(DownloadTest.DAILYMOTION_FILE)
+		if os.path.exists(DownloadTest.METACAFE_FILE):
+			os.remove(DownloadTest.METACAFE_FILE)
+		if os.path.exists(DownloadTest.BLIP_FILE):
+			os.remove(DownloadTest.BLIP_FILE)
+		if os.path.exists(DownloadTest.XVIDEO_FILE):
+			os.remove(DownloadTest.XVIDEO_FILE)
+		if os.path.exists(DownloadTest.VIMEO_FILE):
+			os.remove(DownloadTest.VIMEO_FILE)
+		if os.path.exists(DownloadTest.SOUNDCLOUD_FILE):
+			os.remove(DownloadTest.SOUNDCLOUD_FILE)
+		if os.path.exists(DownloadTest.STANDFORD_FILE):
+			os.remove(DownloadTest.STANDFORD_FILE)
+		if os.path.exists(DownloadTest.COLLEGEHUMOR_FILE):
+			os.remove(DownloadTest.COLLEGEHUMOR_FILE)
+		if os.path.exists(DownloadTest.XNXX_FILE):
+			os.remove(DownloadTest.XNXX_FILE)
+
+def md5_for_file(filename, block_size=2**20):
+	with open(filename) as f:
+		md5 = hashlib.md5()
+		while True:
+			data = f.read(block_size)
+			if not data:
+				break
+			md5.update(data)
+			return md5.hexdigest()

test/test_utils.py

@@ -0,0 +1,79 @@
+# -*- coding: utf-8 -*-
+
+# Various small unit tests
+
+import unittest
+
+#from youtube_dl.utils import htmlentity_transform
+from youtube_dl.utils import timeconvert
+from youtube_dl.utils import sanitize_filename
+from youtube_dl.utils import unescapeHTML
+from youtube_dl.utils import orderedSet
+
+
+class TestUtil(unittest.TestCase):
+	def test_timeconvert(self):
+		self.assertTrue(timeconvert('') is None)
+		self.assertTrue(timeconvert('bougrg') is None)
+
+	def test_sanitize_filename(self):
+		self.assertEqual(sanitize_filename(u'abc'), u'abc')
+		self.assertEqual(sanitize_filename(u'abc_d-e'), u'abc_d-e')
+
+		self.assertEqual(sanitize_filename(u'123'), u'123')
+
+		self.assertEqual(u'abc_de', sanitize_filename(u'abc/de'))
+		self.assertFalse(u'/' in sanitize_filename(u'abc/de///'))
+
+		self.assertEqual(u'abc_de', sanitize_filename(u'abc/<>\\*|de'))
+		self.assertEqual(u'xxx', sanitize_filename(u'xxx/<>\\*|'))
+		self.assertEqual(u'yes no', sanitize_filename(u'yes? no'))
+		self.assertEqual(u'this - that', sanitize_filename(u'this: that'))
+
+		self.assertEqual(sanitize_filename(u'AT&T'), u'AT&T')
+		self.assertEqual(sanitize_filename(u'ä'), u'ä')
+		self.assertEqual(sanitize_filename(u'кириллица'), u'кириллица')
+
+		forbidden = u'"\0\\/'
+		for fc in forbidden:
+			for fbc in forbidden:
+				self.assertTrue(fbc not in sanitize_filename(fc))
+
+	def test_sanitize_filename_restricted(self):
+		self.assertEqual(sanitize_filename(u'abc', restricted=True), u'abc')
+		self.assertEqual(sanitize_filename(u'abc_d-e', restricted=True), u'abc_d-e')
+
+		self.assertEqual(sanitize_filename(u'123', restricted=True), u'123')
+
+		self.assertEqual(u'abc_de', sanitize_filename(u'abc/de', restricted=True))
+		self.assertFalse(u'/' in sanitize_filename(u'abc/de///', restricted=True))
+
+		self.assertEqual(u'abc_de', sanitize_filename(u'abc/<>\\*|de', restricted=True))
+		self.assertEqual(u'xxx', sanitize_filename(u'xxx/<>\\*|', restricted=True))
+		self.assertEqual(u'yes_no', sanitize_filename(u'yes? no', restricted=True))
+		self.assertEqual(u'this_-_that', sanitize_filename(u'this: that', restricted=True))
+
+		self.assertEqual(sanitize_filename(u'aäb中国的c', restricted=True), u'a_b_c')
+		self.assertTrue(sanitize_filename(u'ö', restricted=True) != u'') # No empty filename
+
+		forbidden = u'"\0\\/&: \'\t\n'
+		for fc in forbidden:
+			for fbc in forbidden:
+				self.assertTrue(fbc not in sanitize_filename(fc, restricted=True))
+
+		# Handle a common case more neatly
+		self.assertEqual(sanitize_filename(u'大声带 - Song', restricted=True), u'Song')
+		self.assertEqual(sanitize_filename(u'总统: Speech', restricted=True), u'Speech')
+		# .. but make sure the file name is never empty
+		self.assertTrue(sanitize_filename(u'-', restricted=True) != u'')
+		self.assertTrue(sanitize_filename(u':', restricted=True) != u'')
+
+	def test_ordered_set(self):
+		self.assertEqual(orderedSet([1,1,2,3,4,4,5,6,7,3,5]), [1,2,3,4,5,6,7])
+		self.assertEqual(orderedSet([]), [])
+		self.assertEqual(orderedSet([1]), [1])
+		#keep the list ordered
+		self.assertEqual(orderedSet([135,1,1,1]), [135,1])
+
+	def test_unescape_html(self):
+		self.assertEqual(unescapeHTML(u"%20;"), u"%20;")

youtube-dl.1

@@ -0,0 +1,248 @@
+.TH youtube-dl 1 "" 
+.SH NAME
+.PP
+youtube-dl
+.SH SYNOPSIS
+.PP
+\f[B]youtube-dl\f[] [OPTIONS] URL [URL...]
+.SH DESCRIPTION
+.PP
+\f[B]youtube-dl\f[] is a small command-line program to download videos
+from YouTube.com and a few more sites.
+It requires the Python interpreter, version 2.x (x being at least 6),
+and it is not platform specific.
+It should work in your Unix box, in Windows or in Mac OS X.
+It is released to the public domain, which means you can modify it,
+redistribute it or use it however you like.
+.SH OPTIONS
+.IP
+.nf
+\f[C]
+-h,\ --help\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ print\ this\ help\ text\ and\ exit
+--version\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ print\ program\ version\ and\ exit
+-U,\ --update\ \ \ \ \ \ \ \ \ \ \ \ \ update\ this\ program\ to\ latest\ version
+-i,\ --ignore-errors\ \ \ \ \ \ continue\ on\ download\ errors
+-r,\ --rate-limit\ LIMIT\ \ \ download\ rate\ limit\ (e.g.\ 50k\ or\ 44.6m)
+-R,\ --retries\ RETRIES\ \ \ \ number\ of\ retries\ (default\ is\ 10)
+--dump-user-agent\ \ \ \ \ \ \ \ display\ the\ current\ browser\ identification
+--user-agent\ UA\ \ \ \ \ \ \ \ \ \ specify\ a\ custom\ user\ agent
+--list-extractors\ \ \ \ \ \ \ \ List\ all\ supported\ extractors\ and\ the\ URLs\ they
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ would\ handle
+\f[]
+.fi
+.SS Video Selection:
+.IP
+.nf
+\f[C]
+--playlist-start\ NUMBER\ \ playlist\ video\ to\ start\ at\ (default\ is\ 1)
+--playlist-end\ NUMBER\ \ \ \ playlist\ video\ to\ end\ at\ (default\ is\ last)
+--match-title\ REGEX\ \ \ \ \ \ download\ only\ matching\ titles\ (regex\ or\ caseless
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ sub-string)
+--reject-title\ REGEX\ \ \ \ \ skip\ download\ for\ matching\ titles\ (regex\ or
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ caseless\ sub-string)
+--max-downloads\ NUMBER\ \ \ Abort\ after\ downloading\ NUMBER\ files
+\f[]
+.fi
+.SS Filesystem Options:
+.IP
+.nf
+\f[C]
+-t,\ --title\ \ \ \ \ \ \ \ \ \ \ \ \ \ use\ title\ in\ file\ name
+--id\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ use\ video\ ID\ in\ file\ name
+-l,\ --literal\ \ \ \ \ \ \ \ \ \ \ \ [deprecated]\ alias\ of\ --title
+-A,\ --auto-number\ \ \ \ \ \ \ \ number\ downloaded\ files\ starting\ from\ 00000
+-o,\ --output\ TEMPLATE\ \ \ \ output\ filename\ template.\ Use\ %(title)s\ to\ get\ the
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ title,\ %(uploader)s\ for\ the\ uploader\ name,
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ %(autonumber)s\ to\ get\ an\ automatically\ incremented
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ number,\ %(ext)s\ for\ the\ filename\ extension,
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ %(upload_date)s\ for\ the\ upload\ date\ (YYYYMMDD),
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ %(extractor)s\ for\ the\ provider\ (youtube,\ metacafe,
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ etc),\ %(id)s\ for\ the\ video\ id\ and\ %%\ for\ a\ literal
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ percent.\ Use\ -\ to\ output\ to\ stdout.
+--restrict-filenames\ \ \ \ \ Restrict\ filenames\ to\ only\ ASCII\ characters,\ and
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ avoid\ "&"\ and\ spaces\ in\ filenames
+-a,\ --batch-file\ FILE\ \ \ \ file\ containing\ URLs\ to\ download\ (\[aq]-\[aq]\ for\ stdin)
+-w,\ --no-overwrites\ \ \ \ \ \ do\ not\ overwrite\ files
+-c,\ --continue\ \ \ \ \ \ \ \ \ \ \ resume\ partially\ downloaded\ files
+--no-continue\ \ \ \ \ \ \ \ \ \ \ \ do\ not\ resume\ partially\ downloaded\ files\ (restart
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ from\ beginning)
+--cookies\ FILE\ \ \ \ \ \ \ \ \ \ \ file\ to\ read\ cookies\ from\ and\ dump\ cookie\ jar\ in
+--no-part\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ do\ not\ use\ .part\ files
+--no-mtime\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ do\ not\ use\ the\ Last-modified\ header\ to\ set\ the\ file
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ modification\ time
+--write-description\ \ \ \ \ \ write\ video\ description\ to\ a\ .description\ file
+--write-info-json\ \ \ \ \ \ \ \ write\ video\ metadata\ to\ a\ .info.json\ file
+\f[]
+.fi
+.SS Verbosity / Simulation Options:
+.IP
+.nf
+\f[C]
+-q,\ --quiet\ \ \ \ \ \ \ \ \ \ \ \ \ \ activates\ quiet\ mode
+-s,\ --simulate\ \ \ \ \ \ \ \ \ \ \ do\ not\ download\ the\ video\ and\ do\ not\ write\ anything
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ to\ disk
+--skip-download\ \ \ \ \ \ \ \ \ \ do\ not\ download\ the\ video
+-g,\ --get-url\ \ \ \ \ \ \ \ \ \ \ \ simulate,\ quiet\ but\ print\ URL
+-e,\ --get-title\ \ \ \ \ \ \ \ \ \ simulate,\ quiet\ but\ print\ title
+--get-thumbnail\ \ \ \ \ \ \ \ \ \ simulate,\ quiet\ but\ print\ thumbnail\ URL
+--get-description\ \ \ \ \ \ \ \ simulate,\ quiet\ but\ print\ video\ description
+--get-filename\ \ \ \ \ \ \ \ \ \ \ simulate,\ quiet\ but\ print\ output\ filename
+--get-format\ \ \ \ \ \ \ \ \ \ \ \ \ simulate,\ quiet\ but\ print\ output\ format
+--no-progress\ \ \ \ \ \ \ \ \ \ \ \ do\ not\ print\ progress\ bar
+--console-title\ \ \ \ \ \ \ \ \ \ display\ progress\ in\ console\ titlebar
+-v,\ --verbose\ \ \ \ \ \ \ \ \ \ \ \ print\ various\ debugging\ information
+\f[]
+.fi
+.SS Video Format Options:
+.IP
+.nf
+\f[C]
+-f,\ --format\ FORMAT\ \ \ \ \ \ video\ format\ code
+--all-formats\ \ \ \ \ \ \ \ \ \ \ \ download\ all\ available\ video\ formats
+--prefer-free-formats\ \ \ \ prefer\ free\ video\ formats\ unless\ a\ specific\ one\ is
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ requested
+--max-quality\ FORMAT\ \ \ \ \ highest\ quality\ format\ to\ download
+-F,\ --list-formats\ \ \ \ \ \ \ list\ all\ available\ formats\ (currently\ youtube\ only)
+--write-srt\ \ \ \ \ \ \ \ \ \ \ \ \ \ write\ video\ closed\ captions\ to\ a\ .srt\ file
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ (currently\ youtube\ only)
+--srt-lang\ LANG\ \ \ \ \ \ \ \ \ \ language\ of\ the\ closed\ captions\ to\ download
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ (optional)\ use\ IETF\ language\ tags\ like\ \[aq]en\[aq]
+\f[]
+.fi
+.SS Authentication Options:
+.IP
+.nf
+\f[C]
+-u,\ --username\ USERNAME\ \ account\ username
+-p,\ --password\ PASSWORD\ \ account\ password
+-n,\ --netrc\ \ \ \ \ \ \ \ \ \ \ \ \ \ use\ .netrc\ authentication\ data
+\f[]
+.fi
+.SS Post-processing Options:
+.IP
+.nf
+\f[C]
+-x,\ --extract-audio\ \ \ \ \ \ convert\ video\ files\ to\ audio-only\ files\ (requires
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ ffmpeg\ or\ avconv\ and\ ffprobe\ or\ avprobe)
+--audio-format\ FORMAT\ \ \ \ "best",\ "aac",\ "vorbis",\ "mp3",\ "m4a",\ or\ "wav";
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ best\ by\ default
+--audio-quality\ QUALITY\ \ ffmpeg/avconv\ audio\ quality\ specification,\ insert\ a
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ value\ between\ 0\ (better)\ and\ 9\ (worse)\ for\ VBR\ or\ a
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ specific\ bitrate\ like\ 128K\ (default\ 5)
+-k,\ --keep-video\ \ \ \ \ \ \ \ \ keeps\ the\ video\ file\ on\ disk\ after\ the\ post-
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ processing;\ the\ video\ is\ erased\ by\ default
+\f[]
+.fi
+.SH CONFIGURATION
+.PP
+You can configure youtube-dl by placing default arguments (such as
+\f[C]--extract-audio\ --no-mtime\f[] to always extract the audio and not
+copy the mtime) into \f[C]/etc/youtube-dl.conf\f[] and/or
+\f[C]~/.local/config/youtube-dl.conf\f[].
+.SH FAQ
+.SS Can you please put the -b option back?
+.PP
+Most people asking this question are not aware that youtube-dl now
+defaults to downloading the highest available quality as reported by
+YouTube, which will be 1080p or 720p in some cases, so you no longer
+need the -b option.
+For some specific videos, maybe YouTube does not report them to be
+available in a specific high quality format you\[aq]\[aq]re interested
+in.
+In that case, simply request it with the -f option and youtube-dl will
+try to download it.
+.SS I get HTTP error 402 when trying to download a video. What\[aq]s
+this?
+.PP
+Apparently YouTube requires you to pass a CAPTCHA test if you download
+too much.
+We\[aq]\[aq]re considering to provide a way to let you solve the
+CAPTCHA (https://github.com/rg3/youtube-dl/issues/154), but at the
+moment, your best course of action is pointing a webbrowser to the
+youtube URL, solving the CAPTCHA, and restart youtube-dl.
+.SS I have downloaded a video but how can I play it?
+.PP
+Once the video is fully downloaded, use any video player, such as
+vlc (http://www.videolan.org) or mplayer (http://www.mplayerhq.hu/).
+.SS The links provided by youtube-dl -g are not working anymore
+.PP
+The URLs youtube-dl outputs require the downloader to have the correct
+cookies.
+Use the \f[C]--cookies\f[] option to write the required cookies into a
+file, and advise your downloader to read cookies from that file.
+Some sites also require a common user agent to be used, use
+\f[C]--dump-user-agent\f[] to see the one in use by youtube-dl.
+.SS ERROR: no fmt_url_map or conn information found in video info
+.PP
+youtube has switched to a new video info format in July 2011 which is
+not supported by old versions of youtube-dl.
+You can update youtube-dl with \f[C]sudo\ youtube-dl\ --update\f[].
+.SS ERROR: unable to download video
+.PP
+youtube requires an additional signature since September 2012 which is
+not supported by old versions of youtube-dl.
+You can update youtube-dl with \f[C]sudo\ youtube-dl\ --update\f[].
+.SS SyntaxError: Non-ASCII character
+.PP
+The error
+.IP
+.nf
+\f[C]
+File\ "youtube-dl",\ line\ 2
+SyntaxError:\ Non-ASCII\ character\ \[aq]\\x93\[aq]\ ...
+\f[]
+.fi
+.PP
+means you\[aq]re using an outdated version of Python.
+Please update to Python 2.6 or 2.7.
+.PP
+To run youtube-dl under Python 2.5, you\[aq]ll have to manually check it
+out like this:
+.IP
+.nf
+\f[C]
+git\ clone\ git://github.com/rg3/youtube-dl.git
+cd\ youtube-dl
+python\ -m\ youtube_dl\ --help
+\f[]
+.fi
+.PP
+Please note that Python 2.5 is not supported anymore.
+.SS What is this binary file? Where has the code gone?
+.PP
+Since June 2012 (#342) youtube-dl is packed as an executable zipfile,
+simply unzip it (might need renaming to \f[C]youtube-dl.zip\f[] first on
+some systems) or clone the git repository, as laid out above.
+If you modify the code, you can run it by executing the
+\f[C]__main__.py\f[] file.
+To recompile the executable, run \f[C]make\ youtube-dl\f[].
+.SS The exe throws a \f[I]Runtime error from Visual C++\f[]
+.PP
+To run the exe you need to install first the Microsoft Visual C++ 2008
+Redistributable
+Package (http://www.microsoft.com/en-us/download/details.aspx?id=29).
+.SH COPYRIGHT
+.PP
+youtube-dl is released into the public domain by the copyright holders.
+.PP
+This README file was originally written by Daniel Bolton
+(<https://github.com/dbbolton>) and is likewise released into the public
+domain.
+.SH BUGS
+.PP
+Bugs and suggestions should be reported at:
+<https://github.com/rg3/youtube-dl/issues>
+.PP
+Please include:
+.IP \[bu] 2
+Your exact command line, like
+\f[C]youtube-dl\ -t\ "http://www.youtube.com/watch?v=uHlDtZ6Oc3s&feature=channel_video_title"\f[].
+A common mistake is not to escape the \f[C]&\f[].
+Putting URLs in quotes should solve this problem.
+.IP \[bu] 2
+The output of \f[C]youtube-dl\ --version\f[]
+.IP \[bu] 2
+The output of \f[C]python\ --version\f[]
+.IP \[bu] 2
+The name and version of your Operating System ("Ubuntu 11.04 x64" or
+"Windows 7 x64" is usually enough).

youtube-dl.bash-completion

@@ -0,0 +1,14 @@
+__youtube-dl()
+{
+    local cur prev opts
+    COMPREPLY=()
+    cur="${COMP_WORDS[COMP_CWORD]}"
+    opts="--all-formats --audio-format --audio-quality --auto-number --batch-file --console-title --continue --cookies --dump-user-agent --extract-audio --format --get-description --get-filename --get-format --get-thumbnail --get-title --get-url --help --id --ignore-errors --keep-video --list-extractors --list-formats --literal --match-title --max-downloads --max-quality --netrc --no-continue --no-mtime --no-overwrites --no-part --no-progress --output --password --playlist-end --playlist-start --prefer-free-formats --quiet --rate-limit --reject-title --restrict-filenames --retries --simulate --skip-download --srt-lang --title --update --user-agent --username --verbose --version --write-description --write-info-json --write-srt"
+
+    if [[ ${cur} == * ]] ; then
+        COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) )
+        return 0
+    fi
+}
+
+complete -F __youtube-dl youtube-dl

youtube-dl.dev

@@ -1,6 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-import youtube_dl
-
-youtube_dl.main()

youtube_dl/FileDownloader.py

@@ -13,7 +13,7 @@ import urllib2
 
 if os.name == 'nt':
 	import ctypes
-	
+
 from utils import *
 
 
@@ -44,37 +44,38 @@ class FileDownloader(object):
 
 	Available options:
 
-	username:         Username for authentication purposes.
-	password:         Password for authentication purposes.
-	usenetrc:         Use netrc for authentication instead.
-	quiet:            Do not print messages to stdout.
-	forceurl:         Force printing final URL.
-	forcetitle:       Force printing title.
-	forcethumbnail:   Force printing thumbnail URL.
-	forcedescription: Force printing description.
-	forcefilename:    Force printing final filename.
-	simulate:         Do not download the video files.
-	format:           Video format code.
-	format_limit:     Highest quality format to try.
-	outtmpl:          Template for output names.
-	ignoreerrors:     Do not stop on download errors.
-	ratelimit:        Download speed limit, in bytes/sec.
-	nooverwrites:     Prevent overwriting files.
-	retries:          Number of times to retry for HTTP error 5xx
-	continuedl:       Try to continue downloads if possible.
-	noprogress:       Do not print the progress bar.
-	playliststart:    Playlist item to start at.
-	playlistend:      Playlist item to end at.
-	matchtitle:       Download only matching titles.
-	rejecttitle:      Reject downloads for matching titles.
-	logtostderr:      Log messages to stderr instead of stdout.
-	consoletitle:     Display progress in console window's titlebar.
-	nopart:           Do not use temporary .part files.
-	updatetime:       Use the Last-modified header to set output file timestamps.
-	writedescription: Write the video description to a .description file
-	writeinfojson:    Write the video description to a .info.json file
-	writesubtitles:   Write the video subtitles to a .srt file
-	subtitleslang:    Language of the subtitles to download
+	username:          Username for authentication purposes.
+	password:          Password for authentication purposes.
+	usenetrc:          Use netrc for authentication instead.
+	quiet:             Do not print messages to stdout.
+	forceurl:          Force printing final URL.
+	forcetitle:        Force printing title.
+	forcethumbnail:    Force printing thumbnail URL.
+	forcedescription:  Force printing description.
+	forcefilename:     Force printing final filename.
+	simulate:          Do not download the video files.
+	format:            Video format code.
+	format_limit:      Highest quality format to try.
+	outtmpl:           Template for output names.
+	restrictfilenames: Do not allow "&" and spaces in file names
+	ignoreerrors:      Do not stop on download errors.
+	ratelimit:         Download speed limit, in bytes/sec.
+	nooverwrites:      Prevent overwriting files.
+	retries:           Number of times to retry for HTTP error 5xx
+	continuedl:        Try to continue downloads if possible.
+	noprogress:        Do not print the progress bar.
+	playliststart:     Playlist item to start at.
+	playlistend:       Playlist item to end at.
+	matchtitle:        Download only matching titles.
+	rejecttitle:       Reject downloads for matching titles.
+	logtostderr:       Log messages to stderr instead of stdout.
+	consoletitle:      Display progress in console window's titlebar.
+	nopart:            Do not use temporary .part files.
+	updatetime:        Use the Last-modified header to set output file timestamps.
+	writedescription:  Write the video description to a .description file
+	writeinfojson:     Write the video description to a .info.json file
+	writesubtitles:    Write the video subtitles to a .srt file
+	subtitleslang:     Language of the subtitles to download
 	"""
 
 	params = None
@@ -93,6 +94,9 @@ class FileDownloader(object):
 		self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
 		self.params = params
 
+		if '%(stitle)s' in self.params['outtmpl']:
+			self.to_stderr(u'WARNING: %(stitle)s is deprecated. Use the %(title)s and the --restrict-filenames flag(which also secures %(uploader)s et al) instead.')
+
 	@staticmethod
 	def format_bytes(bytes):
 		if bytes is None:
@@ -139,23 +143,23 @@ class FileDownloader(object):
 		new_min = max(bytes / 2.0, 1.0)
 		new_max = min(max(bytes * 2.0, 1.0), 4194304) # Do not surpass 4 MB
 		if elapsed_time < 0.001:
-			return long(new_max)
+			return int(new_max)
 		rate = bytes / elapsed_time
 		if rate > new_max:
-			return long(new_max)
+			return int(new_max)
 		if rate < new_min:
-			return long(new_min)
-		return long(rate)
+			return int(new_min)
+		return int(rate)
 
 	@staticmethod
 	def parse_bytes(bytestr):
-		"""Parse a string indicating a byte quantity into a long integer."""
+		"""Parse a string indicating a byte quantity into an integer."""
 		matchobj = re.match(r'(?i)^(\d+(?:\.\d+)?)([kMGTPEZY]?)$', bytestr)
 		if matchobj is None:
 			return None
 		number = float(matchobj.group(1))
 		multiplier = 1024.0 ** 'bkmgtpezy'.index(matchobj.group(2).lower())
-		return long(round(number * multiplier))
+		return int(round(number * multiplier))
 
 	def add_info_extractor(self, ie):
 		"""Add an InfoExtractor object to the end of the list."""
@@ -173,7 +177,6 @@ class FileDownloader(object):
 		if not self.params.get('quiet', False):
 			terminator = [u'\n', u''][skip_eol]
 			output = message + terminator
-
 			if 'b' not in self._screen_file.mode or sys.version_info[0] < 3: # Python 2 lies about the mode of sys.stdout/sys.stderr
 				output = output.encode(preferredencoding(), 'ignore')
 			self._screen_file.write(output)
@@ -181,7 +184,8 @@ class FileDownloader(object):
 
 	def to_stderr(self, message):
 		"""Print message to stderr."""
-		print >>sys.stderr, message.encode(preferredencoding())
+		assert type(message) == type(u'')
+		sys.stderr.write((message + u'\n').encode(preferredencoding()))
 
 	def to_cons_title(self, message):
 		"""Set console/terminal window title to message."""
@@ -321,7 +325,7 @@ class FileDownloader(object):
 		"""Generate the output filename."""
 		try:
 			template_dict = dict(info_dict)
-			template_dict['epoch'] = unicode(long(time.time()))
+			template_dict['epoch'] = unicode(int(time.time()))
 			template_dict['autonumber'] = unicode('%05d' % self._num_downloads)
 			filename = self.params['outtmpl'] % template_dict
 			return filename
@@ -334,17 +338,22 @@ class FileDownloader(object):
 
 		title = info_dict['title']
 		matchtitle = self.params.get('matchtitle', False)
-		if matchtitle and not re.search(matchtitle, title, re.IGNORECASE):
-			return u'[download] "' + title + '" title did not match pattern "' + matchtitle + '"'
+		if matchtitle:
+			matchtitle = matchtitle.decode('utf8')
+			if not re.search(matchtitle, title, re.IGNORECASE):
+				return u'[download] "' + title + '" title did not match pattern "' + matchtitle + '"'
 		rejecttitle = self.params.get('rejecttitle', False)
-		if rejecttitle and re.search(rejecttitle, title, re.IGNORECASE):
-			return u'"' + title + '" title matched reject pattern "' + rejecttitle + '"'
+		if rejecttitle:
+			rejecttitle = rejecttitle.decode('utf8')
+			if re.search(rejecttitle, title, re.IGNORECASE):
+				return u'"' + title + '" title matched reject pattern "' + rejecttitle + '"'
 		return None
 
 	def process_info(self, info_dict):
 		"""Process a single dictionary returned by an InfoExtractor."""
 
-		info_dict['stitle'] = sanitize_filename(info_dict['title'])
+		# Keep for backwards compatibility
+		info_dict['stitle'] = info_dict['title']
 
 		reason = self._match_entry(info_dict)
 		if reason is not None:
@@ -357,20 +366,21 @@ class FileDownloader(object):
 				raise MaxDownloadsReached()
 
 		filename = self.prepare_filename(info_dict)
-		
+		filename = sanitize_filename(filename, self.params.get('restrictfilenames'))
+
 		# Forced printings
 		if self.params.get('forcetitle', False):
-			print info_dict['title'].encode(preferredencoding(), 'xmlcharrefreplace')
+			print(info_dict['title'].encode(preferredencoding(), 'xmlcharrefreplace'))
 		if self.params.get('forceurl', False):
-			print info_dict['url'].encode(preferredencoding(), 'xmlcharrefreplace')
+			print(info_dict['url'].encode(preferredencoding(), 'xmlcharrefreplace'))
 		if self.params.get('forcethumbnail', False) and 'thumbnail' in info_dict:
-			print info_dict['thumbnail'].encode(preferredencoding(), 'xmlcharrefreplace')
+			print(info_dict['thumbnail'].encode(preferredencoding(), 'xmlcharrefreplace'))
 		if self.params.get('forcedescription', False) and 'description' in info_dict:
-			print info_dict['description'].encode(preferredencoding(), 'xmlcharrefreplace')
+			print(info_dict['description'].encode(preferredencoding(), 'xmlcharrefreplace'))
 		if self.params.get('forcefilename', False) and filename is not None:
-			print filename.encode(preferredencoding(), 'xmlcharrefreplace')
+			print(filename.encode(preferredencoding(), 'xmlcharrefreplace'))
 		if self.params.get('forceformat', False):
-			print info_dict['format'].encode(preferredencoding(), 'xmlcharrefreplace')
+			print(info_dict['format'].encode(preferredencoding(), 'xmlcharrefreplace'))
 
 		# Do nothing else if in simulate mode
 		if self.params.get('simulate', False):
@@ -399,10 +409,10 @@ class FileDownloader(object):
 			except (OSError, IOError):
 				self.trouble(u'ERROR: Cannot write description file ' + descfn)
 				return
-				
+
 		if self.params.get('writesubtitles', False) and 'subtitles' in info_dict and info_dict['subtitles']:
 			# subtitles download errors are already managed as troubles in relevant IE
-			# that way it will silently go on when used with unsupporting IE 
+			# that way it will silently go on when used with unsupporting IE
 			try:
 				srtfn = filename.rsplit('.', 1)[0] + u'.srt'
 				self.report_writesubtitles(srtfn)
@@ -448,7 +458,7 @@ class FileDownloader(object):
 				except (ContentTooShortError, ), err:
 					self.trouble(u'ERROR: content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
 					return
-	
+
 			if success:
 				try:
 					self.post_process(filename, info_dict)
@@ -474,6 +484,7 @@ class FileDownloader(object):
 				# Extract information from URL and process it
 				videos = ie.extract(url)
 				for video in videos or []:
+					video['extractor'] = ie.IE_NAME
 					try:
 						self.increment_downloads()
 						self.process_info(video)

youtube_dl/PostProcessor.py

@@ -71,13 +71,14 @@ class FFmpegExtractAudioPP(PostProcessor):
 
 	@staticmethod
 	def detect_executables():
-		available = {'avprobe' : False, 'avconv' : False, 'ffmpeg' : False, 'ffprobe' : False}
-		for path in os.environ["PATH"].split(os.pathsep):
-			for program in available.keys():
-				exe_file = os.path.join(path, program)
-				if os.path.isfile(exe_file) and os.access(exe_file, os.X_OK):
-					available[program] = exe_file
-		return available
+		def executable(exe):
+			try:
+				subprocess.Popen([exe, '-version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
+			except OSError:
+				return False
+			return exe
+		programs = ['avprobe', 'avconv', 'ffmpeg', 'ffprobe']
+		return dict((program, executable(program)) for program in programs)
 
 	def get_audio_codec(self, path):
 		if not self._exes['ffprobe'] and not self._exes['avprobe']: return None
@@ -142,14 +143,20 @@ class FFmpegExtractAudioPP(PostProcessor):
 				extension = 'mp3'
 				more_opts = []
 				if self._preferredquality is not None:
-					more_opts += [self._exes['avconv'] and '-b:a' or '-ab', self._preferredquality]
+					if int(self._preferredquality) < 10:
+						more_opts += [self._exes['avconv'] and '-q:a' or '-aq', self._preferredquality]
+					else:
+						more_opts += [self._exes['avconv'] and '-b:a' or '-ab', self._preferredquality + 'k']
 		else:
 			# We convert the audio (lossy)
 			acodec = {'mp3': 'libmp3lame', 'aac': 'aac', 'm4a': 'aac', 'vorbis': 'libvorbis', 'wav': None}[self._preferredcodec]
 			extension = self._preferredcodec
 			more_opts = []
 			if self._preferredquality is not None:
-				more_opts += [self._exes['avconv'] and '-b:a' or '-ab', self._preferredquality]
+				if int(self._preferredquality) < 10:
+					more_opts += [self._exes['avconv'] and '-q:a' or '-aq', self._preferredquality]
+				else:
+					more_opts += [self._exes['avconv'] and '-b:a' or '-ab', self._preferredquality + 'k']
 			if self._preferredcodec == 'aac':
 				more_opts += ['-f', 'adts']
 			if self._preferredcodec == 'm4a':

youtube_dl/__init__.py

@@ -1,6 +1,8 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
+from __future__ import with_statement
+
 __authors__  = (
 	'Ricardo Garcia Gonzalez',
 	'Danny Colligan',
@@ -19,7 +21,7 @@ __authors__  = (
 	)
 
 __license__ = 'Public Domain'
-__version__ = '2012.09.27'
+__version__ = '2012.11.28'
 
 UPDATE_URL = 'https://raw.github.com/rg3/youtube-dl/master/youtube-dl'
 UPDATE_URL_VERSION = 'https://raw.github.com/rg3/youtube-dl/master/LATEST_VERSION'
@@ -46,7 +48,7 @@ from PostProcessor import *
 def updateSelf(downloader, filename):
 	''' Update the program file with the latest version from the repository '''
 	# Note: downloader only used for options
-	
+
 	if not os.access(filename, os.W_OK):
 		sys.exit('ERROR: no write permissions on %s' % filename)
 
@@ -64,7 +66,7 @@ def updateSelf(downloader, filename):
 		directory = os.path.dirname(exe)
 		if not os.access(directory, os.W_OK):
 			sys.exit('ERROR: no write permissions on %s' % directory)
-			
+
 		try:
 			urlh = urllib2.urlopen(UPDATE_URL_EXE)
 			newcontent = urlh.read()
@@ -73,20 +75,18 @@ def updateSelf(downloader, filename):
 				outf.write(newcontent)
 		except (IOError, OSError), err:
 			sys.exit('ERROR: unable to download latest version')
-			
+
 		try:
 			bat = os.path.join(directory, 'youtube-dl-updater.bat')
 			b = open(bat, 'w')
-			
-			print >> b, """
+			b.write("""
 echo Updating youtube-dl...
 ping 127.0.0.1 -n 5 -w 1000 > NUL
 move /Y "%s.new" "%s"
 del "%s"
-			""" %(exe, exe, bat)
-			
+			\n""" %(exe, exe, bat))
 			b.close()
-			
+
 			os.startfile(bat)
 		except (IOError, OSError), err:
 			sys.exit('ERROR: unable to overwrite current version')
@@ -186,16 +186,18 @@ def parseOpts():
 	general.add_option('-r', '--rate-limit',
 			dest='ratelimit', metavar='LIMIT', help='download rate limit (e.g. 50k or 44.6m)')
 	general.add_option('-R', '--retries',
-			dest='retries', metavar='RETRIES', help='number of retries (default is 10)', default=10)
+			dest='retries', metavar='RETRIES', help='number of retries (default is %default)', default=10)
 	general.add_option('--dump-user-agent',
 			action='store_true', dest='dump_user_agent',
 			help='display the current browser identification', default=False)
+	general.add_option('--user-agent',
+			dest='user_agent', help='specify a custom user agent', metavar='UA')
 	general.add_option('--list-extractors',
 			action='store_true', dest='list_extractors',
 			help='List all supported extractors and the URLs they would handle', default=False)
 
 	selection.add_option('--playlist-start',
-			dest='playliststart', metavar='NUMBER', help='playlist video to start at (default is 1)', default=1)
+			dest='playliststart', metavar='NUMBER', help='playlist video to start at (default is %default)', default=1)
 	selection.add_option('--playlist-end',
 			dest='playlistend', metavar='NUMBER', help='playlist video to end at (default is last)', default=-1)
 	selection.add_option('--match-title', dest='matchtitle', metavar='REGEX',help='download only matching titles (regex or caseless sub-string)')
@@ -261,13 +263,18 @@ def parseOpts():
 
 	filesystem.add_option('-t', '--title',
 			action='store_true', dest='usetitle', help='use title in file name', default=False)
+	filesystem.add_option('--id',
+			action='store_true', dest='useid', help='use video ID in file name', default=False)
 	filesystem.add_option('-l', '--literal',
-			action='store_true', dest='useliteral', help='use literal title in file name', default=False)
+			action='store_true', dest='usetitle', help='[deprecated] alias of --title', default=False)
 	filesystem.add_option('-A', '--auto-number',
 			action='store_true', dest='autonumber',
 			help='number downloaded files starting from 00000', default=False)
 	filesystem.add_option('-o', '--output',
-			dest='outtmpl', metavar='TEMPLATE', help='output filename template. Use %(stitle)s to get the title, %(uploader)s for the uploader name, %(autonumber)s to get an automatically incremented number, %(ext)s for the filename extension, %(upload_date)s for the upload date (YYYYMMDD), and %% for a literal percent. Use - to output to stdout.')
+			dest='outtmpl', metavar='TEMPLATE', help='output filename template. Use %(title)s to get the title, %(uploader)s for the uploader name, %(autonumber)s to get an automatically incremented number, %(ext)s for the filename extension, %(upload_date)s for the upload date (YYYYMMDD), %(extractor)s for the provider (youtube, metacafe, etc), %(id)s for the video id and %% for a literal percent. Use - to output to stdout.')
+	filesystem.add_option('--restrict-filenames',
+			action='store_true', dest='restrictfilenames',
+			help='Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames', default=False)
 	filesystem.add_option('-a', '--batch-file',
 			dest='batchfile', metavar='FILE', help='file containing URLs to download (\'-\' for stdin)')
 	filesystem.add_option('-w', '--no-overwrites',
@@ -292,12 +299,12 @@ def parseOpts():
 			help='write video metadata to a .info.json file', default=False)
 
 
-	postproc.add_option('--extract-audio', action='store_true', dest='extractaudio', default=False,
+	postproc.add_option('-x', '--extract-audio', action='store_true', dest='extractaudio', default=False,
 			help='convert video files to audio-only files (requires ffmpeg or avconv and ffprobe or avprobe)')
 	postproc.add_option('--audio-format', metavar='FORMAT', dest='audioformat', default='best',
 			help='"best", "aac", "vorbis", "mp3", "m4a", or "wav"; best by default')
-	postproc.add_option('--audio-quality', metavar='QUALITY', dest='audioquality', default='128K',
-			help='ffmpeg/avconv audio bitrate specification, 128k by default')
+	postproc.add_option('--audio-quality', metavar='QUALITY', dest='audioquality', default='5',
+			help='ffmpeg/avconv audio quality specification, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K (default 5)')
 	postproc.add_option('-k', '--keep-video', action='store_true', dest='keepvideo', default=False,
 			help='keeps the video file on disk after the post-processing; the video is erased by default')
 
@@ -326,6 +333,7 @@ def gen_extractors():
 	"""
 	return [
 		YoutubePlaylistIE(),
+		YoutubeChannelIE(),
 		YoutubeUserIE(),
 		YoutubeSearchIE(),
 		YoutubeIE(),
@@ -351,6 +359,9 @@ def gen_extractors():
 		MixcloudIE(),
 		StanfordOpenClassroomIE(),
 		MTVIE(),
+		YoukuIE(),
+		XNXXIE(),
+		GooglePlusIE(),
 
 		GenericIE()
 	]
@@ -368,6 +379,9 @@ def _real_main():
 				jar.load()
 		except (IOError, OSError), err:
 			sys.exit(u'ERROR: unable to open cookie file')
+	# Set user agent
+	if opts.user_agent is not None:
+		std_headers['User-Agent'] = opts.user_agent
 
 	# Dump user agent
 	if opts.dump_user_agent:
@@ -413,10 +427,10 @@ def _real_main():
 		parser.error(u'using .netrc conflicts with giving username/password')
 	if opts.password is not None and opts.username is None:
 		parser.error(u'account username missing')
-	if opts.outtmpl is not None and (opts.useliteral or opts.usetitle or opts.autonumber):
-		parser.error(u'using output template conflicts with using title, literal title or auto number')
-	if opts.usetitle and opts.useliteral:
-		parser.error(u'using title conflicts with using literal title')
+	if opts.outtmpl is not None and (opts.usetitle or opts.autonumber or opts.useid):
+		parser.error(u'using output template conflicts with using title, video ID or auto number')
+	if opts.usetitle and opts.useid:
+		parser.error(u'using title conflicts with using video ID')
 	if opts.username is not None and opts.password is None:
 		opts.password = getpass.getpass(u'Type account password and press return:')
 	if opts.ratelimit is not None:
@@ -444,6 +458,10 @@ def _real_main():
 	if opts.extractaudio:
 		if opts.audioformat not in ['best', 'aac', 'mp3', 'vorbis', 'm4a', 'wav']:
 			parser.error(u'invalid audio format specified')
+	if opts.audioquality:
+		opts.audioquality = opts.audioquality.strip('k').strip('K')
+		if not opts.audioquality.isdigit():
+			parser.error(u'invalid audio quality specified')
 
 	# File downloader
 	fd = FileDownloader({
@@ -463,15 +481,14 @@ def _real_main():
 		'format_limit': opts.format_limit,
 		'listformats': opts.listformats,
 		'outtmpl': ((opts.outtmpl is not None and opts.outtmpl.decode(preferredencoding()))
-			or (opts.format == '-1' and opts.usetitle and u'%(stitle)s-%(id)s-%(format)s.%(ext)s')
-			or (opts.format == '-1' and opts.useliteral and u'%(title)s-%(id)s-%(format)s.%(ext)s')
+			or (opts.format == '-1' and opts.usetitle and u'%(title)s-%(id)s-%(format)s.%(ext)s')
 			or (opts.format == '-1' and u'%(id)s-%(format)s.%(ext)s')
-			or (opts.usetitle and opts.autonumber and u'%(autonumber)s-%(stitle)s-%(id)s.%(ext)s')
-			or (opts.useliteral and opts.autonumber and u'%(autonumber)s-%(title)s-%(id)s.%(ext)s')
-			or (opts.usetitle and u'%(stitle)s-%(id)s.%(ext)s')
-			or (opts.useliteral and u'%(title)s-%(id)s.%(ext)s')
+			or (opts.usetitle and opts.autonumber and u'%(autonumber)s-%(title)s-%(id)s.%(ext)s')
+			or (opts.usetitle and u'%(title)s-%(id)s.%(ext)s')
+			or (opts.useid and u'%(id)s.%(ext)s')
 			or (opts.autonumber and u'%(autonumber)s-%(id)s.%(ext)s')
 			or u'%(id)s.%(ext)s'),
+		'restrictfilenames': opts.restrictfilenames,
 		'ignoreerrors': opts.ignoreerrors,
 		'ratelimit': opts.ratelimit,
 		'nooverwrites': opts.nooverwrites,
@@ -515,7 +532,7 @@ def _real_main():
 			parser.error(u'you must provide at least one URL')
 		else:
 			sys.exit()
-	
+
 	try:
 		retcode = fd.download(all_urls)
 	except MaxDownloadsReached:

youtube_dl/utils.py

@@ -19,13 +19,18 @@ except ImportError:
 	import StringIO
 
 std_headers = {
-	'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:5.0.1) Gecko/20100101 Firefox/5.0.1',
+	'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20100101 Firefox/10.0',
 	'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
 	'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
 	'Accept-Encoding': 'gzip, deflate',
 	'Accept-Language': 'en-us,en;q=0.5',
 }
 
+try:
+    compat_str = unicode # Python 2
+except NameError:
+    compat_str = str
+
 def preferredencoding():
 	"""Get preferred encoding.
 
@@ -83,7 +88,6 @@ class IDParser(HTMLParser.HTMLParser):
 		HTMLParser.HTMLParser.__init__(self)
 
 	def error(self, message):
-		print >> sys.stderr, self.getpos()
 		if self.error_count > 10 or self.started:
 			raise HTMLParser.HTMLParseError(message, self.getpos())
 		self.rawdata = '\n'.join(self.html.split('\n')[self.getpos()[0]:]) # skip one line
@@ -190,14 +194,36 @@ def timeconvert(timestr):
 	if timetuple is not None:
 		timestamp = email.utils.mktime_tz(timetuple)
 	return timestamp
-	
-def sanitize_filename(s):
-	"""Sanitizes a string so it could be used as part of a filename."""
+
+def sanitize_filename(s, restricted=False):
+	"""Sanitizes a string so it could be used as part of a filename.
+	If restricted is set, use a stricter subset of allowed characters.
+	"""
 	def replace_insane(char):
-		if char in u' .\\/|?*<>:"' or ord(char) < 32:
+		if char == '?' or ord(char) < 32 or ord(char) == 127:
+			return ''
+		elif char == '"':
+			return '' if restricted else '\''
+		elif char == ':':
+			return '_-' if restricted else ' -'
+		elif char in '\\/|*<>':
+			return '_'
+		if restricted and (char in '&\'' or char.isspace()):
+			return '_'
+		if restricted and ord(char) > 127:
 			return '_'
 		return char
-	return u''.join(map(replace_insane, s)).strip('_')
+
+	result = u''.join(map(replace_insane, s))
+	while '__' in result:
+		result = result.replace('__', '_')
+	result = result.strip('_')
+	# Common case of "Foreign band name - English song title"
+	if restricted and result.startswith('-_'):
+		result = result[2:]
+	if not result:
+		result = '_'
+	return result
 
 def orderedSet(iterable):
 	""" Remove all duplicates from the input iterable """