Strip silence from tracks, fix regression

.gitignore

@@ -3,3 +3,4 @@ settings.py
 *.pyc
 tracks/
 log.log
+*.swp

FMAdownloader.py

@@ -1,6 +1,12 @@
 #!/usr/bin/env python
 
-from urllib2 import urlopen
+from __future__ import print_function, unicode_literals
+try:
+    # Try Python3
+    from urllib.request import Request, urlopen
+except ImportError:
+    # Fall back to Python2
+    from urllib2 import Request, urlopen
 from json import loads
 from os.path import isfile
 from os import remove
@@ -19,24 +25,38 @@
 
 URL = '{base}?{settings}&api_key={api_key}'.format(base=BASE_URL, settings='&'.join(SETTINGS), api_key=API_KEY)
 
-s = urlopen(URL)
-tracks = loads(s.read())['dataset']
+headers = {'User-Agent' : 'FMA-playlist 0.1'}
+req = Request(URL, None, headers)
+s = urlopen(req)
+tracks = loads(s.read().decode('windows-1252'))['dataset']
 s.close()
 
-for track in tracks:
-    path = 'tracks/{file}'.format(file=track['track_file'].split('/')[-1])
-    if isfile(path):
-        print 'Already got {title}'.format(title=track['track_title'].encode('utf-8', errors='replace'))
+# Keep track of how many iterations we must make
+n = len(tracks)
+# Number of digits to use, so the progress indication has the same length
+number_width = str((n // 10) + 1)
+
+for i, track in enumerate(tracks):
+    filename = track['track_file'].split('/')[-1]
+    path = 'tracks/{file}'.format(file=filename)
+    download_path = 'downloaded_tracks/{file}'.format(file=filename)
+
+    # Print progress indication
+    print(('{i:0' + number_width + '}/{n:0' + number_width + '} ').format(i=i+1, n=n), end="")
+
+    if isfile(path) or isfile(download_path):
+        print('Already got {title}'.format(title=track['track_title']))
         continue
-    print 'Downloading {title}'.format(title=track['track_title'].encode('utf-8', errors='replace'))
-    f = open(path, "wb")
+    print('Downloading {title}'.format(title=track['track_title']))
+    f = open(download_path, "wb")
     try:
-        s = urlopen('{url}/download'.format(url=track['track_url']))
+        url = '{url}/download'.format(url=track['track_url'])
+        s = urlopen(Request(url, None, headers))
         f.write(s.read())
     except:
         f.close()
-        remove(path)
-        print 'Error while downloading, removed track'
+        remove(download_path)
+        print('Error while downloading, removed track')
     finally:
         s.close()
     f.close()

README.md

@@ -1,3 +1,67 @@
 #FMA-playlist
 
-Generate a playlist from the Free Music Archive API
+Generate a playlist from the Free Music Archive API.
+
+## Features
+
+* Download mp3 files from the 
+  [Free Music Archive](http://freemusicarchive.org/) and 
+  place them in a dedicated folder.
+* Strip silence from the start and end of the files.
+
+## Setting up
+
+The instructions assume you're using something similar to Ubuntu and are
+somewhat familiar with Linux.
+
+1.  Install dependencies.
+
+    `sudo apt-get install sox libsox-fmt-mp3`
+
+2.  Clone this repository and `cd` into it.
+
+3.  Customize your criteria by editing `FMAdownloader.py`. By default, the top
+    1000 tracks available for commercial use are chosen.
+
+4.  Run the program for the first time.
+
+    `make run`
+
+    You will first be prompted for an API key, which you can obtain by
+    reading and agreeing to [FMA API ToS](https://freemusicarchive.org/api/agreement).
+    The key will be saved to `settings.py` and will be reused on further runs.
+    Create `settings.py` with the appropriate content _before_ running the program
+    if you cannot use the prompt (e.g. during automated installation).
+
+    Next, all tracks will be downloaded to `downloaded_tracks/`, one at a time 
+    (so we're not a burden for FMA). After that, they will have silence 
+    removed from either end of them before they're moved to `tracks/`.
+
+5.  Optional: to add new tracks as the search result changes over time, add
+    `make run` to your crontab.
+
+    `crontab -e`
+
+    Add a line which runs `make run` once every month or so, like this:
+
+    `0 10 1 * * make -C /path/to/FMA-playlist run > /path/to/FMA-playlist/runlog.log 2>&1`
+
+    in which you replace `/path/to/FMA-playlist` with the actual path to this
+    directory. (As a side note, you might want to remove the redirects if you want an
+    email each time the job is run, and thus confirm that it works.)
+
+Once you've done all this, the `tracks/` directory can be used as a playlist
+of music from the Free Music Archive. Example of how to use it with [LiquidSoap][ls]:
+
+```
+fma = playlist("/path/to/FMA-playlist/tracks")
+fma = normalize(fma, gain_max=1000.0, target=-1.0, window=2.0)
+fma = nrj(fma)
+fma = smart_crossfade(fma)
+fma = strip_blank(max_blank=12., threshold=-80., track_sensitive=false, fma)
+# Remember that fma is fallible; use mksafe if you want to output it directly
+# Also use some mechanism to attribute the artist (you must follow the CC licenses!)
+```
+
+[ls]: http://savonet.sourceforge.net/index.html
+

makefile

@@ -0,0 +1,43 @@
+# Makefile for FMAplaylist, used for running the program (not installation!)
+# See https://www.gnu.org/software/make/manual/make.html for a comprehensive, yet
+# friendly introduction to makefiles.
+
+# Default recipe; will download files and process them.
+run : download
+	@# We must invoke make again, so files_to_process is generated once again
+	@# and includes any new, downloaded files.
+	@make process
+
+# Download files from FMA into the downloaded_tracks directory.
+download : configure
+	python FMAdownloader.py
+
+# Implicit recipe describing how to process a single, downloaded track
+tracks/%.mp3 : downloaded_tracks/%.mp3
+	@echo Processing "$<"
+	@# Remove silence from the start and end of the track, and
+	@# add 0.3 seconds of silence to the end.
+	@# See http://digitalcardboard.com/blog/2009/08/25/the-sox-of-silence/ for
+	@# an explaination of this command.
+	@sox $< $@ silence 1 0.1 0.1% reverse silence 1 0.1 0.1% reverse pad 0.0 0.3
+	@# Remove the file from the downloaded folder (no need to waste space)
+	@rm $<
+
+# Variable listing the tracks inside the tracks directory that need to be made.
+# This is done by looking at what tracks are in the downloaded_tracks directory.
+# See https://www.gnu.org/software/make/manual/make.html#Wildcard-Function
+files_to_process := $(patsubst downloaded_tracks/%,tracks/%,$(wildcard downloaded_tracks/*))
+
+# Process downloaded files and move them to the tracks directory.
+process : $(files_to_process)
+
+# Configure the program, making it ready to be used.
+configure : settings.py
+
+# Populate settings.py with the API_KEY.
+settings.py : 
+	@echo "To use the Free Music Archive API, you need to obtain an API key."
+	@echo "Visit https://freemusicarchive.org/api/agreement if you don't have one already."
+	@echo 
+	@read -p "API key: " input; \
+	echo API_KEY = \"$$input\" > settings.py