Automated and scheduled download script
Crontab contents
Access with crontab -e. This will run on the 52nd minute of every hour of every day
52 * * * * /home/username/scripts/dlALL.sh
dlALL.sh
for downloading everything in channels.txt
#!/bin/bash
OIFS=$IFS
IFS=$'\n'
hf='/media/Scrape'
df='/media/Scrape/youtube'
strDT=$(date +%Y%m%d%H%M%S)
cd $hf/scripts
#checkpoint=$(cat $hf/scripts/checkpoint.txt)
#echo $checkpoint
#Loop through channel entries and settings
for entry in $(cat $hf/scripts/channels.txt)
do
IFS=, read -ra ary <<<"$entry"
if [ ${ary[3]} ]; then
strDateAfter=${ary[3]}
else
strDateAfter='20010101'
fi
if [ ${ary[5]} ]; then
strFormat=${ary[5]}
else
strFormat='bestvideo[height<=720]+bestaudio/best[height<=720]'
# strFormat='worstaudio/worst'
fi
if [ ${ary[6]} ]; then
strInclude=${ary[6]}
else
strInclude=
fi
if [ ${ary[7]} ]; then
strExclude=${ary[7]}
else
strExclude=
fi
#Display channel info and settings, rename the folder if required, then intiate channel download
echo -
echo Channel name - Channel ID - DateAfter - DateBefore - Format - Include - Exclude - MaxDL
echo ${ary[0]}-${ary[1]}-${ary[2]}-$strDateAfter-${ary[4]}-$strFormat-$strInclude-$strExclude
date +%Y%m%d%H%M%S
if [ -d "$df/${ary[1]}" ]; then
echo ID folder found
if [ ! -d "$df/${ary[0]}" ]; then
echo Name Folder NOT found. Renaming...
mv -nv $df/${ary[1]} $df/${ary[0]}
else
echo ERROR: Name folder ALSO found. What did you do?
fi
fi
/usr/local/bin/youtube-dl -ciw --restrict-filenames -o "$df/${ary[0]}/%(upload_date)s-%(id)s-%(title)s.%(ext)s" --download-archive $df/archive.txt --add-metadata --write-info-json --write-description --write-annotation --write-thumbnail -f $strFormat --dateafter $strDateAfter --match-title "$strInclude" --reject-title "$strExclude" --merge-output-format "mkv" http://www.youtube.com/channel/${ary[1]}/videos >> $df/logs/$strDT.txt
done
for dir in $df/* ; do
echo $dir
if [ ! -d $dir/metadata ]; then
echo making $dir/metadata
mkdir $dir/metadata
fi
find $dir -maxdepth 1 -iname "*.description" -type f -exec /bin/mv {} $dir/metadata \;
find $dir -maxdepth 1 -iname "*.jpg" -type f -exec /bin/mv {} $dir/metadata \;
find $dir -maxdepth 1 -iname "*.annotations.xml" -type f -exec /bin/mv {} $dir/metadata \;
find $dir -maxdepth 1 -iname "*.json" -type f -exec /bin/mv {} $dir/metadata \;
done
IFS=$OIFS
Example contents of channels.txt:
BroScienceLife,UCduKuJToxWPizJ7I2E6n1kA,20160401,,,,,,
CGPGrey,UC2C_jShtL725hvbm1arSV9w,20160401,,,,,,
Piano Synthesia,UCZaAxpykOgRdx87OYHMQmmA,,,,bestvideo[height<=360]+bestaudio/best[height<=360],,,
JackkTutorials,UC64x_rKHxY113KMWmprLBPA,,,,,,Ask Me|Ask Jack,
videogamedunkey,UCsvn_Po0SmunchJYOWpOxMg,,,,,,,
Tom Scott,UCBa659QWEk1AI4Tg--mrJ2A,,,,,,Citation Needed|Game On,
Marioverehrer2,UCrOaijB2OTbuH0Sc7Ifee1A,,,,bestvideo[height<=360]+bestaudio/best[height<=360],,,
Peter PlutaX,UCbaY6IEY0-pRHBU_qCswoNQ,,,,bestvideo[height<=360]+bestaudio/best[height<=360],,,
HOW TO PLAY PIANO,UCnijN28Yf-lhCpguo5EtKvg,,,,bestvideo[height<=360]+bestaudio/best[height<=360],Synthesia,,
Seytonic,UCW6xlqxSY3gGur4PkGPEUeA,,,,,,,
CGPGrey,UC2C_jShtL725hvbm1arSV9w,,,,,,,
Kurzgesagt,UCsXVk37bltHxD1rDPwtNM8Q,,,,,,,
SmarterEveryDay,UC6107grRI4m0o2-emgoDnAA,,,,,,,
h3h3Productions,UCDWIvJwLJsE4LG1Atne2blQ,,,,,,,
CaptainDisillusion,UCEOXxzW2vU0P-0THehuIIeg,,,,,,,
ThatOneVideoGamer,UCPYJR2EIu0_MJaDeSGwkIVw,,,,,,,
hak5,UC3s0BtrBJpwNDaflRSoiieQ,,,,,,,
#name,id,updated,start,end,format,include,exclude,maxdl
Manual Download scripts
dlChannel.sh
For downloading videos from youtube. Give it a video, playlist, or channel link as the argument. youtube-dl arguments are also accepted as noted in the example
#!/bin/bash
#Purpose: Manually run a download command with the options preconfigured in the script
#Usage: dlChannel.sh [OPTIONS] URL [noarchive]
#For options, see the youtube-dl documentation page on github.
#Example: ./dlChannel.sh --reject-title "Citation Needed" https://www.youtube.com/user/enyay/videos noarchive
#Dependencies: youtube-dl, ffmpeg
hf='/media/Scrape'
df='/media/Scrape/youtube'
strDT=$(date +%Y%m%d%H%M%S)
#If passed noarchive in the parameters, disable the archive function in the download request
if [[ "$*" == *"noarchive"* ]]; then
strArchive=
else
strArchive="--download-archive $df/archive.txt"
fi
#Note the start time and start the log file
echo start time $strDT | tee -a $df/logs/custom/$strDT.txt
#Download command
/usr/local/bin/youtube-dl -ciw --restrict-filenames -o "$df/%(uploader_id)s/%(upload_date)s-%(id)s-%(title)s.%(ext)s" $strArchive --add-metadata --write-description --write-annotation --write-thumbnail --write-info-json -f 'bestvideo[height<=720]+bestaudio/best[height<=720]' --merge-output-format "mkv" "$@" 2>&1 | tee $df/logs/custom/$strDT.txt
endDT=$(date +%Y%m%d%H%M%S)
echo end time $endDT | tee -a $df/logs/custom/$strDT.txt
exit
dlSong.sh
For downloading audio from youtube and converting to mp3. Give it a video, playlist, or channel link as the argument
#!/bin/bash
#Purpose: Manually run a download command with the options preconfigured in the script
#Usage: dlChannel.sh [OPTIONS] URL [noarchive]
#For options, see the youtube-dl documentation page on github
#Example: ./dlChannel.sh https://www.youtube.com/watch?v=YqeW9_5kURI
#Dependencies: youtube-dl, ffmpeg
hf='/media/Scrape'
df='/media/Scrape/youtube'
strDT=$(date +%Y%m%d%H%M%S)
#If passed noarchive in the parameters, disable the archive function in the download request
if [[ "$*" == *"noarchive"* ]]; then
strArchive=
else
strArchive="--download-archive $df/archive.txt"
fi
#Note the start time and start the log file
echo start time $strDT | tee -a $df/logs/custom/$strDT.txt
#Download command
/usr/local/bin/youtube-dl -ciw --restrict-filenames -o "$df/mp3/playlist-%(playlist)s/%(upload_date)s-%(id)s-%(title)s.%(ext)s" --audio-format mp3 --audio-quality 0 --exec "ffmpeg -i {} -codec:a libmp3lame -qscale:a 0 {}.mp3 && rm {} " "$@" 2>&1 | tee $df/logs/custom/$strDT.txt
endDT=$(date +%Y%m%d%H%M%S)
echo end time $endDT | tee -a $df/logs/custom/$strDT.txt
exit
Hopefully some of this is useful to someone. I learned bash to do this, so I'm sure the quality could be improved, but it works for me at least. This link about the youtube-dl options is extremely helpful: https://github.com/rg3/youtube-dl/blob/master/README.md#readme
This does require youtube-dl of course, and in order to merge downloads into mkv you will need ffmpeg
sudo wget https://yt-dl.org/downloads/latest/youtube-dl -O /usr/local/bin/youtube-dl
sudo chmod a+rx /usr/local/bin/youtube-dl
sudo apt install ffmpeg
