#!/usr/bin/perl
# VidProc
# Transcode videos that have been either ripped by ripper or copied from a TiVo
#    to H.264 format
# by Ed Salisbury (ed@edsalisbury.net)
# http://www.edsalisbury.net
# (c)2009 Ed Salisbury, Some Rights Reserved
#
# External Utilities Required:
# * HandBrakeCLI
# * MPlayer/Mencoder
#
# Notes:
# * Settings for HandBrake and mencoder are what I have come up with after
#   doing a fair bit of research, and seem to work pretty well - if you have
#   any concrete suggestions on *better* general-purpose settings, let me know.
#
# License:
# Except where otherwise noted, this work is licensed under Creative Commons
#   Attribution ShareAlike 3.0.
#
# You are free:
#   * to Share -- to copy, distribute and transmit the work
#   * to Remix -- to adapt the work
#
# Under the following conditions:
#   * Attribution. You must attribute the work in the manner specified by the
#     author or licensor (but not in any way that suggests that they endorse
#     you or your use of the work).
#   * Share Alike. If you alter, transform, or build upon this work, you may
#     distribute the resulting work only under the same, similar or a
#     compatible license.
#   * For any reuse or distribution, you must make clear to others the license
#     terms of this work. The best way to do this is with a link to the
#     license's web page (http://creativecommons.org/licenses/by-sa/3.0/)
#   * Any of the above conditions can be waived if you get permission from the
#     copyright holder.
#   * Nothing in this license impairs or restricts the author's moral rights.

use warnings;
use strict;
use File::Copy;
use File::Basename;

sub system_int($);

# Locations
my $DEST = "/path/to/destdir";
my $QUEUE = "/path/to/queue.txt";

# Encoding Options
my $DEINT = "pp=md";    # Deinterlacer
my $VBITRATE = 1000;    # Video Bitrate
my $SAMPLE_RATE = 44.1; # Audio Sample Rate
my $ABITRATE = 192;     # Audio Bitrate
my $OVERSCAN = 6;       # Overscan size (# of lines)

# File Extensions
my $SRC_EXT = "mpg";
my $DEST_EXT = "mp4";

# External Utilities
my $HANDBRAKE = "/usr/local/bin/HandBrakeCLI";
my $MENCODER = "/usr/bin/mencoder";
my $MPLAYER = "/usr/bin/mplayer";

# Add backslashes to spaces
$DEST =~ s/ /\\ /g;

while (1)
{
    # Read in the queue
    open(QUEUE, $QUEUE);
    my $line = <QUEUE>;
    my @fields;
    if ($line)
    {
        chomp ($line);
        @fields = split(/\|/, $line);
    }
    else
    {
        print "Waiting for new stuff to show up in the queue...\n";
        close(QUEUE);
        sleep(60);
        next;
    }
    close (QUEUE);

    my $input = $fields[0];

    print "Processing $input\n";

    # Fix spaces
    $input =~ s/ /\\ /g;
    $input =~ s/'/\\'/g;
    $input =~ s/&/\\&/g;
    $input =~ s/;/\\;/g;
    my @extlist = (".$SRC_EXT");

    my $base = basename($input, @extlist);

    if ($fields[1])
    {
        # DVD, since it has a title number
        my $title = $fields[1];
        my $output = "$DEST/${base}_" . $fields[1] . ".$DEST_EXT";

        system_int("$HANDBRAKE --input $input --output $output --title $title --turbo --encoder x264 --vb $VBITRATE --audio 1 --aencoder faac --mixdown 6ch --arate $SAMPLE_RATE --ab $ABITRATE --detelecine --decomb --loosePixelratio --markers --two-pass --x264opts ref=3:mixed-refs:bframes=6:weightb:direct=auto:b-pyramid:me=umh:subme=9:analyse=all:8x8dct:trellis=1:nr=150:no-fast-pskip=1:psy-rd=1,1");
    }
    else
    {
        # TiVo file, since it has no title number
        my $output = "$DEST/${base}.$DEST_EXT";
        my $line;
        my $width;
        my $height;

        # Get video dimensions
        print "Getting video information... ";
        my @id = `$MPLAYER -nojoystick -nolirc -vo null -ao null -identify -frames 0 $input 2>&1`;
        foreach (@id)
        {
            if (/ID_VIDEO_WIDTH=(\d+)/)
            {
                $width = $1;
            }
            if (/ID_VIDEO_HEIGHT=(\d+)/)
            {
                $height = $1;
            }
        }
        if (!$width || !$height)
        {
            print "FAILED!";
        }
        else
        {
            print "(${width}x$height)\n";
        }

        # Crop overscan
        $height-=$OVERSCAN;

        my %croptest;

        # Detect crop region
        print "Detecting crop region... ";
        my @cropdetect = `$MPLAYER -nojoystick -nolirc -vo null -ao null -vf crop=$width:$height:0:$OVERSCAN,cropdetect -ss 600 -endpos 180 $input 2>&1`;
        foreach $line (@cropdetect)
        {
            if ($line =~ /-vf crop=([\d:]+)/)
            {
                $croptest{$1}++;
            }
        }

        # Get the crop value with the most hits
        my @croplist = sort { $croptest{$b} <=> $croptest{$a} } keys (%croptest);
        my $crop = $croplist[0];

        if (!$crop)
        {
            print "FAILED\n";
            exit();
        }
        print "($crop)\n";
        print '-' x 80 . "\n";

        print "Encoding first pass\n";
        print '-' x 80 . "\n";
        system_int("$MENCODER -ovc x264 -x264encopts pass=1:turbo:bitrate=$VBITRATE:bframes=1:me=umh:partitions=all:trellis=1:qp_step=4:qcomp=0.7:direct_pred=auto:keyint=300 -vf $DEINT,crop=$crop,scale=-1:-10,harddup -oac copy -ofps 30000/1001 $input -o /dev/null");

        print "\n\n";
        print '-' x 80 . "\n";
        print "Encoding second pass\n";
        print '-' x 80 . "\n";
        system_int("$MENCODER -ovc x264 -x264encopts pass=2:turbo:bitrate=$VBITRATE:bframes=1:me=umh:partitions=all:trellis=1:qp_step=4:qcomp=0.7:direct_pred=auto:keyint=300 -vf $DEINT,crop=$crop,scale=-1:-10:,harddup -oac copy -ofps 30000/1001 $input -o $output");
    }
    print "Done. Removing from queue\n";
    print "\n\n";
    open(IN, $QUEUE);
    open(OUT, ">$QUEUE.tmp");
    while (my $qline = <IN>)
    {
        chomp ($qline);
        unless ($line eq $qline)
        {
            print OUT "$qline\n";
        }
    }
    close (IN);
    close (OUT);
    unlink($QUEUE);
    move("$QUEUE.tmp", $QUEUE);
}

# Interruptible System Command
sub system_int($)
{
    my ($cmd) = @_;
    my $pid = fork();
    my $rc;
    if ($pid == 0)
    {
        exec($cmd);
    }
    else
    {
        waitpid($pid,0);
        $rc = $?;
    }
    return $rc;
}

It uses the same queue file that the other scripts will write to, so think of this as a continuation of the previous articles. To use it, stick it somewhere like /usr/local/bin, and then change the locations to be wherever your queue is, and where you want your videos to end up. The transcoding settings can get a little hairy, so I would say don’t mess with the actual command lines unless you know what you’re doing (but, if you did, you probably wouldn’t need this script!)

External Utilities Needed:

HandBrakeCLI – this can be downloaded from here
mplayer/mencoder – This can be installed via apt-get

A couple of Notes/FAQs:

Overscan removal:
If you deal with letterboxed videos copied from TV, you’ll know what a pain overscan can be – Overscan is the generic term for the fuzzy line above the video that gets really annoying when watching on your monitor, but not on your TV. This can wreak havoc with detecting where to crop the video, so I remove it.

HandBrake *AND* mencoder?? Why?
I found that for some reason, Handbrake didn’t like working with the files that were copied from the TiVo – I dealt with lots of weird sync issues. I posted it on the Handbrake forum, but never got a solution. I ended up just using mencoder for the TiVo files, and Handbrake for the DVD rips, and left it at that.

Your setting XXX sucks – you should use YYY!
If you can state that for any type of program (animation, TV, letterboxed, HD, etc.) that this is correct (and that it doesn’t make the process take much longer), I’ll happily thank you and update the script (as well as give you a mention here.) I spent a bit of time trying to find settings that would work well for what I wanted, and ended up choosing these. They’re not going to have the best quality, but the size and the time to transcode were good, so that’s what I stuck with.

Information you will need:

• Your TiVo’s IP Address (needs to be static)
• Your TiVo’s Media Access Key (MAK) – this can be found on tivo.com after you’ve logged in
• The location where you want your files to be dumped (they are *big* – figure 1 GB per hour for SD, and like 5 GB per hour for HD)
• The location where the files will eventually be stored

The external utilities you will need:

• Curl – Install via sudo apt-get install curl
• Tivodecode – download from Sourceforge

You will also need to install the following CPAN perl modules:

• Net::TiVo
• XML::Simple
• Text::Unidecode

After you have all of that stuff installed and info gathered, it’s time to install my tivo_dump script:

#!/usr/bin/perl
# TiVo Dump
# Copy TiVo programs from a Series3 TiVo
# by Ed Salisbury (ed@edsalisbury.net)
# http://www.edsalisbury.net
# (c)2009 Ed Salisbury, Some Rights Reserved
#
# External Utilities Required:
# * curl
# * tivodecode
#
# External Perl Modules Required:
# * Net::TiVo
# * XML::Simple
# * Text::Unidecode;
#
# Usage:
# tivo_dump
#
# License:
# Except where otherwise noted, this work is licensed under Creative Commons
#   Attribution ShareAlike 3.0.
#
# You are free:
#   * to Share - to copy, distribute and transmit the work
#   * to Remix - to adapt the work
#
# Under the following conditions:
#   * Attribution. You must attribute the work in the manner specified by the
#     author or licensor (but not in any way that suggests that they endorse
#     you or your use of the work).
#   * Share Alike. If you alter, transform, or build upon this work, you may
#     distribute the resulting work only under the same, similar or a
#     compatible license.
#   * For any reuse or distribution, you must make clear to others the license
#     terms of this work. The best way to do this is with a link to the
#     license's web page (http://creativecommons.org/licenses/by-sa/3.0/)
#   * Any of the above conditions can be waived if you get permission from the
#     copyright holder.
#   * Nothing in this license impairs or restricts the author's moral rights.

use warnings;
use strict;
use Net::TiVo;
use XML::Simple;
use utf8;
use Text::Unidecode;

sub fix_chars($);

# User-Configurable Variables
my $HOST = "";      # Hostname/IP of the TiVo
my $MAK = "";       # The Media Access Key of the TiVo
my $VIDEO_DIR = ""; # Where the programs get saved
my $COPY_SUGGESTIONS = 0; # If you want to copy "Suggested" programs, set this to 1

my $USER = "tivo";
my $TMPFILE = "/tmp/$$.xml";
my $PROGRAMS_FILE = "/home/username/.tivo_programs";
$|++;

# External Utilities
my $CURL = "/usr/bin/curl";
my $TIVODECODE = "/usr/local/bin/tivodecode";

my @PREV;

# Connect to the TiVo
print "Connecting to TiVo at $HOST... ";
my $tivo = Net::TiVo->new(host => $HOST, mac => $MAK);
my @folders = $tivo->folders();
if (@folders)
{
    print "OK\n";
}
else
{
    print "FAIL\n";
    exit();
}

# Load the file that has the previously saved programs
open(IN, $PROGRAMS_FILE);
while (<IN>)
{
    chop();
    push(@PREV, $_);
}
close (IN);

# Go through each folder on the TiVo
foreach my $folder (@folders)
{
    foreach my $item ($folder->{'xmlref'}{'Item'})
    {
        foreach my $video (@$item)
        {
            # Only process videos, not folders
            if ($video->{'Links'}{'Content'}{'ContentType'} eq "video/x-tivo-raw-tts")
            {
                # Choose video type based on the icon
                if ($video->{'Links'}{'CustomIcon'} && $video->{'Links'}{'CustomIcon'}{'Url'} eq "urn:tivo:image:suggestion-recording" && !$COPY_SUGGESTIONS)
                {
                    next;
                }
                if ($video->{'Links'}{'CustomIcon'} &&
                   ($video->{'Links'}{'CustomIcon'}{'Url'} eq "urn:tivo:image:in-progress-transfer" ||
                    $video->{'Links'}{'CustomIcon'}{'Url'} eq "urn:tivo:image:in-progress-recording"))
                {
                   next;
                }

                # Get program and episode titles
                my $program_title = $video->{'Details'}{'Title'};

                if ($video->{'Details'}{'EpisodeTitle'})
                {
                    $program_title .= " - " . $video->{'Details'}{'EpisodeTitle'};
                }

                # Get Program ID and Video URL
                my $video_url = $video->{'Links'}{'Content'}{'Url'};
                my $program_id = $video->{'Details'}{'ProgramId'};

                if (!$program_id)
                {
                    next;
                }

                # If previously copied, skip
                if (grep /^$program_id$/, @PREV)
                {
                    print "Skipping $program_title.\n";
                    next;
                }

                # get details XML file
                print "Getting details for $program_title... ";
                my $details_xml = $video->{'Links'}{'TiVoVideoDetails'}{'Url'};

                system("$CURL --digest -s -k -u $USER:$MAK -c /tmp/cookies.txt -o $TMPFILE \"$details_xml\"");
                if (-f $TMPFILE)
                {
                    print "OK\nProcessing details file... ";
                    my $xml = XML::Simple->new();
                    my $doc = $xml->XMLin($TMPFILE);
                    my %meta;
                    my $filepath;
                    my $filename;

                    # Get rating and convert to proper form
                    $meta{'tvRating'} = $doc->{'showing'}{'tvRating'}{'content'};
                    if ($meta{'tvRating'})
                    {
                        if ($meta{'tvRating'} eq "Y_7") { $meta{'tvRating'} = 'x1'; }
                        elsif ($meta{'tvRating'} eq "PG") { $meta{'tvRating'} = 'x4'; }
                        elsif ($meta{'tvRating'} eq "_14") { $meta{'tvRating'} = 'x5'; }
                        else { $meta{'tvRating'} = "x7"; }
                    }

                    # Get other data
                    $meta{'vActor'} = $doc->{'showing'}{'program'}{'vActor'}{'element'};
                    $meta{'vDirector'} = $doc->{'showing'}{'program'}{'vDirector'}{'element'};
                    $meta{'vProgramGenre'} = $doc->{'showing'}{'program'}{'vProgramGenre'}{'element'};
                    $meta{'vSeriesGenre'} = $doc->{'showing'}{'program'}{'series'}{'vSeriesGenre'}{'element'};
                    $meta{'seriesTitle'} = $doc->{'showing'}{'program'}{'series'}{'seriesTitle'};
                    $meta{'title'} = $doc->{'showing'}{'program'}{'title'};
                    $meta{'isEpisode'} = $doc->{'showing'}{'program'}{'isEpisode'};
                    $meta{'originalAirDate'} = $doc->{'showing'}{'program'}{'originalAirDate'};
                    $meta{'episodeTitle'} = $doc->{'showing'}{'program'}{'episodeTitle'};
                    $meta{'description'} = $doc->{'showing'}{'program'}{'description'};

                    if (defined $meta{'description'})
                    {
                        $meta{'description'} =~ s/\s+Copyright.*$//;
                    }

                    # Process Titles

                    $meta{'episodeNumber'} = $doc->{'showing'}{'program'}{'episodeNumber'};

                    my $series_title = '';
                    my $episode_number = '';
                    my $episode_title = '';
                    my $title = '';

                    if ($meta{'seriesTitle'})
                    {
                        $series_title = fix_chars($meta{'seriesTitle'});
                    }
                    if ($meta{'episodeNumber'})
                    {
                        $episode_number = fix_chars($meta{'episodeNumber'});
                    }
                    if ($meta{'episodeTitle'})
                    {
                        $episode_title = fix_chars($meta{'episodeTitle'});
                    }
                    if ($meta{'title'})
                    {
                        $title = fix_chars($meta{'title'});
                    }

                    if ($series_title && $episode_number && $episode_title)
                    {
                        $filepath = "$VIDEO_DIR/$series_title";
                        $filename = "$filepath/$series_title - $episode_number - $episode_title";
                    }
                    elsif ($series_title && $episode_title)
                    {
                        $filepath = "$VIDEO_DIR/$series_title";
                        $filename = "$filepath/$series_title - $episode_title";
                    }
                    elsif ($title)
                    {
                        $filepath = "$VIDEO_DIR/$title";
                        $filename = "$filepath/$title";
                    }
                    else
                    {
                        $filepath = "$VIDEO_DIR";
                        $filename = "$filepath/Unknown";
                    }
                    print "OK\n";

                    unless (-d $filepath)
                    {
                        print "Path $filepath doesn't exist, creating... ";
                        mkdir($filepath);
                        if (-d $filepath)
                        {
                            print "OK\n";
                        }
                        else
                        {
                            print "FAIL\n";
                            exit;
                        }
                    }

                    # Get the video with curl
                    print "Getting video... ";
                    system("$CURL --digest -s -k -u $USER:$MAK -c /tmp/cookies.txt -o \"$filename.tivo\" \"$video_url\"");
                    if (-f "$filename.tivo")
                    {
                        my $filesize = (stat("$filename.tivo"))[7];

                        if ($filesize > 0)
                        {
                            print "OK\n";

                            # Convert to MPG
                            print "Converting video to MPG format... ";
                            system("$TIVODECODE -m $MAK -o \"$filename.mpg\" \"$filename.tivo\" > /dev/null 2>&1");
                            if (-f "$filename.mpg")
                            {
                                print "OK\n";
                                unlink "$filename.tivo";
                                open(OUT, ">>$PROGRAMS_FILE");
                                print OUT "$program_id\n";
                                close(OUT);
                            }
                            else
                            {
                                print "FAIL\n";
                                exit();
                            }

                            # Output metadata file
                            print "Outputting metadata... ";
                            open (OUT, ">$filename.mpg.txt");

                            foreach my $key (keys %meta)
                            {
                                if ($meta{$key})
                                {
                                    if ($meta{$key} =~ /^ARRAY/)
                                    {
                                        foreach my $item (@{$meta{$key}})
                                        {
                                            unless ($item =~ /^HASH/)
                                            {
                                                print OUT "$key : " . fix_chars($item) . "\n";
                                            }
                                        }
                                    }
                                    else
                                    {
                                        unless ($item =~ /^HASH/)
                                        {
                                            if ($key eq "originalAirDate")
                                            {
                                                print OUT "$key : " . $meta{$key} . "\n";
                                            }
                                            else
                                            {
                                                print OUT "$key : " . fix_chars($meta{$key}) . "\n";
                                            }
                                        }
                                    }
                                }
                            }
                            unlink($TMPFILE);
                            close(OUT);
                            print "OK\n";
                        }
                        else
                        {
                            print "FAIL\n";
                        }
                    }
                    else
                    {
                        print "FAIL\n";
                        exit();
                    }
                }
                else
                {
                    print "FAIL\n";
                    exit();
                }
            }
        }
    }
}

# Convert any offending characters
sub fix_chars($)
{
    my ($data) = @_;

    $data = unidecode($data);

    $data =~ s/\:/ -/g;
    $data =~ s/\//-/g;
    $data =~ s/\\/-/g;
    $data =~ s/\?/-/g;
    $data =~ s/\*/-/g;

    return $data;
}

Put the script wherever you want it – (I usually put my scripts into /usr/local/bin) – Edit the script, and fill in the $HOST, $MAK, and $VIDEO_DIR variables with the data gathered in step 1. Then do a test run. It will take a while to copy the data. If it’s working fine, add to cron to have it copy the files regularly. I copy mine at midnight:

0 0 * * * /usr/local/bin/tivo_dump >/dev/null 2>&1

This says, run every day at midnight and send the output to /dev/null (otherwise you’ll get emails every day, which you may or may not want).

Note: Most of this script deals with metadata – it writes a text file (filename).txt with the metadata about the program, which is in pyTivo format. For more info, see my guide on how to set up pyTivo as a media server for TiVos.

First, install ffmpeg:

$ sudo apt-get install ffmpeg

Then, install git (needed to grab pyTivo):

$ sudo apt-get install git-core

Then, grab the latest version of the pyTivo script (Look at the Current Release page to verify that you have the right version):

$ cd /usr/share
$ sudo git clone git://repo.or.cz/pyTivo/wmcbrine.git
$ sudo mv wmcbrine pyTivo

Edit /usr/share/pyTivo/pyTivo.conf:

$ cd /usr/share/pyTivo
$ sudo cp pyTivo.conf.dist pyTivo.conf
$ sudo vi pyTivo.conf

The only thing that I changed is to comment out the default video share, and add my own:

[Movies]
type=video
path=/data/Video/Movies
[Television]
type=video
path=/data/Video/Television
[Music]
type=music
path=/data/Audio/Music

Run pyTivo:

$ sudo python /usr/share/pyTivo/pyTivo.py

Verify that things are working correctly — on the Tivo, go to “Now Playing List” and look for the shares.  If they appear and you can browse to them, you’re almost done!

One issue I had was with transferring the videos.  When I transferred them to the TiVo, it had a message of “unknown mpeg2 codec” or something to that effect.  Apparently the package for ffmpeg on Intrepid doesn’t have all of the codecs needed.  To install them, run the following:

$ sudo apt-get install libavcodec-unstripped-51

After I did this, videos were able to be transferred just fine.

The last thing to do is to make pyTiVo start automatically on bootup. To do this, simply copy the following script to /etc/init.d:

#!/bin/bash
# chkconfig: 2345 99 05
# description: pyTivo server

### INIT INFO
# Provides: pytivo
# Required-Start: $network
# Required-Stop: $network
# Default-Start: 2 3 4 5
# Default-Stop: 0 1 6
# Short-description: pyTivo server
# Description: Start and stop the pyTivo server.
### END INIT INFO

RETVAL=0

start() {
echo -n "Starting pyTivo: "
pgrep -f pyTivo.py
RETVAL=$?
[ $RETVAL -eq 0 ] && echo "pyTivo already running: Exiting" && exit 1

# this call actually starts pyTivo.
python /usr/share/pyTivo/pyTivo.py > /dev/null 2>&1 &
RETVAL=$?
[ $RETVAL -eq 0 ] && echo -n "done"
echo
return $RETVAL
}

stop() {
echo -n "Stopping pyTivo: "
pkill -f pyTivo.py
RETVAL=$?
echo
[ $RETVAL -eq 0 ] && echo -n "done"
echo
return $RETVAL
}

# See how we were called.
case "$1" in
start)
start
;;
stop)
stop
;;
restart|reload)
stop
sleep 1
start
RETVAL=$?
;;
*)
echo "Usage: $0 {start|stop|restart}"
exit 1
esac
exit $RETVAL

Note: I didn’t write this – I just grabbed it from somewhere when setting it up – let me know if you wrote it so that I can give you credit!

After creating the script, make it executable:

$ sudo chmod u+x /etc/init.d/pytivo

Then create links to /etc/rc3.d and /etc/rc1.d:

$ sudo ln -s /etc/init.d/pytivo /etc/rc3.d/S99pytivo
$ sudo ln -s /etc/init.d/pytivo /etc/rc1.d/K99pytivo

This should make it so that pytivo starts automatically upon bootup.

Hopefully you’ve found this guide useful. Next time, I’ll be talking about how to make the videos show up perfectly on the TiVo, with appropriate titles, etc.

1. Getting the dimensions of the video
2. Cropping letterbox and overscan
3. Encoding the video

I had a *lot* of issues with AV Sync as well as dropped frames using HandBrake (that no one seemed to know why) so I went with mencoder.  I also left the audio as-is instead of transcoding into AAC.  It’s not a “proper” h.264 file with AC-3, but it works for my purposes.

First off, we need to figure out the dimensions of the video.  The reason is, we want to crop off the timecode/overscan line on the top of the video so that 1. it looks better, and 2. we can actually use cropdetect to figure out where the black bars are.   Here’s how to get video dimensions:

mplayer -nojoystick -nolirc -vo null -ao null -identify -frames 0 $input

The options I’m using:

• -nojoystick – prevents mplayer spewing out warnings about joysticks
• -nolirc – same thing here – we’re not using LIRC, so don’t warn us
• -vo null – No video output device
• -ao null – No audio output device
• -identify – Spit out the info
• -frames 0 – We don’t need to actually look at any frames, so we set to 0
• $input – The input file

This will output quite a few interesting lines — the ones we care about are ID_VIDEO_WIDTH and ID_VIDEO_HEIGHT. These tell us the dimensions of the video.

For the next step, we’re going to want to strip off the overscan line — to do this, we subtract 6 pixels from the height, (I’ll use a variable called $OVERSCAN) and then send that to mplayer again, this time to detect the crop area that we want:

$height -= $OVERSCAN;
mplayer -nojoystick -nolirc -vo null -ao null -vf crop=$width:$height:0:$OVERSCAN,cropdetect -ss 600 -endpos 180 $input

The new options here are:

• -vf crop – specify the crop area like this width:height:x:y – x and y are the starting position from the left/top – since we want to pre-crop off the overscan, we have to remove from height as well as start from that position.
• cropdetect – This tells mplayer to spit out a line for every frame saying where it thinks you’ll want to crop the video
• -ss 600 – This tells it to start 600 seconds into the video – I wanted to get the middle somewhere, just because the file could start with the end of the previous program, etc.
• -endpos 180 – This says to stop 180 seconds after it starts (bad name in my opinion) – I want to grab 3 minutes worth of video, in case we happen to be in the middle of a commercial break

This will spit out a ton of lines with crop info, and we want to grab the most common crop sizes.  To do this, I use some perl:

foreach $line (@cropdetect)
{
    if ($line =~ /-vf crop=([\d:]+)/)
    {
        $croptest{$1}++;
    }
}

# Get the crop value with the most hits
my @croplist = sort { $croptest{$b} <=> $croptest{$a} } keys (%croptest);
my $crop = $croplist[0];

This will grab the crop value that’s the most popular.  (The sort line brings it to the first element in the array)

Now that we know where we want to crop, we can begin encoding.  I use a 2-pass encoding method, which is the accepted standard for great quality encoding.  You can go with 1-pass or more than 2, but I think 2 is the sweet spot.  A lot of these options I’m not actually sure what they do, but will explain the ones that I do know about.

mencoder -ovc x264 -x264encopts pass=1:turbo:bitrate=$VBITRATE:bframes=1:me=umh:partitions=all:trellis=1:qp_step=4:qcomp=0.7:direct_pred=auto:keyint=300 -vf pp=ci,crop=$crop,scale=-1:-10,harddup -oac copy -ofps 30000/1001 $input -o /dev/null

Options:

• -ovc x264 – Use x264 for the Output Video Codec
• -x264encopts – Specify options to pass to x264
• pass=1 – 1st pass
• bitrate=$VBITRATE – I set $VBITRATE to 1500 – which is 1500 kbits, you can go less for smaller files/lower quality
• pp=ci – Deinterlacing – tivo files are interlaced, so I wanted to get rid of that
• crop=$crop – use the crop value that we determined earlier
• scale=-1:-10 – This will downscale the video so that it uses sizes divisible by 16 – there’s encoding and even playback issues if you don’t do this.  You can upscale by using -10:-1, but I didn’t really see the point in upscaling.
• harddup – This means that you want to duplicate frame by frame (hard duplicate) – if you don’t, you’ll run into A/V sync issues
• -oac copy – Audio is passthrough – don’t touch it
• -ofps 30000/1001 – Hard sets the FPS to 29.97 – I didn’t want the frame rate to jump back and forth like it does, so I wanted to hard set this – haven’t found a good reason not to
• -o /dev/null – The first pass will just log to a logfile, it won’t write to the output file

After the first pass is done, we do another pass, which has almost the same parameters:

mencoder -ovc x264 -x264encopts pass=2:turbo:bitrate=$VBITRATE:bframes=1:me=umh:partitions=all:trellis=1:qp_step=4:qcomp=0.7:direct_pred=auto:keyint=300 -vf pp=ci,crop=$crop,scale=-1:-10:,harddup -oac copy -ofps 30000/1001 $input -o $output

New options:

• pass=2 – 2nd pass
• -o $output – this time output to our output file

When this is completed, you should have a video file with h.264 for the video, and AC3 for the audio.  This resulted in file sizes at about 60% of the originals, which is good enough for me.  I hope someone found this useful.  I’ll be putting up a script soon that will do all of this work for you (and even work from a queue), but till then, I wanted to get the details out there.

Update: I have created another guide that includes the script that I use here.