#!/usr/bin/perl
# Usage in command line: perl duplicate_remove.perl <file-to-be-convertet>
# Remove duplicated lines in text-files

my $origfile = $ARGV[0];
my $outfile  = "no_duplicates_" . $origfile; 
my %hTmp;
 
open (IN, "<$origfile")  or die "Couldn't open input file: $!"; 
open (OUT, ">$outfile") or die "Couldn't open output file: $!"; 
 
while (my $sLine = <IN>) {
  next if $sLine =~ m/^\s*$/;  #remove empty lines
                               #Without the above, still destroys empty lines except for the first one.
  print OUT $sLine unless ($hTmp{$sLine}++);
}
close OUT;
close IN;
print "The file $origfile is convertet! Look for $outfile in the same directory";

Name the source duplicate_remove.perl and open Terminal.app in the same directory, and write: perl duplicate_remove.perl <file-to-be-convertet>

Remove duplicated lines with Perl

command

Perl