#/bin/bash
usage="\
$0 [-o OUTFILE] FILE1 [FILE2] [FILE3]...
Concatenate, sort and remove duplicate lines from a list of files.
If outfile isn't provided, will output to ./unduplicated.
"

test $# -eq 0 && echo -e "$usage\n\nError: At least one argument is needed" && exit 1

if test $1 = "-o" ; then
	outfile="$2"
	shift 2
else
	outfile="./unduplicated"
fi

test $# -eq 0 && echo -e "$usage\n\nError: At least one file is required" && exit 1

file="$@"
#echo $file

input=`mktemp`
output=`mktemp`
#echo -e "Unduplicator:\n\tinfiles: $file\n\tinput:\t$input\n\toutput: $output\n\t	outfile: $outfile"


sort "$file" > "$input"

#ls -lh $input $output
#i=0

while read -r line ; do
	#i=$((i+1))
#	echo "$i: $line ($file)"
	test "$line" != "$linee" &&	echo "$line" >> $output # || echo "duplicate found : $line"
	linee="$line"

done < "$input"

cat $output > $outfile
rm $input $output
exit 0