43 lines
890 B
Bash
43 lines
890 B
Bash
#/bin/bash
|
|
usage="\
|
|
$0 [-o OUTFILE] FILE1 [FILE2] [FILE3]...
|
|
Concatenate, sort and remove duplicate lines from a list of files.
|
|
If outfile isn't provided, will output to ./unduplicated.
|
|
"
|
|
|
|
test $# -eq 0 && echo -e "$usage\n\nError: At least one argument is needed" && exit 1
|
|
|
|
if test $1 = "-o" ; then
|
|
outfile="$2"
|
|
shift 2
|
|
else
|
|
outfile="./unduplicated"
|
|
fi
|
|
|
|
test $# -eq 0 && echo -e "$usage\n\nError: At least one file is required" && exit 1
|
|
|
|
file="$@"
|
|
#echo $file
|
|
|
|
input=`mktemp`
|
|
output=`mktemp`
|
|
#echo -e "Unduplicator:\n\tinfiles: $file\n\tinput:\t$input\n\toutput: $output\n\t outfile: $outfile"
|
|
|
|
|
|
sort "$file" > "$input"
|
|
|
|
#ls -lh $input $output
|
|
#i=0
|
|
|
|
while read -r line ; do
|
|
#i=$((i+1))
|
|
# echo "$i: $line ($file)"
|
|
test "$line" != "$linee" && echo "$line" >> $output # || echo "duplicate found : $line"
|
|
linee="$line"
|
|
|
|
done < "$input"
|
|
|
|
cat $output > $outfile
|
|
rm $input $output
|
|
exit 0
|