# Delete all duplicates in list of files supplied as arguments,

#!/bin/bash
# Delete all duplicates in list of files supplied as arguments,
# based on MD5 hash. Retains first copy of each file in
# alphanumerically sorted list of the input file names/paths.
# Example: rmdup.sh *.tar.bz2
# The Free Thinker, 2022.

oldhash=
OLDIFS="$IFS"
IFS=$'\n'
for line in `md5sum $@ | sort -k 1,1 | uniq -D -w 32`
do
hash=${line:0:32}
if [ "$hash" = "$oldhash" ]
then
file="${line:34}"
echo Deleting: "$file"
rm "$file"
fi
oldhash=$hash
done
IFS="$OLDIFS"