[拙作]一个基于md5hash的文件查重器

Jockey

1. #!/bin/sh
2. # Filename: ckdup.sh
3. # Author: Jockey dot Kyd at gmail dot com
4. # Date: Mar.18th, 2007
5. # Licence: No licence. Feel free to do whatever you want.
6. #           But NO WARRANTY. Use it on your own risk.
7. #
8. # This script aims to check duplicate files based on their
9. # MD5 hash. Although collisions have been found with MD5
10. # Message-Digest algorithm, I believe that it's extremely
11. # difficult to find different files have a same hash, if
12. # possible after all.
13. #
14. # The script accepts three options and they should be placed
15. # immediately after the script name. For each option except
16. # '-h', it's mandatory to specify at least one file to check.
17. #
18. #        -h        Print usage infomation.
19. #        -r        Recursive mode. No effect with '-h'.
20. #        -v        Verbose output to tell which file is being
21. #                checked. No effect with '-h'.
22. #
23. # In output, the duplicate files are put together. The delimiter
24. # between different groups is a blank line.
25. #
26. # NOTE: The script will ignore hidden files unless you specify.
27. #        However, they'll be ignored anyway in recursion. It's
28. #        NOT recommended to include any hidden file under any
29. #        circumstances.
30. #
31. # To avoid converting delimiters into spaces automatically.
32. IFS=''
34. usage ()
35. {
36.         echo "usage: ""$0"" [-hrv] file..." 1>&2
37.         exit 1
38. }
40. # This is the wrapper function to generate output needed by
41. # genraw() in the format of '<MD5HASH> <FILENAME>'. One and
42. # only one command as a piece of wrapped code should be placed
43. # here, depending on what tool and OS you're using. Writing
44. # your own command is a matter of course if none fits.
45. genmd5 ()
46. {
47.         # For md(1) on Mac OS X, FreeBSD and OpenBSD-CURRENT
48.         md5 -r "$@"
50.         # For md(1) on NetBSD
51.         #md5 -n "$@"
53.         # For md5sum(1) from FSF on GNU/Linux
54.         #md5sum "$@"
55. }
57. # This is the function to generate variable containing raw
58. # output which will be further processed later.
59. genraw ()
60. {
61.         for i in "$@"; do
62.                 if [ ! -e "$i" ]; then
63.                         echo "$0"": missing file: ""$i" 1>&2
64.                         continue
65.                 elif "$recur" && [ \( -d "$i" \) -a \( -x "$i" \) ]; then
66.                         i=$(echo $i | sed 's/\/$//')
67.                         genraw "$i"/*
68.                 elif [ -f "$i" ]; then
69.                         if "$verbose"; then
70.                                 echo "Checking: ""$i"
71.                         fi
72.                         raw=$(genmd5 "$i")"\n""$raw"
73.                 fi
74.         done
75. }
77. recur=false
78. help=false
79. verbose=false
80. while getopts ':rhv' opt; do
81.         case "$opt" in
82.                 r )
83.                         recur=true ;;
84.                 v )
85.                         verbose=true ;;
86.                 h | \? )
87.                         help=true ;;
88.         esac
89. done
90. shift $(($OPTIND - 1))
91. if "$help"; then
92.         usage
93. elif [ "$#" = 0 ]; then
94.         echo "$0"": no files to check." 1>&2
95.         usage
96. fi
98. # Call the function genraw() to generate the variable.
99. # NOTE: If the wildcard cannot be expanded, the script
100. #        will give a warning of missing file but never
101. #        mind, since you're not in bad case, just ignore
102. #        it.
103. genraw "$@"
105. # Regenerate the variable's value as a whole.
106. raw=$(echo -e "$raw")
108. # This variable contains same MD5 hash of duplicate files
109. # line by line.
110. dup=$(echo "$raw" | awk '{print $1}' | sort | uniq -c | awk '$1 != 1 { print $2 }')
112. # If no files are duplicate, exit normally.
113. if [ -z "$dup" ]; then
114.         if "$verbose";then
115.                 echo -e '\n'
116.         fi
117.         echo "No duplicate files found."
118.         exit 0
119. fi
121. isfirst=true
122. echo "$dup" | {
123.         while read i; do
124.                 if "$isfirst"; then
125.                         if "$verbose"; then
126.                                 echo -e "\n"
127.                         fi
128.                         echo "The following files are duplicate:"
129.                         isfirst=false
130.                 fi
131.                 echo
132.                 echo "$raw" | grep "$i" | awk '{
133.                                                         i = 2
134.                                                         j = $i
135.                                                         while (i<NF) {
136.                                                                 i++
137.                                                                 j = j" "$i
138.                                                         }
139.                                                         print j
140.                                                 }'
141.         done
142. }
144. exit 0

复制代码

[color="Red"]注意：genmd5()需要按照自己系统的实际情况进行修改。我是在Mac OS X上写的，所以默认工具用的是 md5(1) 。
为了排版我把源文件上传在附件中...我是菜鸟，注释写得很糟糕，代码中潜在的bug也没找不出来（如果存在的话）...交流为主便不计较这么多。各位多多包涵。:thank
使用示例：

晨想

看效果图不错。
建议：可以加上个 SHA1SUM，这样两个一起比较，冲撞率基本应该就是0了。

Jockey

Post by 晨想
建议：可以加上个 SHA1SUM，这样两个一起比较，冲撞率基本应该就是0了。

所以我提供了genmd5()这个函数wrapper供个人修改，除了函数本身名字牵强外，使用任何cksum算法的程序都能放进去（如果要两个cksum算法同时起作用，那么就得一个文件一个文件处理，把两个hash拼接成段，这无关紧要，因为hash不会输出，用户可以忽略内部运行的细节），只要保证cksum程序的输出格式是（没有“<”和“>”符号）：

1. <HASH码> <文件名>

复制代码

这个脚本还能改进很多，譬如前几天我在递归一个很深的目录时想到可以添加一个“-l <num>”选项指定递归的目录深度；作为Gentoo的饭，也可以加一个“-c”选项加入ansi转义码后多色彩输出，会更醒目一些。