|
0
|
1 #! /bin/sh |
|
|
2 |
|
|
3 # fetch a blacklist dcc.dcc-servers.net DCC servers |
|
|
4 |
|
|
5 # This script should be run about twice an hour at an arbitrary time |
|
|
6 # instead of 0,30 to minimize pile-ups on the FTP server. |
|
|
7 |
|
|
8 # -h homedir set DCC home directory |
|
|
9 |
|
|
10 # -o blklist file of local blacklist entries |
|
|
11 |
|
|
12 # -p pgm fetch, wget, curl, or ftp |
|
|
13 |
|
|
14 # -s srvr-host host name of source of the common blacklist |
|
|
15 |
|
|
16 # Rhyolite Software DCC 1.3.103-1.50 $Revision$ |
|
|
17 # @configure_input@ |
|
|
18 |
|
|
19 |
|
|
20 # other, probably local blacklist files |
|
|
21 OTHER_BLS= |
|
|
22 |
|
|
23 exec </dev/null |
|
|
24 |
|
|
25 HTTP_REFERER=DCC-1.3.103-script; export HTTP_REFERER |
|
|
26 |
|
|
27 DCC_HOMEDIR=@prefix@ |
|
|
28 SRVR_BL=dcc-servers-blacklist |
|
|
29 SRVRS= |
|
|
30 PGM=@FETCH_CMD@ |
|
|
31 FORCE= |
|
|
32 MIRROR= |
|
|
33 |
|
|
34 USAGE="`basename $0`: [-xf] [-h homedir] [-o blklist] [-p pgm] [-s srvr-host]" |
|
|
35 while getopts "xfh:o:p:s:m:" c; do |
|
|
36 case $c in |
|
|
37 x) set -x;; |
|
|
38 f) FORCE=yes;; |
|
|
39 h) DCC_HOMEDIR="$OPTARG";; |
|
|
40 o) OTHER_BLS="$OTHER_BLS $OPTARG";; |
|
|
41 p) PGM=$OPTARG;; |
|
|
42 s) SRVRS="$SRVRS $OPTARG";; |
|
|
43 m) MIRROR=$OPTARG;; |
|
|
44 *) echo "$USAGE" 1>&2; exit 1;; |
|
|
45 esac |
|
|
46 done |
|
|
47 shift `expr $OPTIND - 1 || true` |
|
|
48 if test "$#" -ne 0; then |
|
|
49 echo "$USAGE" 1>&2; exit 1 |
|
|
50 fi |
|
|
51 if test -z "$SRVRS"; then |
|
|
52 SRVRS="http://www.dcc-servers.net/dcc http://www.rhyolite.com/dcc ftp://ftp.dcc-servers.net ftp://ftp.rhyolite.com" |
|
|
53 fi |
|
|
54 URLS= |
|
|
55 for SRVR in $SRVRS; do |
|
|
56 if expr "$SRVR" : '.[hft]*tp://' >/dev/null; then |
|
|
57 if expr "$SRVR" : ".*/$SRVR_BL"'$' >/dev/null; then |
|
|
58 URLS="$URLS $SRVR" |
|
|
59 else |
|
|
60 URLS="$URLS $SRVR/$SRVR_BL" |
|
|
61 fi |
|
|
62 else |
|
|
63 URLS="$URLS ftp://$SRVR/$SRVR_BL" |
|
|
64 fi |
|
|
65 done |
|
|
66 |
|
|
67 # dccd expects this target |
|
|
68 TGT_BL=$DCC_HOMEDIR/blacklist |
|
|
69 NEW_BL=new-blacklist |
|
|
70 MARKER=fetch-failed |
|
|
71 BDIR=$DCC_HOMEDIR/$SRVR_BL |
|
|
72 FLOG=$BDIR/fetch-log |
|
|
73 |
|
|
74 if test ! -d $BDIR; then |
|
|
75 mkdir $BDIR |
|
|
76 fi |
|
|
77 cd $BDIR |
|
|
78 |
|
|
79 # use fetch, wget, curl, or ftp that understands URLs |
|
|
80 if test ! -s $SRVR_BL; then |
|
|
81 echo "# initial place keeper" >$SRVR_BL |
|
|
82 fi |
|
|
83 mv $SRVR_BL $SRVR_BL.old |
|
|
84 rm -f $FLOG |
|
|
85 BASE_PGM=`basename "$PGM"` |
|
|
86 if test "$BASE_PGM" = wget; then |
|
|
87 BASE_PGM= |
|
|
88 for URL in $URLS; do |
|
|
89 echo "$URL:" >>$FLOG |
|
|
90 # Do not use --mirror because -r results in a 0 exit status |
|
|
91 # even on failure. |
|
|
92 # Do not use --no-remove-listing, -nr, or --dont-remove-listing |
|
|
93 # because none of them are supported by all versions of wget. |
|
|
94 # At least some versions of wget exit with 0 after having done |
|
|
95 # nothing but emitting a usage message. |
|
|
96 if $PGM -nd --retr-symlinks -N --no-host-directories \ |
|
|
97 --passive-ftp @FETCH_WGET_OPTS@ $URL >>$FLOG 2>&1; then |
|
|
98 if test -s $SRVR_BL; then |
|
|
99 if test -z "`sed -n -e 2q \ |
|
|
100 -e 's/.*DOCTYPE.*/HTML/p' \ |
|
|
101 -e 's/<HEAD>/HTML/p' -e 's/<head>/HTML/p' \ |
|
|
102 $SRVR_BL`"; then |
|
|
103 break; |
|
|
104 fi |
|
|
105 # do not leave a broken file |
|
|
106 rm $SRVR_BL |
|
|
107 fi |
|
|
108 fi |
|
|
109 echo >>$FLOG |
|
|
110 done |
|
|
111 fi |
|
|
112 |
|
|
113 if test "$BASE_PGM" = fetch; then |
|
|
114 BASE_PGM= |
|
|
115 for URL in $URLS; do |
|
|
116 echo "$URL:" >>$FLOG |
|
|
117 $PGM -p -m $URL >>$FLOG 2>&1 |
|
|
118 if test -s $SRVR_BL; then |
|
|
119 break; |
|
|
120 fi |
|
|
121 echo >>$FLOG |
|
|
122 done |
|
|
123 fi |
|
|
124 |
|
|
125 if test "$BASE_PGM" = curl; then |
|
|
126 BASE_PGM= |
|
|
127 for URL in $URLS; do |
|
|
128 echo "$URL:" >>$FLOG |
|
|
129 $PGM @FETCH_CURL_OPTS@ --connect-timeout 30 --max-time 600\ |
|
|
130 $URL -o $SRVR_BL >>$FLOG 2>&1 |
|
|
131 # --fail does not work on at least some versions of curl |
|
|
132 if test -s $SRVR_BL; then |
|
|
133 if test -z "`sed -n -e 2q \ |
|
|
134 -e 's/.*DOCTYPE.*/HTML/p' \ |
|
|
135 -e 's/<HEAD>/HTML/p' -e 's/<head>/HTML/p' \ |
|
|
136 $SRVR_BL`"; then |
|
|
137 break; |
|
|
138 fi |
|
|
139 # do not leave a broken file |
|
|
140 rm $SRVR_BL |
|
|
141 fi |
|
|
142 echo >>$FLOG |
|
|
143 done |
|
|
144 fi |
|
|
145 |
|
|
146 if test "$BASE_PGM" = ftp; then |
|
|
147 BASE_PGM= |
|
|
148 for URL in $URLS; do |
|
|
149 echo "$URL:" >>$FLOG |
|
|
150 $PGM -p $URL >>$FLOG 2>&1 |
|
|
151 if test -s $SRVR_BL; then |
|
|
152 break; |
|
|
153 fi |
|
|
154 echo >>$FLOG |
|
|
155 done |
|
|
156 # if that did not work, try ancient FTP |
|
|
157 if test ! -s $SRVR_BL; then |
|
|
158 for URL in $URLS; do |
|
|
159 HOST=`expr "$URL" : "ftp://\([^/]*\)/"` |
|
|
160 RFILE=`expr "$URL" : "ftp://[^/]*/\(.*\)"` |
|
|
161 if test -z "$RFILE" -o -z "$HOST"; then |
|
|
162 continue |
|
|
163 fi |
|
|
164 echo "$URL:" >>$FLOG |
|
|
165 (echo "user anonymous `hostname`"; echo "get $RFILE $SRVR_BL") \ |
|
|
166 | ftp -n $HOST >>$FLOG 2>&1 |
|
|
167 if test -s $SRVR_BL; then |
|
|
168 break; |
|
|
169 fi |
|
|
170 # some versions of ftp like to leave empty files |
|
|
171 rm -f $SRVR_BL |
|
|
172 echo >>$FLOG |
|
|
173 done |
|
|
174 fi |
|
|
175 fi |
|
|
176 |
|
|
177 if test -s $SRVR_BL; then |
|
|
178 rm -f $MARKER $SRVR_BL.old |
|
|
179 if test -n "$MIRROR"; then |
|
|
180 cp $SRVR_BL $MIRROR |
|
|
181 fi |
|
|
182 else |
|
|
183 # complain only when the list is more than a day old, and then only |
|
|
184 # once per day |
|
|
185 OLD_MARKER="`find $MARKER -follow -mtime -1 2>/dev/null`" |
|
|
186 if test -z "$OLD_MARKER"; then |
|
|
187 echo "Unable to fetch blacklist; see $FLOG" > $MARKER |
|
|
188 date >>$MARKER |
|
|
189 fi |
|
|
190 # continue so that we include new versions of local blacklists |
|
|
191 mv $SRVR_BL.old $SRVR_BL |
|
|
192 fi |
|
|
193 |
|
|
194 # add the local lists last so that they can override |
|
|
195 rm -f $NEW_BL |
|
|
196 cat $SRVR_BL >$NEW_BL |
|
|
197 for NM in $OTHER_BLS; do |
|
|
198 if test -s "$NM"; then |
|
|
199 echo >>$NEW_BL |
|
|
200 echo >>$NEW_BL |
|
|
201 echo >>$NEW_BL |
|
|
202 echo "# local file $NM" >>$NEW_BL |
|
|
203 cat $NM >>$NEW_BL |
|
|
204 fi |
|
|
205 done |
|
|
206 if test -z "$FORCE" && cmp $NEW_BL ../blacklist 1>/dev/null 2>&1; then : |
|
|
207 else |
|
|
208 # copy it in case the target is a symbolic link |
|
|
209 cp $NEW_BL $TGT_BL |
|
|
210 fi |
|
|
211 rm $NEW_BL |
