Mercurial > notdcc

comparison misc/fetchblack.in @ 0:c7f6b056b673

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
First import of vendor version
author Peter Gervai <grin@grin.hu>
date Tue, 10 Mar 2009 13:49:58 +0100
parents
children
comparison
equal deleted inserted replaced
-1:000000000000 0:c7f6b056b673
1 #! /bin/sh
2
3 # fetch a blacklist dcc.dcc-servers.net DCC servers
4
5 # This script should be run about twice an hour at an arbitrary time
6 # instead of 0,30 to minimize pile-ups on the FTP server.
7
8 # -h homedir set DCC home directory
9
10 # -o blklist file of local blacklist entries
11
12 # -p pgm fetch, wget, curl, or ftp
13
14 # -s srvr-host host name of source of the common blacklist
15
16 # Rhyolite Software DCC 1.3.103-1.50 $Revision$
17 # @configure_input@
18
19
20 # other, probably local blacklist files
21 OTHER_BLS=
22
23 exec </dev/null
24
25 HTTP_REFERER=DCC-1.3.103-script; export HTTP_REFERER
26
27 DCC_HOMEDIR=@prefix@
28 SRVR_BL=dcc-servers-blacklist
29 SRVRS=
30 PGM=@FETCH_CMD@
31 FORCE=
32 MIRROR=
33
34 USAGE="`basename $0`: [-xf] [-h homedir] [-o blklist] [-p pgm] [-s srvr-host]"
35 while getopts "xfh:o:p:s:m:" c; do
36 case $c in
37 x) set -x;;
38 f) FORCE=yes;;
39 h) DCC_HOMEDIR="$OPTARG";;
40 o) OTHER_BLS="$OTHER_BLS $OPTARG";;
41 p) PGM=$OPTARG;;
42 s) SRVRS="$SRVRS $OPTARG";;
43 m) MIRROR=$OPTARG;;
44 *) echo "$USAGE" 1>&2; exit 1;;
45 esac
46 done
47 shift `expr $OPTIND - 1 || true`
48 if test "$#" -ne 0; then
49 echo "$USAGE" 1>&2; exit 1
50 fi
51 if test -z "$SRVRS"; then
52 SRVRS="http://www.dcc-servers.net/dcc http://www.rhyolite.com/dcc ftp://ftp.dcc-servers.net ftp://ftp.rhyolite.com"
53 fi
54 URLS=
55 for SRVR in $SRVRS; do
56 if expr "$SRVR" : '.[hft]*tp://' >/dev/null; then
57 if expr "$SRVR" : ".*/$SRVR_BL"'$' >/dev/null; then
58 URLS="$URLS $SRVR"
59 else
60 URLS="$URLS $SRVR/$SRVR_BL"
61 fi
62 else
63 URLS="$URLS ftp://$SRVR/$SRVR_BL"
64 fi
65 done
66
67 # dccd expects this target
68 TGT_BL=$DCC_HOMEDIR/blacklist
69 NEW_BL=new-blacklist
70 MARKER=fetch-failed
71 BDIR=$DCC_HOMEDIR/$SRVR_BL
72 FLOG=$BDIR/fetch-log
73
74 if test ! -d $BDIR; then
75 mkdir $BDIR
76 fi
77 cd $BDIR
78
79 # use fetch, wget, curl, or ftp that understands URLs
80 if test ! -s $SRVR_BL; then
81 echo "# initial place keeper" >$SRVR_BL
82 fi
83 mv $SRVR_BL $SRVR_BL.old
84 rm -f $FLOG
85 BASE_PGM=`basename "$PGM"`
86 if test "$BASE_PGM" = wget; then
87 BASE_PGM=
88 for URL in $URLS; do
89 echo "$URL:" >>$FLOG
90 # Do not use --mirror because -r results in a 0 exit status
91 # even on failure.
92 # Do not use --no-remove-listing, -nr, or --dont-remove-listing
93 # because none of them are supported by all versions of wget.
94 # At least some versions of wget exit with 0 after having done
95 # nothing but emitting a usage message.
96 if $PGM -nd --retr-symlinks -N --no-host-directories \
97 --passive-ftp @FETCH_WGET_OPTS@ $URL >>$FLOG 2>&1; then
98 if test -s $SRVR_BL; then
99 if test -z "`sed -n -e 2q \
100 -e 's/.*DOCTYPE.*/HTML/p' \
101 -e 's/<HEAD>/HTML/p' -e 's/<head>/HTML/p' \
102 $SRVR_BL`"; then
103 break;
104 fi
105 # do not leave a broken file
106 rm $SRVR_BL
107 fi
108 fi
109 echo >>$FLOG
110 done
111 fi
112
113 if test "$BASE_PGM" = fetch; then
114 BASE_PGM=
115 for URL in $URLS; do
116 echo "$URL:" >>$FLOG
117 $PGM -p -m $URL >>$FLOG 2>&1
118 if test -s $SRVR_BL; then
119 break;
120 fi
121 echo >>$FLOG
122 done
123 fi
124
125 if test "$BASE_PGM" = curl; then
126 BASE_PGM=
127 for URL in $URLS; do
128 echo "$URL:" >>$FLOG
129 $PGM @FETCH_CURL_OPTS@ --connect-timeout 30 --max-time 600\
130 $URL -o $SRVR_BL >>$FLOG 2>&1
131 # --fail does not work on at least some versions of curl
132 if test -s $SRVR_BL; then
133 if test -z "`sed -n -e 2q \
134 -e 's/.*DOCTYPE.*/HTML/p' \
135 -e 's/<HEAD>/HTML/p' -e 's/<head>/HTML/p' \
136 $SRVR_BL`"; then
137 break;
138 fi
139 # do not leave a broken file
140 rm $SRVR_BL
141 fi
142 echo >>$FLOG
143 done
144 fi
145
146 if test "$BASE_PGM" = ftp; then
147 BASE_PGM=
148 for URL in $URLS; do
149 echo "$URL:" >>$FLOG
150 $PGM -p $URL >>$FLOG 2>&1
151 if test -s $SRVR_BL; then
152 break;
153 fi
154 echo >>$FLOG
155 done
156 # if that did not work, try ancient FTP
157 if test ! -s $SRVR_BL; then
158 for URL in $URLS; do
159 HOST=`expr "$URL" : "ftp://\([^/]*\)/"`
160 RFILE=`expr "$URL" : "ftp://[^/]*/\(.*\)"`
161 if test -z "$RFILE" -o -z "$HOST"; then
162 continue
163 fi
164 echo "$URL:" >>$FLOG
165 (echo "user anonymous `hostname`"; echo "get $RFILE $SRVR_BL") \
166 | ftp -n $HOST >>$FLOG 2>&1
167 if test -s $SRVR_BL; then
168 break;
169 fi
170 # some versions of ftp like to leave empty files
171 rm -f $SRVR_BL
172 echo >>$FLOG
173 done
174 fi
175 fi
176
177 if test -s $SRVR_BL; then
178 rm -f $MARKER $SRVR_BL.old
179 if test -n "$MIRROR"; then
180 cp $SRVR_BL $MIRROR
181 fi
182 else
183 # complain only when the list is more than a day old, and then only
184 # once per day
185 OLD_MARKER="`find $MARKER -follow -mtime -1 2>/dev/null`"
186 if test -z "$OLD_MARKER"; then
187 echo "Unable to fetch blacklist; see $FLOG" > $MARKER
188 date >>$MARKER
189 fi
190 # continue so that we include new versions of local blacklists
191 mv $SRVR_BL.old $SRVR_BL
192 fi
193
194 # add the local lists last so that they can override
195 rm -f $NEW_BL
196 cat $SRVR_BL >$NEW_BL
197 for NM in $OTHER_BLS; do
198 if test -s "$NM"; then
199 echo >>$NEW_BL
200 echo >>$NEW_BL
201 echo >>$NEW_BL
202 echo "# local file $NM" >>$NEW_BL
203 cat $NM >>$NEW_BL
204 fi
205 done
206 if test -z "$FORCE" && cmp $NEW_BL ../blacklist 1>/dev/null 2>&1; then :
207 else
208 # copy it in case the target is a symbolic link
209 cp $NEW_BL $TGT_BL
210 fi
211 rm $NEW_BL