|
0
|
1 /* Distributed Clearinghouse Checksum database cleaner |
|
|
2 * |
|
|
3 * Copyright (c) 2008 by Rhyolite Software, LLC |
|
|
4 * |
|
|
5 * This agreement is not applicable to any entity which sells anti-spam |
|
|
6 * solutions to others or provides an anti-spam solution as part of a |
|
|
7 * security solution sold to other entities, or to a private network |
|
|
8 * which employs the DCC or uses data provided by operation of the DCC |
|
|
9 * but does not provide corresponding data to other users. |
|
|
10 * |
|
|
11 * Permission to use, copy, modify, and distribute this software without |
|
|
12 * changes for any purpose with or without fee is hereby granted, provided |
|
|
13 * that the above copyright notice and this permission notice appear in all |
|
|
14 * copies and any distributed versions or copies are either unchanged |
|
|
15 * or not called anything similar to "DCC" or "Distributed Checksum |
|
|
16 * Clearinghouse". |
|
|
17 * |
|
|
18 * Parties not eligible to receive a license under this agreement can |
|
|
19 * obtain a commercial license to use DCC by contacting Rhyolite Software |
|
|
20 * at sales@rhyolite.com. |
|
|
21 * |
|
|
22 * A commercial license would be for Distributed Checksum and Reputation |
|
|
23 * Clearinghouse software. That software includes additional features. This |
|
|
24 * free license for Distributed ChecksumClearinghouse Software does not in any |
|
|
25 * way grant permision to use Distributed Checksum and Reputation Clearinghouse |
|
|
26 * software |
|
|
27 * |
|
|
28 * THE SOFTWARE IS PROVIDED "AS IS" AND RHYOLITE SOFTWARE, LLC DISCLAIMS ALL |
|
|
29 * WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES |
|
|
30 * OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL RHYOLITE SOFTWARE, LLC |
|
|
31 * BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES |
|
|
32 * OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, |
|
|
33 * WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, |
|
|
34 * ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS |
|
|
35 * SOFTWARE. |
|
|
36 * |
|
|
37 * Rhyolite Software DCC 1.3.103-1.254 $Revision$ |
|
|
38 */ |
|
|
39 |
|
|
40 #include "srvr_defs.h" |
|
|
41 #include "dcc_ck.h" |
|
|
42 #include <signal.h> |
|
|
43 |
|
|
44 static DCC_EMSG dcc_emsg; |
|
|
45 |
|
|
46 static DCC_WF dbclean_wf; |
|
|
47 static DCC_WHITE_TBL dbclean_white_tbl; |
|
|
48 static DCC_CLNT_CTXT *ctxt; |
|
|
49 static DCC_OP_RESP aop_resp; |
|
|
50 static int flods_off; |
|
|
51 static int dccd_unlocked; /* dccd has been told to unlock */ |
|
|
52 |
|
|
53 static DCC_SRVR_NM srvr = DCC_SRVR_NM_DEF; |
|
|
54 static DCC_CLNT_ID srvr_clnt_id = DCC_ID_INVALID; |
|
|
55 static const ID_TBL *srvr_clnt_tbl; |
|
|
56 static u_char info_flags = 0; |
|
|
57 #ifdef USE_DBCLEAN_F |
|
|
58 static u_char db_mode = DB_OPEN_MMAP_WRITE; |
|
|
59 #else |
|
|
60 static u_char db_mode = DB_OPEN_MMAP_WRITE_NOSYNC; |
|
|
61 #endif |
|
|
62 |
|
|
63 static u_char cleardb; /* 1=clear the database */ |
|
|
64 static enum { |
|
|
65 NORMAL_MODE, |
|
|
66 REPAIR_MODE, /* database broken */ |
|
|
67 QUICK_MODE, /* too big for window */ |
|
|
68 HASH_MODE, /* hash table full */ |
|
|
69 NO_CRON_MODE, /* work around missing cron job */ |
|
|
70 DEL_MODE /* after deletion */ |
|
|
71 } clean_mode = NORMAL_MODE; |
|
|
72 static u_char standalone; /* 1=don't talk to dccd */ |
|
|
73 static u_char keep_white; /* 1=do not rebuild whitelist */ |
|
|
74 |
|
|
75 static int exit_value = -1; |
|
|
76 |
|
|
77 static const char *homedir; |
|
|
78 static u_char cur_db_created; |
|
|
79 static const char *cur_db_nm_str = DB_DCC_NAME; |
|
|
80 static DCC_PATH cur_db_nm; |
|
|
81 static DCC_PATH cur_hash_nm; |
|
|
82 static int old_db_fd = -1; |
|
|
83 static DB_HADDR old_db_hash_used; |
|
|
84 static DB_PARMS old_db_parms; |
|
|
85 static DB_PARMS new_db_parms; |
|
|
86 static DB_PTR old_db_pos, new_db_csize; |
|
|
87 static off_t new_db_fsize; |
|
|
88 static u_int new_db_pagesize; |
|
|
89 static FLOD_MMAPS new_flod_mmaps; |
|
|
90 static u_char adj_delay_pos; |
|
|
91 static u_char new_db_created; |
|
|
92 static DCC_PATH new_db_nm; |
|
|
93 static int new_db_fd = -1; |
|
|
94 static u_char new_hash_created; |
|
|
95 static DCC_PATH new_hash_nm; |
|
|
96 static DCC_PATH old_db_nm; |
|
|
97 |
|
|
98 static int expire_secs = -1; |
|
|
99 static int def_expire_secs = DB_EXPIRE_SECS_DEF; |
|
|
100 static int expire_spamsecs = -1; |
|
|
101 static int def_expire_spamsecs = DB_EXPIRE_SPAMSECS_DEF; |
|
|
102 static int have_expire_parms = 0; |
|
|
103 static double def_exp_ratio = 0.0; |
|
|
104 static DB_EX_SECS new_ex_secs; |
|
|
105 static DB_EX_TS new_ex_ts; |
|
|
106 |
|
|
107 static DB_HADDR new_hash_len; |
|
|
108 |
|
|
109 static int expired_rcds, comp_rcds, obs_rcds, expired_cks; |
|
|
110 static int white_cks, kept_cks; |
|
|
111 |
|
|
112 static DCC_TS future_ts; |
|
|
113 |
|
|
114 #define RESTART_DELAY (60*5) |
|
|
115 #define SHORT_DELAY 30 |
|
|
116 |
|
|
117 static struct timeval clean_start; |
|
|
118 |
|
|
119 static struct timeval progress_rpt_last; /* when previous progress report */ |
|
|
120 static struct timeval progress_rpt_checked; /* when last checked */ |
|
|
121 static struct timeval progress_rpt_start; /* start of progress reporting */ |
|
|
122 #define REPORT_INTERVAL_SECS (5*60) |
|
|
123 #define REPORT_INTERVAL_FAST_SECS 10 |
|
|
124 #define UNLOCK_INTERVAL_USECS (DCC_US/2) |
|
|
125 static int progress_rpt_cnt; /* operations until next check */ |
|
|
126 static int progress_rpt_base; |
|
|
127 static u_char progress_rpt_started; /* 1=have started reporting progress */ |
|
|
128 static int progress_rpt_percent; |
|
|
129 |
|
|
130 static u_char write_new_flush(u_char); |
|
|
131 static u_char write_new_rcd(const void *, int); |
|
|
132 static void write_new_hdr(u_char); |
|
|
133 static void unlink_whine(const char *, u_char); |
|
|
134 static void rename_bail(const char *, const char *); |
|
|
135 static u_char expire(DB_PTR); |
|
|
136 static u_char copy_db(void); |
|
|
137 static u_char catchup(DCC_EMSG); |
|
|
138 static void parse_white(void); |
|
|
139 static void build_hash(void); |
|
|
140 static u_char persist_aop(DCC_AOPS, u_int32_t, int); |
|
|
141 static void dccd_new_db(const char *); |
|
|
142 static void finish(void); |
|
|
143 static void exit_dbclean(int) NRATTRIB; |
|
|
144 static void sigterm(int); |
|
|
145 |
|
|
146 |
|
|
147 static void |
|
|
148 usage(u_char die) |
|
|
149 { |
|
|
150 const char str[] = { |
|
|
151 "usage: [-64dfFNPSVq] [-i id]" |
|
|
152 " [-a [server-addr][,server-port]] [-h homedir]\n" |
|
|
153 " [-G on] [-R mode] [-s hash-size] [-e seconds]" |
|
|
154 " [-E spamsecs]\n" |
|
|
155 " [-L ltype,facility.level]"}; |
|
|
156 static u_char complained; |
|
|
157 |
|
|
158 /* its important to try to run, so don't give up unless necessary */ |
|
|
159 if (die) { |
|
|
160 dcc_logbad(EX_USAGE, complained ? "giving up" : str); |
|
|
161 } else if (!complained) { |
|
|
162 dcc_error_msg("%s\ncontinuing", str); |
|
|
163 complained = 1; |
|
|
164 } |
|
|
165 } |
|
|
166 |
|
|
167 |
|
|
168 int NRATTRIB |
|
|
169 main(int argc, char **argv) |
|
|
170 { |
|
|
171 char hostname[DCC_MAXDOMAINLEN]; |
|
|
172 u_char print_version = 0; |
|
|
173 struct stat cur_db_sb; |
|
|
174 u_int tgt_db_pagesize; |
|
|
175 const char *cp; |
|
|
176 char *p; |
|
|
177 u_long l; |
|
|
178 int i; |
|
|
179 |
|
|
180 gettimeofday(&db_time, 0); |
|
|
181 clean_start = db_time; |
|
|
182 |
|
|
183 dcc_timeval2ts(&future_ts, &clean_start, 24*60*60); |
|
|
184 |
|
|
185 dcc_syslog_init(1, argv[0], 0); |
|
|
186 |
|
|
187 /* this must match DBCLEAN_GETOPTS in cron-dccd.in */ |
|
|
188 while ((i = getopt(argc, argv, "64dfFNPSVqi:a:h:G:R:s:e:E:L:")) != -1) { |
|
|
189 switch (i) { |
|
|
190 case '6': |
|
|
191 #ifndef NO_IPV6 |
|
|
192 info_flags = DCC_INFO_FG_IPV6; |
|
|
193 #endif |
|
|
194 break; |
|
|
195 case '4': |
|
|
196 info_flags = 0; |
|
|
197 break; |
|
|
198 |
|
|
199 case 'd': |
|
|
200 if (db_debug++) |
|
|
201 ++dcc_clnt_debug; |
|
|
202 break; |
|
|
203 |
|
|
204 case 'f': |
|
|
205 db_mode &= ~DB_OPEN_MMAP_WRITE; |
|
|
206 break; |
|
|
207 |
|
|
208 |
|
|
209 case 'F': |
|
|
210 db_mode |= DB_OPEN_MMAP_WRITE; |
|
|
211 break; |
|
|
212 |
|
|
213 case 'N': /* make a new, clear database */ |
|
|
214 cleardb = 1; |
|
|
215 standalone = 1; |
|
|
216 break; |
|
|
217 |
|
|
218 case 'P': |
|
|
219 if (have_expire_parms > 0) |
|
|
220 dcc_logbad(EX_USAGE, |
|
|
221 "do not use -P with -e or -E"); |
|
|
222 have_expire_parms = -1; |
|
|
223 break; |
|
|
224 |
|
|
225 case 'S': |
|
|
226 standalone = 1; |
|
|
227 break; |
|
|
228 |
|
|
229 case 'V': |
|
|
230 fprintf(stderr, DCC_VERSION"\n"); |
|
|
231 print_version = 1; |
|
|
232 break; |
|
|
233 |
|
|
234 case 'q': |
|
|
235 trace_quiet = 1; |
|
|
236 break; |
|
|
237 |
|
|
238 case 'i': |
|
|
239 l = strtoul(optarg, &p, 10); |
|
|
240 if (*p != '\0' |
|
|
241 || l < DCC_SRVR_ID_MIN |
|
|
242 || l > DCC_SRVR_ID_MAX) |
|
|
243 dcc_logbad(EX_USAGE, "invalid DCC ID \"-i %s\"", |
|
|
244 optarg); |
|
|
245 srvr_clnt_id = l; |
|
|
246 break; |
|
|
247 |
|
|
248 case 'a': |
|
|
249 cp = dcc_parse_nm_port(dcc_emsg, optarg, srvr.port, |
|
|
250 hostname, sizeof(hostname), |
|
|
251 &srvr.port, 0, 0, 0, 0); |
|
|
252 if (!cp) { |
|
|
253 dcc_error_msg("%s", dcc_emsg); |
|
|
254 break; |
|
|
255 } |
|
|
256 cp += strspn(cp, DCC_WHITESPACE); |
|
|
257 if (*cp != '\0') { |
|
|
258 dcc_error_msg("unrecognized port number in" |
|
|
259 "\"-a %s\"", optarg); |
|
|
260 break; |
|
|
261 } |
|
|
262 if (hostname[0] == '\0') |
|
|
263 strcpy(srvr.hostname, DCC_SRVR_NM_DEF_HOST); |
|
|
264 else |
|
|
265 BUFCPY(srvr.hostname, hostname); |
|
|
266 break; |
|
|
267 |
|
|
268 case 'h': |
|
|
269 homedir = optarg; |
|
|
270 break; |
|
|
271 |
|
|
272 case 'G': |
|
|
273 dcc_syslog_init(1, argv[0], " grey"); |
|
|
274 if (have_expire_parms > 0) |
|
|
275 dcc_logbad(EX_USAGE, |
|
|
276 "do not use -G with -e or -E"); |
|
|
277 if (strcasecmp(optarg, "on")) |
|
|
278 usage(0); /* be generous and allow -Gasdf */ |
|
|
279 grey_on = 1; |
|
|
280 have_expire_parms = -1; |
|
|
281 cur_db_nm_str = DB_GREY_NAME; |
|
|
282 break; |
|
|
283 |
|
|
284 case 'R': |
|
|
285 if (!strcasecmp(optarg, "bad")) |
|
|
286 clean_mode = REPAIR_MODE; |
|
|
287 else if (!strcasecmp(optarg, "quick")) |
|
|
288 clean_mode = QUICK_MODE; |
|
|
289 else if (!strcasecmp(optarg, "hash")) |
|
|
290 clean_mode = HASH_MODE; |
|
|
291 else if (!strcasecmp(optarg, "cron")) |
|
|
292 clean_mode = NO_CRON_MODE; |
|
|
293 else if (!strcasecmp(optarg, "del")) |
|
|
294 clean_mode = DEL_MODE; |
|
|
295 else |
|
|
296 dcc_logbad(EX_USAGE, |
|
|
297 "unrecognized repair mode -R %s", |
|
|
298 optarg); |
|
|
299 break; |
|
|
300 |
|
|
301 case 's': /* hash table size in entries */ |
|
|
302 new_hash_len = strtoul(optarg, &p, 0); |
|
|
303 if (*p != '\0' |
|
|
304 || new_hash_len < MIN_HASH_ENTRIES |
|
|
305 || new_hash_len > MAX_HASH_ENTRIES) |
|
|
306 dcc_logbad(EX_USAGE, |
|
|
307 "invalid database size \"%s\"", |
|
|
308 optarg); |
|
|
309 break; |
|
|
310 |
|
|
311 case 'e': /* expiration for non-bulk checksums */ |
|
|
312 if (grey_on) |
|
|
313 dcc_logbad(EX_USAGE, |
|
|
314 "do not use -e with -G"); |
|
|
315 if (have_expire_parms < 0) |
|
|
316 dcc_logbad(EX_USAGE, |
|
|
317 "-e cannot be used with -P"); |
|
|
318 have_expire_parms = 1; |
|
|
319 expire_secs = dcc_get_secs(optarg, 0, |
|
|
320 DB_EXPIRE_SECS_MIN, |
|
|
321 DB_EXPIRE_SECS_MAX, -1); |
|
|
322 if (expire_secs < 0) |
|
|
323 dcc_logbad(EX_USAGE, |
|
|
324 "invalid expiration seconds" |
|
|
325 " \"-e %s\"", |
|
|
326 optarg); |
|
|
327 break; |
|
|
328 |
|
|
329 case 'E': /* expiration for bulk checksums */ |
|
|
330 if (grey_on) |
|
|
331 dcc_logbad(EX_USAGE, |
|
|
332 "do not use -E with -G"); |
|
|
333 if (have_expire_parms < 0) |
|
|
334 dcc_logbad(EX_USAGE, |
|
|
335 "do not use -E with -P"); |
|
|
336 have_expire_parms = 1; |
|
|
337 expire_spamsecs = dcc_get_secs(optarg, 0, |
|
|
338 DB_EXPIRE_SECS_MIN, |
|
|
339 DB_EXPIRE_SECS_MAX, -1); |
|
|
340 if (expire_spamsecs < 0) |
|
|
341 dcc_logbad(EX_USAGE, |
|
|
342 "invalid spam expiration seconds" |
|
|
343 " \"-E %s\"", |
|
|
344 optarg); |
|
|
345 break; |
|
|
346 |
|
|
347 case 'L': |
|
|
348 dcc_parse_log_opt(optarg); |
|
|
349 break; |
|
|
350 |
|
|
351 default: |
|
|
352 usage(0); |
|
|
353 } |
|
|
354 } |
|
|
355 argc -= optind; |
|
|
356 argv += optind; |
|
|
357 if (argc != 0) |
|
|
358 usage(1); |
|
|
359 |
|
|
360 if (srvr_clnt_id == DCC_ID_INVALID && !standalone) { |
|
|
361 if (print_version) |
|
|
362 exit(EX_OK); |
|
|
363 usage(1); |
|
|
364 } |
|
|
365 srvr.clnt_id = srvr_clnt_id; |
|
|
366 |
|
|
367 if (srvr.port == 0) |
|
|
368 srvr.port = DCC_GREY2PORT(grey_on); |
|
|
369 |
|
|
370 dcc_clnt_unthread_init(); |
|
|
371 /* move to the target directory |
|
|
372 * and set homedir for fnm2rel_good() */ |
|
|
373 if (!dcc_cdhome(dcc_emsg, homedir, 0)) |
|
|
374 dcc_logbad(dcc_ex_code, "%s", dcc_emsg); |
|
|
375 fnm2rel_good(cur_db_nm, cur_db_nm_str, 0); |
|
|
376 cp = ""; |
|
|
377 switch (clean_mode) { |
|
|
378 case NORMAL_MODE: cp = "cleaning"; break; |
|
|
379 case REPAIR_MODE: cp = "repairing"; break; |
|
|
380 case QUICK_MODE: cp = "quick cleaning"; break; |
|
|
381 case HASH_MODE: cp = "expanding hash table in"; break; |
|
|
382 case NO_CRON_MODE: cp = "work around missing cron job for"; break; |
|
|
383 case DEL_MODE: cp = "clean up deletion in"; break; |
|
|
384 } |
|
|
385 quiet_trace_msg(DCC_VERSION" %s %s", cp, fnm2abs_err(0, cur_db_nm)); |
|
|
386 |
|
|
387 atexit(finish); |
|
|
388 signal(SIGHUP, sigterm); |
|
|
389 signal(SIGTERM, sigterm); |
|
|
390 signal(SIGINT, sigterm); |
|
|
391 #ifdef SIGXFSZ |
|
|
392 signal(SIGXFSZ, SIG_IGN); |
|
|
393 #endif |
|
|
394 |
|
|
395 if (!standalone) { |
|
|
396 i = load_ids(dcc_emsg, srvr_clnt_id, &srvr_clnt_tbl, 1); |
|
|
397 if (i <= 0) |
|
|
398 dcc_logbad(dcc_ex_code, "%s", dcc_emsg); |
|
|
399 memcpy(srvr.passwd, srvr_clnt_tbl->cur_passwd, |
|
|
400 sizeof(srvr.passwd)); |
|
|
401 } |
|
|
402 |
|
|
403 fnm2rel_good(cur_hash_nm, cur_db_nm, DB_HASH_SUFFIX); |
|
|
404 fnm2rel_good(old_db_nm, cur_db_nm, "-old"); |
|
|
405 fnm2rel_good(new_db_nm, cur_db_nm, "-new"); |
|
|
406 fnm2rel_good(new_hash_nm, new_db_nm, DB_HASH_SUFFIX); |
|
|
407 |
|
|
408 /* exclude other instances of this program */ |
|
|
409 if (!lock_dbclean(dcc_emsg, cur_db_nm)) |
|
|
410 dcc_logbad(dcc_ex_code, "%s: dbclean already running?", |
|
|
411 dcc_emsg); |
|
|
412 |
|
|
413 /* create & the lock new database file */ |
|
|
414 new_db_fd = dcc_lock_open(dcc_emsg, new_db_nm, O_RDWR|O_CREAT, |
|
|
415 DCC_LOCK_OPEN_NOWAIT, DCC_LOCK_ALL_FILE, 0); |
|
|
416 if (new_db_fd == -1) |
|
|
417 dcc_logbad(dcc_ex_code, "%s", dcc_emsg); |
|
|
418 if (0 > ftruncate(new_db_fd, 0)) |
|
|
419 dcc_logbad(EX_IOERR, "truncate(%s,0): %s", |
|
|
420 new_db_nm, ERROR_STR()); |
|
|
421 new_db_fsize = 0; |
|
|
422 new_db_created = 1; |
|
|
423 new_db_csize = DB_PTR_BASE; |
|
|
424 |
|
|
425 tgt_db_pagesize = 0; |
|
|
426 if (0 > stat(cur_db_nm, &cur_db_sb)) { |
|
|
427 if (errno != ENOENT) |
|
|
428 dcc_logbad(EX_IOERR, "stat(%s): %s", |
|
|
429 cur_db_nm, ERROR_STR()); |
|
|
430 /* empty a missing database */ |
|
|
431 cleardb = 1; |
|
|
432 } else if (cur_db_sb.st_size == 0) { |
|
|
433 /* empty an empty database */ |
|
|
434 cleardb = 1; |
|
|
435 } else if (grey_on && cur_db_sb.st_size < DB_MIN_MIN_MBYTE*1024*1024) { |
|
|
436 /* Force a relatively large page size for typical tiny |
|
|
437 * greylist databases. Try to use few mmap() pages */ |
|
|
438 tgt_db_pagesize = cur_db_sb.st_size/4; |
|
|
439 if (tgt_db_pagesize < MIN_HASH_ENTRIES*sizeof(HASH_ENTRY)) |
|
|
440 tgt_db_pagesize = MIN_HASH_ENTRIES*sizeof(HASH_ENTRY); |
|
|
441 } |
|
|
442 new_db_pagesize = db_get_pagesize(0, tgt_db_pagesize); |
|
|
443 write_new_hdr(1); |
|
|
444 |
|
|
445 |
|
|
446 if (standalone) { |
|
|
447 u_char busy; |
|
|
448 |
|
|
449 /* open and lock the current database to ensure |
|
|
450 * the daemon is not running */ |
|
|
451 old_db_fd = dcc_lock_open(dcc_emsg, cur_db_nm, O_RDWR, |
|
|
452 DCC_LOCK_OPEN_NOWAIT, |
|
|
453 DCC_LOCK_ALL_FILE, &busy); |
|
|
454 if (busy) |
|
|
455 dcc_logbad(EX_USAGE, "database %s in use: %s", |
|
|
456 cur_db_nm, dcc_emsg); |
|
|
457 if (cleardb |
|
|
458 && stat(cur_db_nm, &cur_db_sb) >= 0) { |
|
|
459 if (cur_db_sb.st_size != 0) |
|
|
460 dcc_logbad(EX_USAGE, "%s already exists", |
|
|
461 cur_db_nm); |
|
|
462 cur_db_created = 1; |
|
|
463 } |
|
|
464 |
|
|
465 /* create and lock the current database if it did not exist |
|
|
466 * to ensure that the server daemon is not running */ |
|
|
467 if (old_db_fd < 0) { |
|
|
468 old_db_fd = dcc_lock_open(dcc_emsg, cur_db_nm, |
|
|
469 O_RDWR|O_CREAT, |
|
|
470 DCC_LOCK_OPEN_NOWAIT, |
|
|
471 DCC_LOCK_ALL_FILE, 0); |
|
|
472 if (old_db_fd < 0) |
|
|
473 dcc_logbad(dcc_ex_code, "%s", dcc_emsg); |
|
|
474 cur_db_created = 1; |
|
|
475 } |
|
|
476 |
|
|
477 } else { |
|
|
478 /* Tell the daemon to start turning off the flooding |
|
|
479 * so we can adjust its positions in the flood map file |
|
|
480 * Try very hard to talk to it because releasing the database |
|
|
481 * can cause some UNIX flavors to stall dccd. */ |
|
|
482 DCC_CLNT_FGS clnt_fgs; |
|
|
483 |
|
|
484 clnt_fgs = DCC_CLNT_FG_SLOW; |
|
|
485 if (grey_on) |
|
|
486 clnt_fgs |= DCC_CLNT_FG_GREY; |
|
|
487 ctxt = dcc_tmp_clnt_init(dcc_emsg, 0, &srvr, |
|
|
488 0, clnt_fgs, info_flags); |
|
|
489 /* try very hard to contact dccd */ |
|
|
490 if (!ctxt) |
|
|
491 ctxt = dcc_tmp_clnt_init(dcc_emsg, 0, &srvr, |
|
|
492 0, clnt_fgs, info_flags); |
|
|
493 if (!ctxt) |
|
|
494 dcc_logbad(EX_DCC_RESTART, "initial contact: %s", |
|
|
495 dcc_emsg); |
|
|
496 |
|
|
497 ++flods_off; |
|
|
498 if (!persist_aop(DCC_AOP_FLOD, DCC_AOP_FLOD_SHUTDOWN, |
|
|
499 SHORT_DELAY)) |
|
|
500 dcc_logbad(dcc_ex_code, "%s", dcc_emsg); |
|
|
501 } |
|
|
502 |
|
|
503 /* resolve whitelisted host names before locking the database */ |
|
|
504 parse_white(); |
|
|
505 |
|
|
506 /* Tell the daemon to unlock the database between operations |
|
|
507 * and insist it stop flooding. */ |
|
|
508 if (!standalone) { |
|
|
509 /* give the daemon a chance to stop pumping the floods */ |
|
|
510 for (;;) { |
|
|
511 if (!persist_aop(DCC_AOP_FLOD, DCC_AOP_FLOD_CHECK, |
|
|
512 SHORT_DELAY)) |
|
|
513 dcc_logbad(EX_UNAVAILABLE, "%s", dcc_emsg); |
|
|
514 |
|
|
515 i = flod_running(aop_resp.resp.val.string); |
|
|
516 if (i < 0) |
|
|
517 dcc_logbad(EX_PROTOCOL, |
|
|
518 "%s: unrecognized \"%s\"", |
|
|
519 dcc_aop2str(0, 0, |
|
|
520 DCC_AOP_FLOD, |
|
|
521 DCC_AOP_FLOD_CHECK), |
|
|
522 aop_resp.resp.val.string); |
|
|
523 if (i == 0) |
|
|
524 break; |
|
|
525 if (time(0) > clean_start.tv_sec+45) { |
|
|
526 if (flods_off < 2) { |
|
|
527 ++flods_off; |
|
|
528 if (!persist_aop(DCC_AOP_FLOD, |
|
|
529 DCC_AOP_FLOD_HALT, |
|
|
530 SHORT_DELAY)) |
|
|
531 dcc_logbad(dcc_ex_code, "%s", |
|
|
532 dcc_emsg); |
|
|
533 continue; |
|
|
534 } |
|
|
535 if (time(0) > clean_start.tv_sec+60) |
|
|
536 dcc_logbad(EX_UNAVAILABLE, |
|
|
537 "failed to stop floods: %s", |
|
|
538 aop_resp.resp.val.string); |
|
|
539 } |
|
|
540 usleep(100*1000); |
|
|
541 } |
|
|
542 dccd_unlocked = 1; |
|
|
543 if (!persist_aop(DCC_AOP_DB_CLEAN, 0, SHORT_DELAY)) |
|
|
544 dcc_logbad(dcc_ex_code, "%s", dcc_emsg); |
|
|
545 /* The daemon adds its own and removes our hold on flooding |
|
|
546 * when we tell it to unlock the database after every |
|
|
547 * operation. */ |
|
|
548 --flods_off; |
|
|
549 } |
|
|
550 |
|
|
551 if (cleardb) { |
|
|
552 quiet_trace_msg(DCC_VERSION" %s database %s", |
|
|
553 cur_db_created ? "creating" : "clearing", |
|
|
554 cur_db_nm); |
|
|
555 |
|
|
556 } else if (clean_mode == REPAIR_MODE) { |
|
|
557 dcc_error_msg("explicit repair of %s", cur_db_nm); |
|
|
558 |
|
|
559 } else { |
|
|
560 if (!db_open(0, old_db_fd, cur_db_nm, 0, |
|
|
561 DB_OPEN_RDONLY |
|
|
562 | (standalone |
|
|
563 ? DB_OPEN_LOCK_NOWAIT : DB_OPEN_LOCK_WAIT))) { |
|
|
564 /* If the hash table is sick, check timestamps only |
|
|
565 * as much as no hash table allows. |
|
|
566 * Then rebuild the hash table. */ |
|
|
567 clean_mode = REPAIR_MODE; |
|
|
568 |
|
|
569 } else { |
|
|
570 if (db_debug) { |
|
|
571 quiet_trace_msg("%s %s", |
|
|
572 db_window_size_str, new_db_nm); |
|
|
573 quiet_trace_msg("%d old hash entries total," |
|
|
574 " %d or %d%% used", |
|
|
575 HADDR2LEN(db_hash_len), |
|
|
576 HADDR2LEN(db_hash_used), |
|
|
577 (int)((HADDR2LEN(db_hash_used) |
|
|
578 * 100.0) |
|
|
579 /HADDR2LEN(db_hash_len))); |
|
|
580 } |
|
|
581 old_db_parms = db_parms; |
|
|
582 old_db_hash_used = db_hash_used; |
|
|
583 |
|
|
584 /* save a handle on the old database to get |
|
|
585 * reports that arrive while we expire it */ |
|
|
586 old_db_fd = dup(db_fd); |
|
|
587 if (old_db_fd < 0) |
|
|
588 dcc_logbad(EX_OSERR, "dup(%s): %s", |
|
|
589 cur_db_nm, ERROR_STR()); |
|
|
590 |
|
|
591 /* read old and create new database file */ |
|
|
592 if (!expire(db_csize)) { |
|
|
593 old_db_hash_used = 0; |
|
|
594 clean_mode = REPAIR_MODE; |
|
|
595 } |
|
|
596 } |
|
|
597 |
|
|
598 if (clean_mode == REPAIR_MODE) |
|
|
599 dcc_error_msg("repairing %s", cur_db_nm); |
|
|
600 } |
|
|
601 |
|
|
602 /* if we are repairing the hash table (including now repairing |
|
|
603 * after encountering problems while expiring), |
|
|
604 * copy the current file with minimal expiring */ |
|
|
605 if (clean_mode == REPAIR_MODE |
|
|
606 && !cleardb |
|
|
607 && !copy_db()) |
|
|
608 exit_dbclean(EX_UNAVAILABLE); |
|
|
609 build_hash(); |
|
|
610 |
|
|
611 /* Copy any records from the old file to the new file that were |
|
|
612 * added to the old file while we were creating the new file. */ |
|
|
613 if (!cleardb |
|
|
614 && !catchup(dcc_emsg)) |
|
|
615 dcc_logbad(dcc_ex_code, "%s", dcc_emsg); |
|
|
616 |
|
|
617 /* we have the new database locked |
|
|
618 * |
|
|
619 * preserve the current data file as "*-old" */ |
|
|
620 rename_bail(cur_db_nm, old_db_nm); |
|
|
621 |
|
|
622 /* delete the current hash file, and install both new files */ |
|
|
623 rename_bail(new_hash_nm, cur_hash_nm); |
|
|
624 strcpy(new_hash_nm, cur_hash_nm); |
|
|
625 new_hash_created = 0; |
|
|
626 if (db_hash_fd >= 0) |
|
|
627 strcpy(db_hash_nm, cur_hash_nm); |
|
|
628 |
|
|
629 rename_bail(new_db_nm, cur_db_nm); |
|
|
630 strcpy(new_db_nm, cur_db_nm); |
|
|
631 new_db_created = 0; |
|
|
632 if (db_fd > 0) |
|
|
633 strcpy(db_nm, cur_db_nm); |
|
|
634 cur_db_created = 0; |
|
|
635 |
|
|
636 if (cleardb) { |
|
|
637 flod_mmap_path_set(); |
|
|
638 unlink_whine(flod_mmap_path, 1); |
|
|
639 if (!db_close(1)) |
|
|
640 exit_dbclean(EX_UNAVAILABLE); |
|
|
641 exit_dbclean(EX_OK); |
|
|
642 } |
|
|
643 |
|
|
644 /* if the daemon was not running, we're finished */ |
|
|
645 if (standalone) { |
|
|
646 /* install the flood positions if things are ok */ |
|
|
647 if (flod_mmaps) { |
|
|
648 memcpy(flod_mmaps, &new_flod_mmaps, |
|
|
649 sizeof(new_flod_mmaps)); |
|
|
650 flod_unmap(0, 0); |
|
|
651 } |
|
|
652 if (!db_close(1)) |
|
|
653 exit_dbclean(EX_UNAVAILABLE); |
|
|
654 exit_dbclean(EX_OK); |
|
|
655 } |
|
|
656 |
|
|
657 /* tell the daemon to switch to the new database. This will leave |
|
|
658 * the daemon stuck waiting for us to unlock the new database. */ |
|
|
659 dccd_new_db("copy late arrivals"); |
|
|
660 |
|
|
661 /* install the flood positions if things are ok */ |
|
|
662 if (flod_mmaps) { |
|
|
663 memcpy(flod_mmaps, &new_flod_mmaps, |
|
|
664 sizeof(new_flod_mmaps)); |
|
|
665 flod_unmap(0, 0); |
|
|
666 } |
|
|
667 |
|
|
668 /* Copy any records from the old file to the new file in the |
|
|
669 * race to tell the daemon to switch to the new file. |
|
|
670 * The new file is still locked from build_hash(). |
|
|
671 * The daemon should be stuck waiting to open it in the |
|
|
672 * DCC_AOP_DB_NEW request via the preceding dccd_new_db(). |
|
|
673 * |
|
|
674 * Since the daemon has switched and probably cannot go back, |
|
|
675 * ignore any errors */ |
|
|
676 catchup(0); |
|
|
677 if (!db_close(1)) |
|
|
678 exit_dbclean(EX_UNAVAILABLE); |
|
|
679 |
|
|
680 /* finish() will be called via exit() to tell the daemon to resume |
|
|
681 * flooding if necessary. However, in the normal case, we removed |
|
|
682 * all counts against flooding before calling dccd_new_db() */ |
|
|
683 exit_dbclean(EX_OK); |
|
|
684 } |
|
|
685 |
|
|
686 |
|
|
687 |
|
|
688 /* adjust output flood positions */ |
|
|
689 static DB_PTR |
|
|
690 adj_mmap(void) |
|
|
691 { |
|
|
692 FLOD_MMAP *mp; |
|
|
693 DB_PTR delta, new_pos; |
|
|
694 |
|
|
695 delta = new_db_csize - old_db_pos; |
|
|
696 new_pos = 0; |
|
|
697 for (mp = new_flod_mmaps.mmaps; |
|
|
698 mp <= LAST(new_flod_mmaps.mmaps); |
|
|
699 ++mp) { |
|
|
700 /* do nothing to marks we have already adjusted */ |
|
|
701 if (!(mp->flags & FLODMAP_FG_MARK)) |
|
|
702 continue; |
|
|
703 if (mp->confirm_pos > old_db_pos) { |
|
|
704 /* note the next mark that will need adjusting |
|
|
705 * but do not adjust it yet */ |
|
|
706 if (new_pos == 0 |
|
|
707 || new_pos > mp->confirm_pos) |
|
|
708 new_pos = mp->confirm_pos; |
|
|
709 } else { |
|
|
710 /* adjust marks not past the current position */ |
|
|
711 mp->confirm_pos += delta; |
|
|
712 mp->flags &= ~FLODMAP_FG_MARK; |
|
|
713 } |
|
|
714 } |
|
|
715 if (adj_delay_pos) { |
|
|
716 if (new_flod_mmaps.delay_pos > old_db_pos) { |
|
|
717 if (new_pos == 0 |
|
|
718 || new_pos > new_flod_mmaps.delay_pos) |
|
|
719 new_pos = new_flod_mmaps.delay_pos; |
|
|
720 } else { |
|
|
721 new_flod_mmaps.delay_pos += delta; |
|
|
722 adj_delay_pos = 0; |
|
|
723 } |
|
|
724 } |
|
|
725 |
|
|
726 return new_pos; |
|
|
727 } |
|
|
728 |
|
|
729 |
|
|
730 |
|
|
731 /* find a checksum |
|
|
732 * Leave db_sts.rcd2 pointing at the record. */ |
|
|
733 static u_char |
|
|
734 get_ck(DB_RCD_CK **ckp, /* point this to the checksum */ |
|
|
735 DCC_CK_TYPES type, const DCC_SUM sum) |
|
|
736 { |
|
|
737 DB_FOUND db_result; |
|
|
738 |
|
|
739 /* We must lock the file to keep the daemon from changing the |
|
|
740 * internal hash table links. */ |
|
|
741 if (!DB_IS_LOCKED() |
|
|
742 && 0 > db_lock()) |
|
|
743 return 0; |
|
|
744 |
|
|
745 dcc_emsg[0] = '\0'; |
|
|
746 db_result = db_lookup(dcc_emsg, type, sum, 0, MAX_HASH_ENTRIES, |
|
|
747 &db_sts.hash, &db_sts.rcd2, ckp); |
|
|
748 switch (db_result) { |
|
|
749 case DB_FOUND_LATER: |
|
|
750 case DB_FOUND_SYSERR: |
|
|
751 dcc_error_msg("hash lookup for %s from "L_HPAT" = %d: %s", |
|
|
752 DB_TYPE2STR(type), old_db_pos, db_result, |
|
|
753 dcc_emsg); |
|
|
754 break; |
|
|
755 |
|
|
756 case DB_FOUND_IT: |
|
|
757 case DB_FOUND_EMPTY: |
|
|
758 case DB_FOUND_CHAIN: |
|
|
759 case DB_FOUND_INTRUDER: |
|
|
760 return 1; |
|
|
761 } |
|
|
762 |
|
|
763 return 0; |
|
|
764 } |
|
|
765 |
|
|
766 |
|
|
767 |
|
|
768 /* check the leading report for not recent checksum |
|
|
769 * on entry db_sts.rcd points to the record under consideration |
|
|
770 * Leave db_sts.rcd2 pointing at the leading record. */ |
|
|
771 static int /* -1=broken database 0=expire 1=keep */ |
|
|
772 get_lead(DCC_CK_TYPES type, const DB_RCD_CK *rcd_ck) |
|
|
773 { |
|
|
774 DB_RCD_CK *lead_ck; |
|
|
775 DCC_TGTS rcd_tgts, lead_tgts; |
|
|
776 |
|
|
777 if (DCC_CK_IS_REP_CMN(grey_on, type)) { |
|
|
778 /* do not keep reputations on systems without reputation code */ |
|
|
779 return 0; |
|
|
780 } |
|
|
781 |
|
|
782 if (!get_ck(&lead_ck, type, rcd_ck->sum)) |
|
|
783 return -1; |
|
|
784 |
|
|
785 if (!lead_ck) { |
|
|
786 dcc_error_msg("no leader for %s %s at "L_HPAT, |
|
|
787 DB_TYPE2STR(type), |
|
|
788 dcc_ck2str_err(type, rcd_ck->sum, 0), |
|
|
789 old_db_pos); |
|
|
790 return -1; |
|
|
791 } |
|
|
792 |
|
|
793 /* We know the target checksum is not recent. Forget the target if |
|
|
794 * both the target and the leader are ancient. The leader might not be |
|
|
795 * the newest checksum, but it usually is. Note also that the target |
|
|
796 * might be the leader. */ |
|
|
797 if (dcc_ts_older_ts(&db_sts.rcd2.d.r->ts, &new_ex_ts[type].spam) |
|
|
798 && dcc_ts_older_ts(&db_sts.rcd.d.r->ts, &new_ex_ts[type].spam)) |
|
|
799 return 0; |
|
|
800 |
|
|
801 lead_tgts = DB_TGTS_CK(lead_ck); |
|
|
802 rcd_tgts = DB_TGTS_CK(rcd_ck); |
|
|
803 |
|
|
804 /* We know either the leader or the target is not ancient. |
|
|
805 * Keep the target if the leader's total is respectable. |
|
|
806 * We might eventually compress the target. */ |
|
|
807 return (lead_tgts >= db_tholds[type]); |
|
|
808 } |
|
|
809 |
|
|
810 |
|
|
811 |
|
|
812 static void |
|
|
813 report_progress_init(void) |
|
|
814 { |
|
|
815 gettimeofday(&db_time, 0); |
|
|
816 progress_rpt_start.tv_sec = db_time.tv_sec; |
|
|
817 progress_rpt_checked = db_time; |
|
|
818 progress_rpt_last = db_time; |
|
|
819 progress_rpt_base = 100; |
|
|
820 progress_rpt_cnt = progress_rpt_base; |
|
|
821 progress_rpt_started = 0; |
|
|
822 } |
|
|
823 |
|
|
824 |
|
|
825 |
|
|
826 static time_t /* us since last check */ |
|
|
827 report_progress(u_char final, |
|
|
828 const char *s1, const char *s2, |
|
|
829 DB_PTR done, DB_PTR total, DB_PTR scale) |
|
|
830 { |
|
|
831 time_t reported_us, checked_us, secs, interval; |
|
|
832 double percent; |
|
|
833 |
|
|
834 if (!total) |
|
|
835 percent = 100.0; |
|
|
836 else |
|
|
837 percent = (done*100.0)/total; |
|
|
838 |
|
|
839 gettimeofday(&db_time, 0); |
|
|
840 checked_us = tv_diff2us(&db_time, &progress_rpt_checked); |
|
|
841 progress_rpt_checked = db_time; |
|
|
842 |
|
|
843 /* Check frequently enough to report or unlock the database. |
|
|
844 * Adjust the number of operations until the next check |
|
|
845 * based on the time spent on the previous */ |
|
|
846 if (checked_us > 0) |
|
|
847 progress_rpt_base = ((progress_rpt_base * 0.5 * DCC_US |
|
|
848 * min(REPORT_INTERVAL_FAST_SECS*DCC_US, |
|
|
849 UNLOCK_INTERVAL_USECS)) |
|
|
850 / checked_us); |
|
|
851 else |
|
|
852 progress_rpt_base = 100; |
|
|
853 if (progress_rpt_base < 100) |
|
|
854 progress_rpt_base = 100; |
|
|
855 if (progress_rpt_base > 10*1000) |
|
|
856 progress_rpt_base = 10*1000; |
|
|
857 progress_rpt_cnt = progress_rpt_base; |
|
|
858 |
|
|
859 interval = ((db_debug > 1) |
|
|
860 ? REPORT_INTERVAL_FAST_SECS |
|
|
861 : REPORT_INTERVAL_SECS); |
|
|
862 |
|
|
863 /* try not to start reporting progress at the end */ |
|
|
864 if (!progress_rpt_started |
|
|
865 && (total*1.0 - done*1.0) / progress_rpt_base <= interval*1.0) |
|
|
866 return checked_us; |
|
|
867 |
|
|
868 reported_us = tv_diff2us(&db_time, &progress_rpt_last); |
|
|
869 if (reported_us >= interval * DCC_US |
|
|
870 || (final && progress_rpt_percent != 100)) { |
|
|
871 progress_rpt_started = 1; |
|
|
872 progress_rpt_percent = percent; |
|
|
873 secs = db_time.tv_sec - progress_rpt_start.tv_sec; |
|
|
874 secs -= secs % interval; |
|
|
875 progress_rpt_last.tv_sec = progress_rpt_start.tv_sec + secs; |
|
|
876 if (db_debug > 1) |
|
|
877 quiet_trace_msg("%s "L_DPAT" of "L_DPAT" %s or %d%%" |
|
|
878 " db_mmaps=%d hash=%d", |
|
|
879 s1, done/scale, total/scale, |
|
|
880 s2, progress_rpt_percent, |
|
|
881 db_stats.db_mmaps, db_stats.hash_mmaps); |
|
|
882 else |
|
|
883 quiet_trace_msg("%s "L_DPAT" of "L_DPAT" %s or %d%%", |
|
|
884 s1, done/scale, total/scale, |
|
|
885 s2, progress_rpt_percent); |
|
|
886 } |
|
|
887 |
|
|
888 |
|
|
889 if (clean_mode == QUICK_MODE |
|
|
890 && !final) { |
|
|
891 if (db_time.tv_sec > clean_start.tv_sec + 30*60) |
|
|
892 dcc_logbad(EX_UNAVAILABLE, "quick cleaning too slow"); |
|
|
893 } |
|
|
894 |
|
|
895 return checked_us; |
|
|
896 } |
|
|
897 |
|
|
898 |
|
|
899 |
|
|
900 /* delete old, less fuzzy checksums in the new record */ |
|
|
901 static void |
|
|
902 fuzzy_obs(DB_RCD *new, DB_RCD_CK **end_ck) |
|
|
903 { |
|
|
904 DB_RCD_CK *rcd_ck; |
|
|
905 DCC_CK_TYPES type; |
|
|
906 int len; |
|
|
907 |
|
|
908 rcd_ck = new->cks; |
|
|
909 while (rcd_ck < *end_ck) { |
|
|
910 type = DB_CK_TYPE(rcd_ck); |
|
|
911 if (!dcc_ts_older_ts(&new->ts, &new_ex_ts[type].all)) { |
|
|
912 ++rcd_ck; |
|
|
913 continue; |
|
|
914 } |
|
|
915 |
|
|
916 ++obs_rcds; |
|
|
917 new->fgs_num_cks = (((new->fgs_num_cks - 1) |
|
|
918 & ~DB_RCD_FG_DELAY) |
|
|
919 | DB_RCD_FG_TRIM); |
|
|
920 --*end_ck; |
|
|
921 len = (char *)*end_ck - (char *)rcd_ck; |
|
|
922 if (len == 0) |
|
|
923 return; |
|
|
924 memmove(rcd_ck, rcd_ck+1, len); |
|
|
925 } |
|
|
926 } |
|
|
927 |
|
|
928 |
|
|
929 |
|
|
930 static void |
|
|
931 adj_def_expire(void) |
|
|
932 { |
|
|
933 double new_dbsize, new_dbsize1, day_rate, db_ratio; |
|
|
934 int spam_secs, secs; |
|
|
935 struct timeval tv; |
|
|
936 char new_dbsize_buf[20], csize_buf[20], old_csize_buf[20]; |
|
|
937 char day_rate_buf[20]; |
|
|
938 |
|
|
939 /* do this only once */ |
|
|
940 if (def_exp_ratio != 0.0) |
|
|
941 return; |
|
|
942 |
|
|
943 /* Compute the ratio of size of the database 24 hours from now |
|
|
944 * to the size of the window. Assume: |
|
|
945 * - We will receive about the same number of reports in the next |
|
|
946 * 24 hours as the last 24. This is a good assumption for |
|
|
947 * weekdays, but as much as 30% wrong about weekends. |
|
|
948 * - Dbclean will be run once per day at the current time. |
|
|
949 * - The size of the database is a linear function of expiration |
|
|
950 * duration. This is tenuous when the spam expiration duration |
|
|
951 * is less than 1 day. |
|
|
952 * Use the maximum of two guesses for tomorrow's database size. |
|
|
953 * One guess is the current size, base on assuming that |
|
|
954 * we will use roughly the same expiration durations and |
|
|
955 * so the database will grow to about size it now has. |
|
|
956 * The other guess uses the previous database size and the |
|
|
957 * avarage data rate. It compensates for short term changes |
|
|
958 * in the rate and for running dbclean more than once per day. */ |
|
|
959 new_dbsize = db_parms.db_csize; |
|
|
960 size2str(csize_buf, sizeof(csize_buf), new_dbsize, 1); |
|
|
961 new_dbsize1 = db_parms.old_db_csize; |
|
|
962 size2str(old_csize_buf, sizeof(old_csize_buf), new_dbsize1, 1); |
|
|
963 day_rate = db_add_rate(&db_parms, 0); |
|
|
964 if (day_rate >= 0.0) |
|
|
965 day_rate *= (24*60*60); |
|
|
966 size2str(day_rate_buf, sizeof(day_rate_buf), day_rate, 1); |
|
|
967 |
|
|
968 /* without information, be pessimistic and assume 1.4 GByte/day */ |
|
|
969 if (day_rate <= 0.0 && !grey_on) |
|
|
970 day_rate = 1.4*1024.0*1024.0*1024.0; |
|
|
971 if (day_rate > 0.0) { |
|
|
972 new_dbsize1 += day_rate; |
|
|
973 if (new_dbsize < new_dbsize1) |
|
|
974 new_dbsize = new_dbsize1; |
|
|
975 } |
|
|
976 |
|
|
977 size2str(new_dbsize_buf, sizeof(new_dbsize_buf), new_dbsize, 1); |
|
|
978 if (db_debug) |
|
|
979 quiet_trace_msg("predict new_dbsize=%s from db_csize=%s" |
|
|
980 " old_db_csize=%s rate=%s", |
|
|
981 new_dbsize_buf, |
|
|
982 csize_buf, old_csize_buf, day_rate_buf); |
|
|
983 |
|
|
984 /* Assume there will be 20% as many bytes used in the hash table |
|
|
985 * as in the database */ |
|
|
986 new_dbsize *= 1.2; |
|
|
987 |
|
|
988 /* we cannot adjust the defaults |
|
|
989 * - 1st time dbclean run |
|
|
990 * - if the previous run used a larger than default value |
|
|
991 * - there is no need to reduce the default because the predicted |
|
|
992 * maximum size is smaller than the target maximum |
|
|
993 */ |
|
|
994 spam_secs = db_parms.ex_secs[DCC_CK_FUZ2].spam; |
|
|
995 if (spam_secs != 0 |
|
|
996 && spam_secs <= DB_EXPIRE_SPAMSECS_DEF |
|
|
997 && new_dbsize > db_max_byte |
|
|
998 && (db_ratio = (db_max_byte / new_dbsize)) < 1.0) { |
|
|
999 def_exp_ratio = (spam_secs * db_ratio) / DB_EXPIRE_SPAMSECS_DEF; |
|
|
1000 |
|
|
1001 /* change the two durations together and so with same errors */ |
|
|
1002 def_expire_spamsecs = DB_EXPIRE_SPAMSECS_DEF * def_exp_ratio; |
|
|
1003 def_expire_secs = DB_EXPIRE_SECS_DEF * def_exp_ratio; |
|
|
1004 |
|
|
1005 def_expire_secs -= def_expire_secs % (60*60); |
|
|
1006 if (def_expire_secs < DB_EXPIRE_SECS_DEF_MIN) |
|
|
1007 def_expire_secs = DB_EXPIRE_SECS_DEF_MIN; |
|
|
1008 |
|
|
1009 def_expire_spamsecs -= def_expire_spamsecs % (24*60*60); |
|
|
1010 if (def_expire_spamsecs < DB_EXPIRE_SPAMSECS_DEF_MIN) |
|
|
1011 def_expire_spamsecs = DB_EXPIRE_SPAMSECS_DEF_MIN; |
|
|
1012 |
|
|
1013 #if DB_MIN_MBYTE == 0 && !defined(GOT_PHYSMEM) |
|
|
1014 if (def_expire_secs == DB_EXPIRE_SECS_DEF_MIN |
|
|
1015 || def_expire_spamsecs == DB_EXPIRE_SPAMSECS_DEF_MIN) |
|
|
1016 quiet_trace_msg("cannot determine physical RAM; rebuild" |
|
|
1017 " with ./configure with-db-memory"); |
|
|
1018 #endif |
|
|
1019 return; |
|
|
1020 } |
|
|
1021 |
|
|
1022 def_exp_ratio = 1.0; |
|
|
1023 |
|
|
1024 /* if the defaults do not need to be reduced now but they |
|
|
1025 * were reduced before, then relax them gently */ |
|
|
1026 if (spam_secs < DB_EXPIRE_SPAMSECS_DEF) { |
|
|
1027 dcc_ts2timeval(&tv, &db_parms.ex_spam[DCC_CK_FUZ2]); |
|
|
1028 secs = clean_start.tv_sec - tv.tv_sec; |
|
|
1029 if (secs > 0 |
|
|
1030 && secs < DB_EXPIRE_SPAMSECS_DEF) |
|
|
1031 def_expire_spamsecs = secs; |
|
|
1032 |
|
|
1033 dcc_ts2timeval(&tv, &db_parms.ex_all[DCC_CK_FUZ2]); |
|
|
1034 secs = clean_start.tv_sec - tv.tv_sec; |
|
|
1035 if (secs > 0 |
|
|
1036 && secs < DB_EXPIRE_SECS_DEF) |
|
|
1037 def_expire_secs = secs; |
|
|
1038 } |
|
|
1039 } |
|
|
1040 |
|
|
1041 |
|
|
1042 |
|
|
1043 /* copy the existing database, discard junk and old entries */ |
|
|
1044 static u_char /* 1=done 0=database broken */ |
|
|
1045 expire(DB_PTR old_db_csize) |
|
|
1046 { |
|
|
1047 #define EXPIRE_BAIL() {alarm(0); flod_unmap(0, 0); db_close(0); return 0;} |
|
|
1048 |
|
|
1049 DCC_TS ts; |
|
|
1050 u_char emptied, reduced_defaults; |
|
|
1051 u_char old_ok[DCC_DIM_CKS]; |
|
|
1052 DB_RCD rcd, new; |
|
|
1053 const DB_RCD_CK *rcd_ck, *rcd_ck2; |
|
|
1054 DB_RCD_CK *new_ck; |
|
|
1055 DCC_TGTS tgts_raw, ck_tgts; |
|
|
1056 u_char needed, obs_lvl, timely; |
|
|
1057 int old_num_cks, new_num_cks, nokeep_num_cks; |
|
|
1058 DB_PTR min_confirm_pos, next_adj_pos; |
|
|
1059 FLOD_MMAP *mp; |
|
|
1060 DCC_CK_TYPES prev_type, type, type2; |
|
|
1061 int rcd_len; |
|
|
1062 struct stat sb; |
|
|
1063 time_t need_unlock; |
|
|
1064 int i; |
|
|
1065 |
|
|
1066 reduced_defaults = 0; |
|
|
1067 if (expire_secs < 0) { |
|
|
1068 adj_def_expire(); |
|
|
1069 if (def_expire_secs > expire_spamsecs |
|
|
1070 && expire_spamsecs > 0) { |
|
|
1071 expire_secs = expire_spamsecs; |
|
|
1072 } else { |
|
|
1073 if (def_expire_secs != DB_EXPIRE_SECS_DEF |
|
|
1074 && def_exp_ratio != 1.0) |
|
|
1075 reduced_defaults = 1; |
|
|
1076 expire_secs = def_expire_secs; |
|
|
1077 } |
|
|
1078 } |
|
|
1079 if (expire_spamsecs < 0) { |
|
|
1080 adj_def_expire(); |
|
|
1081 if (def_expire_spamsecs < expire_secs) { |
|
|
1082 expire_spamsecs = expire_secs; |
|
|
1083 } else { |
|
|
1084 if (def_expire_spamsecs != DB_EXPIRE_SPAMSECS_DEF |
|
|
1085 && def_exp_ratio != 1.0) |
|
|
1086 reduced_defaults = 1; |
|
|
1087 expire_spamsecs = def_expire_spamsecs; |
|
|
1088 } |
|
|
1089 } |
|
|
1090 |
|
|
1091 if (expire_spamsecs > 0 && expire_spamsecs < expire_secs) |
|
|
1092 dcc_logbad(EX_USAGE, |
|
|
1093 "spam expiration -E must be longer than -e"); |
|
|
1094 |
|
|
1095 expired_rcds = 0; |
|
|
1096 expired_cks = 0; |
|
|
1097 kept_cks = white_cks; |
|
|
1098 need_unlock = 0; |
|
|
1099 report_progress_init(); |
|
|
1100 |
|
|
1101 /* Compute timestamps for records we keep. |
|
|
1102 * Use the values from the previous use of dbclean as defaults |
|
|
1103 * unless they are bogus */ |
|
|
1104 memset(old_ok, 0, sizeof(old_ok)); |
|
|
1105 dcc_secs2ts(&ts, clean_start.tv_sec); |
|
|
1106 for (type = DCC_CK_TYPE_FIRST; type <= DCC_CK_TYPE_LAST; ++type) { |
|
|
1107 DB_EX_SEC *th = &db_parms.ex_secs[type]; |
|
|
1108 |
|
|
1109 if (DB_TEST_NOKEEP(db_parms.nokeep_cks, type)) |
|
|
1110 continue; |
|
|
1111 if (DCC_CK_IS_REP_OP(grey_on, type)) |
|
|
1112 continue; |
|
|
1113 |
|
|
1114 if (th->spam <= 0 || th->spam > DB_EXPIRE_SECS_MAX) |
|
|
1115 continue; |
|
|
1116 if (th->all <= 0 || th->all > th->spam) |
|
|
1117 continue; |
|
|
1118 |
|
|
1119 if (dcc_ts_newer_ts(&db_parms.ex_spam[type], &ts)) |
|
|
1120 continue; |
|
|
1121 if (dcc_ts_newer_ts(&db_parms.ex_all[type], &ts)) |
|
|
1122 continue; |
|
|
1123 |
|
|
1124 old_ok[type] = 1; /* old values for this type are ok */ |
|
|
1125 } |
|
|
1126 |
|
|
1127 for (type = DCC_CK_TYPE_FIRST; type <= DCC_CK_TYPE_LAST; ++type) { |
|
|
1128 DB_EX_SEC *new_th = &new_ex_secs[type]; |
|
|
1129 DB_EX_TS_TYPE *new_ts = &new_ex_ts[type]; |
|
|
1130 int old_all = db_parms.ex_secs[type].all; |
|
|
1131 int old_spam = db_parms.ex_secs[type].spam; |
|
|
1132 |
|
|
1133 if (type == DCC_CK_SRVR_ID) { |
|
|
1134 /* keep server-ID declarations 5 weeks or a week longer |
|
|
1135 * than reputations so that they will be flooded 1st */ |
|
|
1136 new_th->all = DB_EXPIRE_SRVR_ID_SECS; |
|
|
1137 new_th->spam = DB_EXPIRE_SRVR_ID_SECS; |
|
|
1138 |
|
|
1139 } else if (grey_on) { |
|
|
1140 if (old_ok[type]) { |
|
|
1141 /* This is the path by which the dccd -G |
|
|
1142 * parameters are used. */ |
|
|
1143 new_th->all = old_all; |
|
|
1144 new_th->spam = old_spam; |
|
|
1145 } else if (DCC_CK_IS_GREY_TRIPLE(1, type)) { |
|
|
1146 new_th->all = DEF_GREY_WINDOW; |
|
|
1147 new_th->spam = DEF_GREY_WHITE; |
|
|
1148 } else if (DCC_CK_IS_GREY_MSG(1, type) |
|
|
1149 || type == DCC_CK_BODY) { |
|
|
1150 new_th->all = DEF_GREY_WINDOW; |
|
|
1151 new_th->spam = DEF_GREY_WINDOW; |
|
|
1152 } else { |
|
|
1153 new_th->all = 1; |
|
|
1154 new_th->spam = 1; |
|
|
1155 } |
|
|
1156 } else if (have_expire_parms < 0 && old_ok[type] |
|
|
1157 && (db_parms.flags & DB_PARM_EXP_SET)) { |
|
|
1158 /* use the old durations they are valid |
|
|
1159 * and we have no expiriation parameters */ |
|
|
1160 new_th->all = old_all; |
|
|
1161 new_th->spam = old_spam; |
|
|
1162 |
|
|
1163 } else { |
|
|
1164 new_th->all = expire_secs; |
|
|
1165 new_th->spam = (DCC_CK_LONG_TERM(type) |
|
|
1166 ? expire_spamsecs |
|
|
1167 : expire_secs); |
|
|
1168 if (reduced_defaults) { |
|
|
1169 quiet_trace_msg("adjust default by" |
|
|
1170 " %4.2f to -e%dhours" |
|
|
1171 " -E%ddays", |
|
|
1172 def_exp_ratio, |
|
|
1173 expire_secs/(60*60), |
|
|
1174 expire_spamsecs |
|
|
1175 / (24*60*60)); |
|
|
1176 reduced_defaults = 0; |
|
|
1177 } |
|
|
1178 } |
|
|
1179 |
|
|
1180 /* compute oldest timestamp for this type of checksum, |
|
|
1181 * without going crazy with "-Enever" */ |
|
|
1182 dcc_secs2ts(&new_ts->spam, |
|
|
1183 clean_start.tv_sec - min(clean_start.tv_sec, |
|
|
1184 new_th->spam)); |
|
|
1185 dcc_secs2ts(&new_ts->all, |
|
|
1186 clean_start.tv_sec - min(clean_start.tv_sec, |
|
|
1187 new_th->all)); |
|
|
1188 } |
|
|
1189 |
|
|
1190 /* put the timestampes into the new file */ |
|
|
1191 write_new_hdr(1); |
|
|
1192 |
|
|
1193 /* if we are running as root, |
|
|
1194 * don't change the owner of the database */ |
|
|
1195 if (getuid() == 0) { |
|
|
1196 if (0 > fstat(old_db_fd, &sb)) |
|
|
1197 dcc_logbad(EX_IOERR, "fstat(%s): %s", |
|
|
1198 old_db_nm, ERROR_STR()); |
|
|
1199 if (0 > fchown(new_db_fd, sb.st_uid, sb.st_gid)) |
|
|
1200 dcc_logbad(EX_IOERR, "fchown(%s,%d,%d): %s", |
|
|
1201 new_db_nm, (int)sb.st_uid, (int)sb.st_gid, |
|
|
1202 ERROR_STR()); |
|
|
1203 } |
|
|
1204 |
|
|
1205 if (DB_PTR_BASE != lseek(old_db_fd, DB_PTR_BASE, SEEK_SET)) |
|
|
1206 dcc_logbad(EX_IOERR, "lseek(%s,%d): %s", |
|
|
1207 cur_db_nm, DB_PTR_BASE, ERROR_STR()); |
|
|
1208 read_rcd_invalidate(0); |
|
|
1209 |
|
|
1210 flod_mmap(0, &db_parms.sn, 0, 1, 1); |
|
|
1211 if (flod_mmaps) |
|
|
1212 memcpy(&new_flod_mmaps, flod_mmaps, sizeof(new_flod_mmaps)); |
|
|
1213 min_confirm_pos = new_flod_mmaps.delay_pos; |
|
|
1214 next_adj_pos = DB_PTR_BASE; |
|
|
1215 for (mp = new_flod_mmaps.mmaps; |
|
|
1216 mp <= LAST(new_flod_mmaps.mmaps); |
|
|
1217 ++mp) { |
|
|
1218 if (mp->rem_hostname[0] == '\0') { |
|
|
1219 mp->flags &= ~FLODMAP_FG_MARK; |
|
|
1220 } else { |
|
|
1221 mp->flags |= FLODMAP_FG_MARK; |
|
|
1222 if (min_confirm_pos > mp->confirm_pos) |
|
|
1223 min_confirm_pos = mp->confirm_pos; |
|
|
1224 } |
|
|
1225 } |
|
|
1226 adj_delay_pos = (new_flod_mmaps.delay_pos != 0) ? 1 : 0; |
|
|
1227 |
|
|
1228 emptied = cleardb; |
|
|
1229 dcc_timeval2ts(&new_flod_mmaps.sn, &clean_start, 0); |
|
|
1230 |
|
|
1231 /* copy the old file to the new, |
|
|
1232 * discarding and compressing old data as we go */ |
|
|
1233 for (old_db_pos = DB_PTR_BASE; |
|
|
1234 old_db_pos < old_db_csize; |
|
|
1235 old_db_pos += rcd_len) { |
|
|
1236 if (--progress_rpt_cnt <= 0) |
|
|
1237 need_unlock += report_progress(0, " processed", |
|
|
1238 "MBytes", |
|
|
1239 old_db_pos, old_db_csize, |
|
|
1240 1024*1024); |
|
|
1241 |
|
|
1242 if (old_db_pos == next_adj_pos) |
|
|
1243 next_adj_pos = adj_mmap(); |
|
|
1244 |
|
|
1245 if (clean_mode != REPAIR_MODE) { |
|
|
1246 /* read the record by mapping if not repairing */ |
|
|
1247 if (!db_map_rcd(0, &db_sts.rcd, old_db_pos, &rcd_len)) |
|
|
1248 EXPIRE_BAIL(); |
|
|
1249 memcpy(&rcd, db_sts.rcd.d.r, rcd_len); |
|
|
1250 } else { |
|
|
1251 rcd_len = read_rcd(0, &rcd, |
|
|
1252 old_db_fd, old_db_pos, cur_db_nm); |
|
|
1253 if (rcd_len <= 0) { |
|
|
1254 if (rcd_len == 0) |
|
|
1255 dcc_error_msg("unexpected EOF in %s at " |
|
|
1256 L_HPAT" instead of " |
|
|
1257 L_HPAT, |
|
|
1258 cur_db_nm, |
|
|
1259 old_db_pos, |
|
|
1260 old_db_csize); |
|
|
1261 /* give up and ask our neighbors to rewind */ |
|
|
1262 emptied = 1; |
|
|
1263 old_db_pos = old_db_csize; |
|
|
1264 break; |
|
|
1265 } |
|
|
1266 } |
|
|
1267 |
|
|
1268 /* skip end-of-page padding */ |
|
|
1269 if (rcd_len == sizeof(rcd)-sizeof(rcd.cks)) |
|
|
1270 continue; |
|
|
1271 |
|
|
1272 if (DB_RCD_ID(&rcd) == DCC_ID_WHITE) { |
|
|
1273 /* skip whitelist entries if whitelist source is ok */ |
|
|
1274 if (!keep_white) |
|
|
1275 continue; |
|
|
1276 /* refresh whitelist entries if source is bad */ |
|
|
1277 dcc_timeval2ts(&rcd.ts, &clean_start, 0); |
|
|
1278 } |
|
|
1279 |
|
|
1280 old_num_cks = DB_NUM_CKS(&rcd); |
|
|
1281 |
|
|
1282 /* expire or throw away deleted reports */ |
|
|
1283 tgts_raw = DB_TGTS_RCD_RAW(&rcd); |
|
|
1284 if (tgts_raw == 0) { |
|
|
1285 ++expired_rcds; |
|
|
1286 expired_cks += old_num_cks; |
|
|
1287 continue; |
|
|
1288 } |
|
|
1289 if (tgts_raw > DCC_TGTS_MAX_DB) { |
|
|
1290 dcc_error_msg("discarding report at "L_HPAT |
|
|
1291 " with bogus target count %#x", |
|
|
1292 old_db_pos, tgts_raw); |
|
|
1293 ++expired_rcds; |
|
|
1294 expired_cks += old_num_cks; |
|
|
1295 continue; |
|
|
1296 } |
|
|
1297 |
|
|
1298 if (dcc_ts_newer_ts(&rcd.ts, &future_ts)) { |
|
|
1299 static int whines = 0; |
|
|
1300 if (whines < 50) |
|
|
1301 dcc_error_msg("discarding report at "L_HPAT |
|
|
1302 " from the future %s%s", |
|
|
1303 old_db_pos, |
|
|
1304 ts2str_err(&rcd.ts), |
|
|
1305 ++whines >= 20 |
|
|
1306 ? "; stop complaining" |
|
|
1307 : ""); |
|
|
1308 ++expired_rcds; |
|
|
1309 expired_cks += old_num_cks; |
|
|
1310 continue; |
|
|
1311 } |
|
|
1312 |
|
|
1313 |
|
|
1314 needed = 0; |
|
|
1315 obs_lvl = 0; |
|
|
1316 timely = 1; |
|
|
1317 nokeep_num_cks = 0; |
|
|
1318 memcpy(&new, &rcd, sizeof(new)-sizeof(new.cks)); |
|
|
1319 new.fgs_num_cks &= (DB_RCD_FG_TRIM | DB_RCD_FG_SUMRY |
|
|
1320 | DB_RCD_FG_DELAY); |
|
|
1321 new_ck = new.cks; |
|
|
1322 for (prev_type = DCC_CK_INVALID, rcd_ck = rcd.cks; |
|
|
1323 rcd_ck < &rcd.cks[old_num_cks]; |
|
|
1324 prev_type = type, ++rcd_ck) { |
|
|
1325 type = DB_CK_TYPE(rcd_ck); |
|
|
1326 if (!DCC_CK_OK_DB(grey_on, type)) { |
|
|
1327 static int whines = 0; |
|
|
1328 if (whines < 20) |
|
|
1329 dcc_error_msg("discarding %s" |
|
|
1330 " checksum at "L_HPAT"%s", |
|
|
1331 DB_TYPE2STR(type), |
|
|
1332 old_db_pos, |
|
|
1333 ++whines >= 20 |
|
|
1334 ? "; stop complaining" |
|
|
1335 : ""); |
|
|
1336 ++expired_cks; |
|
|
1337 new.fgs_num_cks |= DB_RCD_FG_TRIM; |
|
|
1338 new.fgs_num_cks &= ~DB_RCD_FG_DELAY; |
|
|
1339 continue; |
|
|
1340 } |
|
|
1341 |
|
|
1342 if (type <= prev_type |
|
|
1343 && prev_type != DCC_CK_FLOD_PATH) { |
|
|
1344 dcc_error_msg("discarding out of order %s" |
|
|
1345 " checksum at "L_HPAT, |
|
|
1346 DB_TYPE2STR(type), |
|
|
1347 old_db_pos); |
|
|
1348 ++expired_cks; |
|
|
1349 new.fgs_num_cks |= DB_RCD_FG_TRIM; |
|
|
1350 new.fgs_num_cks &= ~DB_RCD_FG_DELAY; |
|
|
1351 continue; |
|
|
1352 } |
|
|
1353 |
|
|
1354 /* Silently discard pure junk from other servers, |
|
|
1355 * provided it is junk by default */ |
|
|
1356 if (DB_TEST_NOKEEP(db_parms.nokeep_cks, type) |
|
|
1357 && DB_GLOBAL_NOKEEP(grey_on, type) |
|
|
1358 && type != DCC_CK_FLOD_PATH |
|
|
1359 && type != DCC_CK_SRVR_ID |
|
|
1360 && DB_RCD_ID(&rcd) != DCC_ID_WHITE) { |
|
|
1361 ++expired_cks; |
|
|
1362 continue; |
|
|
1363 } |
|
|
1364 |
|
|
1365 /* Keep paths except on old records or records that |
|
|
1366 * have been trimmed or compressed. |
|
|
1367 * Never remove paths from server-ID declarations. */ |
|
|
1368 if (type == DCC_CK_FLOD_PATH) { |
|
|
1369 if (DB_RCD_TRIMMED(&new) |
|
|
1370 || DB_RCD_ID(&new) == DCC_ID_COMP) |
|
|
1371 continue; |
|
|
1372 /* forget line number on old whitelist entry */ |
|
|
1373 if (DB_RCD_ID(&rcd) == DCC_ID_WHITE) |
|
|
1374 continue; |
|
|
1375 rcd_ck2 = rcd_ck+1; |
|
|
1376 for (;;) { |
|
|
1377 type2 = DB_CK_TYPE(rcd_ck2); |
|
|
1378 if (type2 == DCC_CK_SRVR_ID |
|
|
1379 || !dcc_ts_older_ts(&rcd.ts, |
|
|
1380 &new_ex_ts[type2 |
|
|
1381 ].all)) { |
|
|
1382 /* keep this path since this report |
|
|
1383 * is a server-ID declaration |
|
|
1384 * or not old */ |
|
|
1385 *new_ck = *rcd_ck; |
|
|
1386 ++new_ck; |
|
|
1387 ++new.fgs_num_cks; |
|
|
1388 ++nokeep_num_cks; |
|
|
1389 break; |
|
|
1390 } |
|
|
1391 if (++rcd_ck2>=&rcd.cks[old_num_cks]) { |
|
|
1392 /* we are discarding this path */ |
|
|
1393 new.fgs_num_cks |= DB_RCD_FG_TRIM; |
|
|
1394 new.fgs_num_cks &= ~DB_RCD_FG_DELAY; |
|
|
1395 break; |
|
|
1396 } |
|
|
1397 } |
|
|
1398 continue; |
|
|
1399 } |
|
|
1400 |
|
|
1401 if (!dcc_ts_older_ts(&rcd.ts, &new_ex_ts[type].all)) { |
|
|
1402 /* This report is recent. |
|
|
1403 * However, obsolete or junk checksums |
|
|
1404 * don't make the report needed */ |
|
|
1405 if (DB_TEST_NOKEEP(db_parms.nokeep_cks, type) |
|
|
1406 && DB_RCD_ID(&rcd) != DCC_ID_WHITE) { |
|
|
1407 ++nokeep_num_cks; |
|
|
1408 } else if (DB_CK_OBS(rcd_ck)) { |
|
|
1409 /* This checksum is obsolete. |
|
|
1410 * If it has the highest level of |
|
|
1411 * fuzziness, then it controls whether |
|
|
1412 * the whole report is needed,. */ |
|
|
1413 if (obs_lvl < db_ck_fuzziness[type]) { |
|
|
1414 obs_lvl = db_ck_fuzziness[type]; |
|
|
1415 needed = 0; |
|
|
1416 } |
|
|
1417 } else { |
|
|
1418 /* This checksum is not obsolete. |
|
|
1419 * If it is at least as fuzzy as any |
|
|
1420 * other checksum, then it can say |
|
|
1421 * the report is needed */ |
|
|
1422 if (obs_lvl <= db_ck_fuzziness[type]) { |
|
|
1423 obs_lvl = db_ck_fuzziness[type]; |
|
|
1424 needed = 1; |
|
|
1425 } |
|
|
1426 } |
|
|
1427 |
|
|
1428 } else { |
|
|
1429 /* This checksum is at least somewhat old. |
|
|
1430 * Throw away delete requests |
|
|
1431 * and other servers' useless checksums */ |
|
|
1432 if (tgts_raw == DCC_TGTS_DEL |
|
|
1433 || DB_TEST_NOKEEP(db_parms.nokeep_cks, |
|
|
1434 type)) { |
|
|
1435 ++expired_cks; |
|
|
1436 new.fgs_num_cks |= DB_RCD_FG_TRIM; |
|
|
1437 new.fgs_num_cks &= ~DB_RCD_FG_DELAY; |
|
|
1438 continue; |
|
|
1439 } |
|
|
1440 /* Throw away old obsolete checksums |
|
|
1441 * and entire reports if the fuzziest |
|
|
1442 * checksum is obsolete */ |
|
|
1443 if (DB_CK_OBS(rcd_ck)) { |
|
|
1444 if (obs_lvl < db_ck_fuzziness[type]) { |
|
|
1445 obs_lvl = db_ck_fuzziness[type]; |
|
|
1446 needed = 0; |
|
|
1447 } |
|
|
1448 ++expired_cks; |
|
|
1449 new.fgs_num_cks |= DB_RCD_FG_TRIM; |
|
|
1450 new.fgs_num_cks &= ~DB_RCD_FG_DELAY; |
|
|
1451 continue; |
|
|
1452 } |
|
|
1453 |
|
|
1454 /* old summaries are unneeded, because |
|
|
1455 * they have already been flooded. |
|
|
1456 * They do not contribute to local counts */ |
|
|
1457 if (DB_RCD_SUMRY(&rcd)) |
|
|
1458 continue; |
|
|
1459 |
|
|
1460 /* The checksum is old enough to compress, so |
|
|
1461 * mark the record as eligible for splitting. */ |
|
|
1462 timely = 0; |
|
|
1463 |
|
|
1464 /* Discard this checksum if its ultimate total |
|
|
1465 * is low or ancient |
|
|
1466 * or if it reaches spam after this report. |
|
|
1467 * To determine the ultimate total, we must |
|
|
1468 * have a hash table to find the newest record, |
|
|
1469 * which contains the final total */ |
|
|
1470 if (clean_mode != REPAIR_MODE) { |
|
|
1471 i = get_lead(type, rcd_ck); |
|
|
1472 if (i < 0) |
|
|
1473 EXPIRE_BAIL(); |
|
|
1474 if (!i) { |
|
|
1475 ++expired_cks; |
|
|
1476 new.fgs_num_cks |= DB_RCD_FG_TRIM; |
|
|
1477 new.fgs_num_cks &= ~DB_RCD_FG_DELAY; |
|
|
1478 continue; |
|
|
1479 } |
|
|
1480 } |
|
|
1481 |
|
|
1482 if (obs_lvl <= db_ck_fuzziness[type]) { |
|
|
1483 /* Since we did not delete this |
|
|
1484 * checksum, we need the record if this |
|
|
1485 * checksum is fuzzy enough to control |
|
|
1486 * our need. */ |
|
|
1487 needed = 1; |
|
|
1488 /* If this is the fuzziest checksum we |
|
|
1489 * have seen, then preceding and so |
|
|
1490 * less fuzzy checksums are obsolete, |
|
|
1491 * if they are old. |
|
|
1492 * Assume that checksums are ordered |
|
|
1493 * in the record by fuzziness. */ |
|
|
1494 if (obs_lvl < db_ck_fuzziness[type]) { |
|
|
1495 obs_lvl = db_ck_fuzziness[type]; |
|
|
1496 if (obs_lvl != DCC_CK_FUZ_LVL_REP |
|
|
1497 && !grey_on) |
|
|
1498 fuzzy_obs(&new, &new_ck); |
|
|
1499 } |
|
|
1500 } |
|
|
1501 } |
|
|
1502 |
|
|
1503 /* Keep this checksum if we decide the whole report |
|
|
1504 * is needed. */ |
|
|
1505 *new_ck = *rcd_ck; |
|
|
1506 |
|
|
1507 ++new_ck; |
|
|
1508 ++new.fgs_num_cks; |
|
|
1509 } |
|
|
1510 |
|
|
1511 /* occassionally let the daemon work with the old file */ |
|
|
1512 if (need_unlock >= UNLOCK_INTERVAL_USECS) { |
|
|
1513 need_unlock = 0; |
|
|
1514 if (!standalone && !db_unlock()) |
|
|
1515 EXPIRE_BAIL(); |
|
|
1516 } |
|
|
1517 |
|
|
1518 /* if none of its checksums are needed, |
|
|
1519 * then discard the entire record */ |
|
|
1520 if (!needed) { |
|
|
1521 expired_cks += DB_NUM_CKS(&new); |
|
|
1522 ++expired_rcds; |
|
|
1523 continue; |
|
|
1524 } |
|
|
1525 |
|
|
1526 new_num_cks = DB_NUM_CKS(&new); |
|
|
1527 kept_cks += new_num_cks - nokeep_num_cks; |
|
|
1528 |
|
|
1529 /* Put the new record into the new file. |
|
|
1530 * |
|
|
1531 * If all of the record is recent, if it contains 1 checksum, |
|
|
1532 * or if all of its totals are the same, then simply add it. |
|
|
1533 * |
|
|
1534 * Otherwise, divide it into records of identical counts |
|
|
1535 * to allow compression or combining with other records. */ |
|
|
1536 if (new_num_cks > 1 |
|
|
1537 && (!timely |
|
|
1538 || DB_RCD_ID(&new) == DCC_ID_COMP |
|
|
1539 || DB_RCD_TRIMMED(&new))) { |
|
|
1540 for (;;) { |
|
|
1541 /* skip the checksums that have the same total |
|
|
1542 * as the first checksum to leave them with the |
|
|
1543 * original new report */ |
|
|
1544 new_ck = new.cks; |
|
|
1545 ck_tgts = DB_TGTS_CK(new_ck); |
|
|
1546 for (i = 1; i < new_num_cks; ++i) { |
|
|
1547 ++new_ck; |
|
|
1548 if (DB_TGTS_CK(new_ck) != ck_tgts) |
|
|
1549 break; |
|
|
1550 } |
|
|
1551 if (new_num_cks <= i) |
|
|
1552 break; |
|
|
1553 new_num_cks -= i; |
|
|
1554 |
|
|
1555 /* write the checksums with the common total */ |
|
|
1556 new.srvr_id_auth = DCC_ID_COMP; |
|
|
1557 new.fgs_num_cks = i; |
|
|
1558 if (!write_new_rcd(&new, |
|
|
1559 sizeof(new) - sizeof(new.cks) |
|
|
1560 + i*sizeof(new.cks[0]))) |
|
|
1561 EXPIRE_BAIL(); |
|
|
1562 |
|
|
1563 /* handle the remaining checksums */ |
|
|
1564 new.fgs_num_cks = new_num_cks; |
|
|
1565 memmove(&new.cks[0], &new.cks[i], |
|
|
1566 new_num_cks*sizeof(new.cks[0])); |
|
|
1567 } |
|
|
1568 } |
|
|
1569 |
|
|
1570 /* write the rest (or all) of the new record */ |
|
|
1571 if (!write_new_rcd(&new, |
|
|
1572 sizeof(new) - sizeof(new.cks) |
|
|
1573 + new_num_cks*sizeof(new.cks[0]))) |
|
|
1574 EXPIRE_BAIL(); |
|
|
1575 } |
|
|
1576 write_new_flush(1); |
|
|
1577 alarm(0); |
|
|
1578 |
|
|
1579 /* do final adjustment of the flooding positions */ |
|
|
1580 adj_mmap(); |
|
|
1581 /* force them to be right if the system crashed with the |
|
|
1582 * flod.map file on the disk more up to date and so after the |
|
|
1583 * database file on the disk */ |
|
|
1584 for (mp = new_flod_mmaps.mmaps; |
|
|
1585 mp <= LAST(new_flod_mmaps.mmaps); |
|
|
1586 ++mp) { |
|
|
1587 if (mp->rem_hostname[0] != '\0' |
|
|
1588 && mp->confirm_pos > new_db_csize) |
|
|
1589 mp->confirm_pos = new_db_csize; |
|
|
1590 } |
|
|
1591 |
|
|
1592 /* We are finished with the old file. |
|
|
1593 * Mark all of its pages MADV_DONTNEED */ |
|
|
1594 rel_db_states(); |
|
|
1595 i = (db_unload(0, 2) != 0); |
|
|
1596 if (!db_close(1)) |
|
|
1597 i = 0; |
|
|
1598 |
|
|
1599 write_new_hdr(emptied); |
|
|
1600 report_progress(1, " processed", "MBytes", |
|
|
1601 old_db_pos, old_db_csize, 1024*1024); |
|
|
1602 if (grey_on) |
|
|
1603 quiet_trace_msg("expired %d records and %d checksums in %s", |
|
|
1604 expired_rcds, expired_cks, cur_db_nm); |
|
|
1605 else |
|
|
1606 quiet_trace_msg("expired %d records and %d checksums," |
|
|
1607 " obsoleted %d checksums in %s", |
|
|
1608 expired_rcds, expired_cks, obs_rcds, cur_db_nm); |
|
|
1609 return i; |
|
|
1610 } |
|
|
1611 |
|
|
1612 |
|
|
1613 |
|
|
1614 /* copy the database copy while doing minimal expiring */ |
|
|
1615 static u_char |
|
|
1616 copy_db(void) |
|
|
1617 { |
|
|
1618 static DB_VERSION_BUF old_version4 = DB_VERSION4_STR; |
|
|
1619 static DB_VERSION_BUF old_version3 = DB_VERSION3_STR; |
|
|
1620 union { |
|
|
1621 DB_HDR hdr; |
|
|
1622 DB_V4_PARMS v4; |
|
|
1623 DB_V3_PARMS v3; |
|
|
1624 } old_db; |
|
|
1625 struct timeval sn; |
|
|
1626 |
|
|
1627 /* do not lock the old database because the daemon must continue |
|
|
1628 * to answer requests */ |
|
|
1629 if (old_db_fd < 0) { |
|
|
1630 old_db_fd = open(cur_db_nm, O_RDONLY, 0); |
|
|
1631 if (old_db_fd == -1) |
|
|
1632 dcc_logbad(EX_IOERR, "open(%s): %s", |
|
|
1633 cur_db_nm, ERROR_STR()); |
|
|
1634 } |
|
|
1635 |
|
|
1636 if (!read_db_hdr(dcc_emsg, &old_db.hdr, old_db_fd, cur_db_nm)) |
|
|
1637 dcc_logbad(dcc_ex_code, "%s", dcc_emsg); |
|
|
1638 if (memcmp(old_db.hdr.p.version, db_version_buf, |
|
|
1639 sizeof(old_db.hdr.p.version))) { |
|
|
1640 if (!memcmp(old_db.v4.version, old_version4, |
|
|
1641 sizeof(old_db.v4.version))) { |
|
|
1642 memset(&old_db_parms, 0, |
|
|
1643 sizeof(old_db_parms)); |
|
|
1644 memcpy(old_db_parms.version, db_version_buf, |
|
|
1645 sizeof(old_db_parms.version)); |
|
|
1646 |
|
|
1647 old_db_parms.db_csize = old_db.v4.db_csize; |
|
|
1648 old_db_parms.pagesize = old_db.v4.pagesize; |
|
|
1649 old_db_parms.sn = old_db.v4.sn; |
|
|
1650 old_db_parms.cleared = old_db.v4.cleared; |
|
|
1651 old_db_parms.cleaned = old_db.v4.cleaned; |
|
|
1652 old_db_parms.cleaned_cron = old_db.v4.cleaned_cron; |
|
|
1653 memcpy(old_db_parms.ex_spam, old_db.v4.ex_spam, |
|
|
1654 sizeof(old_db_parms.ex_spam)); |
|
|
1655 memcpy(old_db_parms.ex_all, old_db.v4.ex_spam, |
|
|
1656 sizeof(old_db_parms.ex_all)); |
|
|
1657 memcpy(old_db_parms.ex_secs, old_db.v4.ex_secs, |
|
|
1658 sizeof(old_db_parms.ex_secs)); |
|
|
1659 old_db_parms.nokeep_cks = old_db.v4.nokeep_cks; |
|
|
1660 old_db_parms.flags = old_db.v4.flags; |
|
|
1661 old_db_parms.old_db_csize = old_db.v4.old_db_csize; |
|
|
1662 old_db_parms.db_added = old_db.v4.db_added; |
|
|
1663 old_db_parms.hash_used = old_db.v4.hash_used; |
|
|
1664 old_db_parms.old_hash_used = old_db.v4.old_hash_used; |
|
|
1665 old_db_parms.hash_added = old_db.v4.hash_added; |
|
|
1666 old_db_parms.rate_secs = old_db.v4.rate_secs; |
|
|
1667 old_db_parms.last_rate_sec = old_db.v4.last_rate_sec; |
|
|
1668 old_db_parms.old_kept_cks = old_db.v4.old_kept_cks; |
|
|
1669 |
|
|
1670 } else if (!memcmp(old_db.v3.version, old_version3, |
|
|
1671 sizeof(old_db.v3.version))) { |
|
|
1672 memset(&old_db_parms, 0, |
|
|
1673 sizeof(old_db_parms)); |
|
|
1674 memcpy(old_db_parms.version, db_version_buf, |
|
|
1675 sizeof(old_db_parms.version)); |
|
|
1676 |
|
|
1677 old_db_parms.db_csize = old_db.v3.db_csize; |
|
|
1678 old_db_parms.pagesize = old_db.v3.pagesize; |
|
|
1679 old_db_parms.sn = old_db.v3.sn; |
|
|
1680 memcpy(old_db_parms.ex_spam, old_db.v3.ex_spam, |
|
|
1681 sizeof(old_db_parms.ex_spam)); |
|
|
1682 memcpy(old_db_parms.ex_secs, old_db.v3.ex_secs, |
|
|
1683 sizeof(old_db_parms.ex_secs)); |
|
|
1684 old_db_parms.nokeep_cks = old_db.v3.nokeep_cks; |
|
|
1685 if (old_db.v3.flags & DB_PARM_V3_FG_GREY) |
|
|
1686 old_db_parms.flags |= DB_PARM_FG_GREY; |
|
|
1687 if (old_db.v3.flags & DB_PARM_V3_FG_CLEARED) |
|
|
1688 old_db_parms.flags |= DB_PARM_FG_CLEARED; |
|
|
1689 old_db_parms.old_db_csize = old_db.v3.old_db_csize; |
|
|
1690 old_db_parms.db_added = old_db.v3.db_added; |
|
|
1691 old_db_parms.hash_used = old_db.v3.hash_used; |
|
|
1692 old_db_parms.old_hash_used = old_db.v3.old_hash_used; |
|
|
1693 old_db_parms.hash_added = old_db.v3.hash_added; |
|
|
1694 old_db_parms.rate_secs = old_db.v3.rate_secs; |
|
|
1695 old_db_parms.last_rate_sec = old_db.v3.last_rate_sec; |
|
|
1696 old_db_parms.old_kept_cks = old_db.v3.old_kept_cks; |
|
|
1697 |
|
|
1698 dcc_ts2timeval(&sn, &old_db_parms.sn); |
|
|
1699 old_db_parms.cleared = sn.tv_sec; |
|
|
1700 old_db_parms.cleaned = sn.tv_sec; |
|
|
1701 if (old_db.v3.flags & DB_PARM_V3_FG_SELF_CLEAN2) { |
|
|
1702 old_db_parms.cleared -= 2*24*60*60; |
|
|
1703 old_db_parms.cleaned -= 24*60*60; |
|
|
1704 } |
|
|
1705 } else { |
|
|
1706 dcc_logbad(EX_IOERR, "%s has the wrong magic \"%.*s\"", |
|
|
1707 cur_db_nm, |
|
|
1708 ISZ(DB_VERSION_BUF), old_db.hdr.p.version); |
|
|
1709 } |
|
|
1710 } else { |
|
|
1711 old_db_parms = old_db.hdr.p; |
|
|
1712 } |
|
|
1713 |
|
|
1714 db_parms.sn = old_db_parms.sn; |
|
|
1715 db_parms.cleared = old_db_parms.cleared; |
|
|
1716 db_parms.cleaned = old_db_parms.cleaned; |
|
|
1717 db_parms.cleaned_cron = old_db_parms.cleaned_cron; |
|
|
1718 memcpy(db_parms.ex_all, old_db_parms.ex_all, |
|
|
1719 sizeof(db_parms.ex_all)); |
|
|
1720 memcpy(db_parms.ex_spam, old_db_parms.ex_spam, |
|
|
1721 sizeof(db_parms.ex_spam)); |
|
|
1722 memcpy(&db_parms.ex_secs, &old_db_parms.ex_secs, |
|
|
1723 sizeof(db_parms.ex_secs)); |
|
|
1724 db_parms.nokeep_cks = old_db_parms.nokeep_cks; |
|
|
1725 db_parms.flags = old_db_parms.flags; |
|
|
1726 |
|
|
1727 set_db_tholds(db_parms.nokeep_cks); |
|
|
1728 |
|
|
1729 return expire(old_db_parms.db_csize); |
|
|
1730 } |
|
|
1731 |
|
|
1732 |
|
|
1733 |
|
|
1734 /* Copy any records from the old file to the new file that were |
|
|
1735 * added to the old file while we were creating the new file. */ |
|
|
1736 static u_char |
|
|
1737 catchup(DCC_EMSG emsg) |
|
|
1738 { |
|
|
1739 DB_HDR old_db_hdr; |
|
|
1740 DB_RCD rcd; |
|
|
1741 int rcd_len; |
|
|
1742 u_char result; |
|
|
1743 int count, old_count; |
|
|
1744 |
|
|
1745 /* Because dccd knows dbclean is running, dccd will have been |
|
|
1746 * keeping its header block more accurate than usual. */ |
|
|
1747 result = 1; |
|
|
1748 count = 0; |
|
|
1749 do { |
|
|
1750 old_count = count; |
|
|
1751 if (!read_db_hdr(dcc_emsg, &old_db_hdr, |
|
|
1752 old_db_fd, old_db_nm)) { |
|
|
1753 emsg = 0; |
|
|
1754 result = 0; |
|
|
1755 break; |
|
|
1756 } |
|
|
1757 if (old_db_hdr.p.db_csize < old_db_pos) { |
|
|
1758 dcc_error_msg("%s mysteriously truncated", old_db_nm); |
|
|
1759 result = 0; |
|
|
1760 break; |
|
|
1761 } |
|
|
1762 if ((off_t)old_db_pos != lseek(old_db_fd, old_db_pos, |
|
|
1763 SEEK_SET)) { |
|
|
1764 dcc_pemsg(EX_IOERR, emsg, "lseek(%s, "L_HPAT"): %s", |
|
|
1765 old_db_nm, old_db_pos, ERROR_STR()); |
|
|
1766 emsg = 0; |
|
|
1767 result = 0; |
|
|
1768 break; |
|
|
1769 } |
|
|
1770 read_rcd_invalidate(0); |
|
|
1771 while (old_db_pos < old_db_hdr.p.db_csize) { |
|
|
1772 rcd_len = read_rcd(emsg, &rcd, |
|
|
1773 old_db_fd, old_db_pos, old_db_nm); |
|
|
1774 if (rcd_len <= 0) { |
|
|
1775 if (rcd_len == 0) |
|
|
1776 dcc_pemsg(EX_IOERR, emsg, |
|
|
1777 "premature EOF in %s" |
|
|
1778 " at "L_HPAT |
|
|
1779 " instead of "L_HPAT, |
|
|
1780 old_db_nm, |
|
|
1781 old_db_pos, |
|
|
1782 old_db_hdr.p.db_csize); |
|
|
1783 emsg = 0; |
|
|
1784 result = 0; |
|
|
1785 break; |
|
|
1786 } |
|
|
1787 /* If something bad happens, we may not be able to |
|
|
1788 * go back to the old file. Carry on to get as much |
|
|
1789 * data as we can although we know the dccd daemon |
|
|
1790 * may croak when we release it */ |
|
|
1791 if (!db_add_rcd(emsg, &rcd)) { |
|
|
1792 emsg = 0; |
|
|
1793 result = 0; |
|
|
1794 break; |
|
|
1795 } |
|
|
1796 old_db_pos += rcd_len; |
|
|
1797 ++count; |
|
|
1798 } |
|
|
1799 } while (result && old_count != count); |
|
|
1800 |
|
|
1801 if (count > 0 && db_debug >= 1) |
|
|
1802 quiet_trace_msg("copied %d late reports%s", |
|
|
1803 count, result ? "" : " with problems"); |
|
|
1804 |
|
|
1805 return result; |
|
|
1806 } |
|
|
1807 |
|
|
1808 |
|
|
1809 |
|
|
1810 /* try to compress old report pointed to by db_sts.rcd with a predecessor */ |
|
|
1811 static void |
|
|
1812 compress_old(void) |
|
|
1813 { |
|
|
1814 DB_PTR prev, prev1; |
|
|
1815 DB_RCD_CK *new_ck, *prev_ck; |
|
|
1816 int new_ck_num, prev_ck_num; |
|
|
1817 DCC_TGTS new_tgts, prev_tgts; |
|
|
1818 DCC_CK_TYPES new_type, prev_type; |
|
|
1819 #define NEWER (db_sts.rcd.d.r) |
|
|
1820 #define OLDER (db_sts.rcd2.d.r) |
|
|
1821 |
|
|
1822 /* Before spending the time to map a preceding checksum, |
|
|
1823 * find at least one checksum worth keeping and that might |
|
|
1824 * be combined or compressed with its predecessor. */ |
|
|
1825 prev = DB_PTR_NULL; |
|
|
1826 prev_type = DCC_CK_INVALID; |
|
|
1827 for (new_ck_num = DB_NUM_CKS(NEWER), |
|
|
1828 new_ck = NEWER->cks; |
|
|
1829 new_ck_num != 0; |
|
|
1830 --new_ck_num, ++new_ck) { |
|
|
1831 if (DB_CK_OBS(new_ck)) |
|
|
1832 continue; |
|
|
1833 new_type = DB_CK_TYPE(new_ck); |
|
|
1834 if (DB_TEST_NOKEEP(db_parms.nokeep_cks, new_type)) |
|
|
1835 continue; |
|
|
1836 /* all of the checksums in this record must be old */ |
|
|
1837 if (!dcc_ts_older_ts(&NEWER->ts, &new_ex_ts[new_type].all)) |
|
|
1838 return; |
|
|
1839 /* you can compress reports only if you have >=2 */ |
|
|
1840 prev1 = DB_PTR_EX(new_ck->prev); |
|
|
1841 if (prev1 != DB_PTR_NULL) { |
|
|
1842 prev = prev1; |
|
|
1843 prev_type = new_type; |
|
|
1844 } |
|
|
1845 } |
|
|
1846 if (prev_type == DCC_CK_INVALID) |
|
|
1847 return; |
|
|
1848 |
|
|
1849 /* having picked a checksum, |
|
|
1850 * map the record containing its predecessor */ |
|
|
1851 prev_ck = db_map_rcd_ck(dcc_emsg, &db_sts.rcd2, prev, prev_type); |
|
|
1852 if (!prev_ck) |
|
|
1853 dcc_logbad(dcc_ex_code, "%s", dcc_emsg); |
|
|
1854 |
|
|
1855 /* The current and previous records must be old |
|
|
1856 * and contain the same useful checksums. */ |
|
|
1857 new_ck_num = DB_NUM_CKS(NEWER); |
|
|
1858 new_ck = NEWER->cks; |
|
|
1859 prev_ck_num = DB_NUM_CKS(OLDER); |
|
|
1860 prev_ck = OLDER->cks; |
|
|
1861 for (;;) { |
|
|
1862 /* we must run out of checksums in the two reports at the |
|
|
1863 * same time */ |
|
|
1864 if (prev_ck_num == 0 || new_ck_num == 0) { |
|
|
1865 if (prev_ck_num == new_ck_num) |
|
|
1866 break; |
|
|
1867 return; |
|
|
1868 } |
|
|
1869 |
|
|
1870 /* ignore paths and other junk */ |
|
|
1871 if (DB_CK_OBS(prev_ck)) { |
|
|
1872 --prev_ck_num; |
|
|
1873 ++prev_ck; |
|
|
1874 continue; |
|
|
1875 } |
|
|
1876 prev_type = DB_CK_TYPE(prev_ck); |
|
|
1877 if (DB_TEST_NOKEEP(db_parms.nokeep_cks, prev_type)) { |
|
|
1878 --prev_ck_num; |
|
|
1879 ++prev_ck; |
|
|
1880 continue; |
|
|
1881 } |
|
|
1882 if (DB_CK_OBS(new_ck)) { |
|
|
1883 --new_ck_num; |
|
|
1884 ++new_ck; |
|
|
1885 continue; |
|
|
1886 } |
|
|
1887 new_type = DB_CK_TYPE(new_ck); |
|
|
1888 if (DB_TEST_NOKEEP(db_parms.nokeep_cks, new_type)) { |
|
|
1889 --new_ck_num; |
|
|
1890 ++new_ck; |
|
|
1891 continue; |
|
|
1892 } |
|
|
1893 |
|
|
1894 /* because the checksums are ordered, |
|
|
1895 * give up at the first difference in checksums */ |
|
|
1896 if (new_type != prev_type |
|
|
1897 || memcmp(new_ck->sum, prev_ck->sum, sizeof(new_ck->sum))) |
|
|
1898 return; |
|
|
1899 |
|
|
1900 /* Give up at the first recent and valuable checksum. */ |
|
|
1901 if (!dcc_ts_older_ts(&OLDER->ts, &new_ex_ts[new_type].all)) |
|
|
1902 return; |
|
|
1903 |
|
|
1904 --prev_ck_num; |
|
|
1905 ++prev_ck; |
|
|
1906 --new_ck_num; |
|
|
1907 ++new_ck; |
|
|
1908 } |
|
|
1909 |
|
|
1910 /* The current and previous records are compatiable. |
|
|
1911 * Add the count of the previous record to the current record |
|
|
1912 * and mark the previous record useless. |
|
|
1913 * The individual totals in the current record are already correct, |
|
|
1914 * so postpone worrying about the deleted record. */ |
|
|
1915 new_tgts = DB_TGTS_RCD_RAW(NEWER); |
|
|
1916 if (new_tgts < DCC_TGTS_TOO_MANY) { |
|
|
1917 prev_tgts = DB_TGTS_RCD(OLDER); |
|
|
1918 if (prev_tgts > DCC_TGTS_TOO_MANY |
|
|
1919 || prev_tgts == 0) |
|
|
1920 return; |
|
|
1921 if (prev_tgts == DCC_TGTS_TOO_MANY) { |
|
|
1922 new_tgts = DCC_TGTS_TOO_MANY; |
|
|
1923 } else { |
|
|
1924 new_tgts += prev_tgts; |
|
|
1925 if (new_tgts > DCC_TGTS_TOO_MANY) |
|
|
1926 new_tgts = DCC_TGTS_TOO_MANY; |
|
|
1927 } |
|
|
1928 DB_TGTS_RCD_SET(NEWER, new_tgts); |
|
|
1929 } |
|
|
1930 |
|
|
1931 /* Mark the previous record to be deleted next time. */ |
|
|
1932 DB_TGTS_RCD_SET(OLDER, 0); |
|
|
1933 /* Mark it dirty so that the need to delete it gets to the file. */ |
|
|
1934 SET_FLUSH_RCD(&db_sts.rcd2, 1); |
|
|
1935 |
|
|
1936 NEWER->srvr_id_auth = DCC_ID_COMP; |
|
|
1937 NEWER->fgs_num_cks &= ~(DB_RCD_FG_TRIM |
|
|
1938 | DB_RCD_FG_SUMRY |
|
|
1939 | DB_RCD_FG_DELAY); |
|
|
1940 /* use the newest timestamp */ |
|
|
1941 if (dcc_ts_older_ts(&NEWER->ts, &OLDER->ts)) |
|
|
1942 NEWER->ts = OLDER->ts; |
|
|
1943 SET_FLUSH_RCD(&db_sts.rcd, 1); |
|
|
1944 |
|
|
1945 ++comp_rcds; |
|
|
1946 |
|
|
1947 #undef NEWER |
|
|
1948 #undef OLDER |
|
|
1949 } |
|
|
1950 |
|
|
1951 |
|
|
1952 |
|
|
1953 /* write a parsed whitelist checksum */ |
|
|
1954 static int |
|
|
1955 white_write(DCC_EMSG emsg, DCC_WF *wf, |
|
|
1956 DCC_CK_TYPES type, DCC_SUM sum, DCC_TGTS tgts) |
|
|
1957 { |
|
|
1958 DB_RCD rcd; |
|
|
1959 int rcd_len; |
|
|
1960 char buf[30]; |
|
|
1961 DCC_FNM_LNO_BUF fnm_buf; |
|
|
1962 |
|
|
1963 /* ignore checksums that clients are never supposed to send |
|
|
1964 * to the server or for some other reason cannot be whitelisted */ |
|
|
1965 switch (type) { |
|
|
1966 case DCC_CK_INVALID: |
|
|
1967 case DCC_CK_ENV_TO: |
|
|
1968 case DCC_CK_G_MSG_R_TOTAL: |
|
|
1969 case DCC_CK_G_TRIPLE_R_BULK: |
|
|
1970 case DCC_CK_SRVR_ID: |
|
|
1971 dcc_pemsg(EX_DATAERR, emsg, |
|
|
1972 "%s checksum cannot be used%s", |
|
|
1973 dcc_type2str_err(type, 0, 0, grey_on), |
|
|
1974 wf_fnm_lno(&fnm_buf, wf)); |
|
|
1975 return 0; |
|
|
1976 |
|
|
1977 case DCC_CK_IP: |
|
|
1978 case DCC_CK_ENV_FROM: |
|
|
1979 case DCC_CK_FROM: |
|
|
1980 case DCC_CK_MESSAGE_ID: |
|
|
1981 case DCC_CK_RECEIVED: |
|
|
1982 case DCC_CK_SUB: |
|
|
1983 case DCC_CK_BODY: |
|
|
1984 case DCC_CK_FUZ1: |
|
|
1985 case DCC_CK_FUZ2: |
|
|
1986 break; /* these are ok */ |
|
|
1987 } |
|
|
1988 |
|
|
1989 if (tgts == DCC_TGTS_OK_MX |
|
|
1990 || tgts == DCC_TGTS_OK_MXDCC |
|
|
1991 || tgts == DCC_TGTS_SUBMIT_CLIENT) { |
|
|
1992 dcc_pemsg(EX_DATAERR, emsg,"\"%s\" ignored%s", |
|
|
1993 dcc_tgts2str(buf, sizeof(buf), tgts, 0), |
|
|
1994 wf_fnm_lno(&fnm_buf, wf)); |
|
|
1995 return 0; |
|
|
1996 } |
|
|
1997 |
|
|
1998 /* Greylist whitelist entries cannot involve blacklisting. |
|
|
1999 * They use DCC_TGTS_GREY_WHITE to signal whitelisting */ |
|
|
2000 if (grey_on) { |
|
|
2001 /* ignore anything except whitelisting */ |
|
|
2002 if (tgts != DCC_TGTS_OK) { |
|
|
2003 dcc_pemsg(EX_DATAERR, emsg, "\"%s\" ignored%s", |
|
|
2004 dcc_tgts2str(buf, sizeof(buf), tgts, 0), |
|
|
2005 wf_fnm_lno(&fnm_buf, wf)); |
|
|
2006 return 0; |
|
|
2007 } |
|
|
2008 tgts = DCC_TGTS_GREY_WHITE; |
|
|
2009 } |
|
|
2010 |
|
|
2011 memset(&rcd, 0, sizeof(rcd)); |
|
|
2012 dcc_timeval2ts(&rcd.ts, &clean_start, 0); |
|
|
2013 rcd.srvr_id_auth = DCC_ID_WHITE; |
|
|
2014 DB_TGTS_RCD_SET(&rcd, tgts); |
|
|
2015 |
|
|
2016 rcd.cks[0].type_fgs = DCC_CK_FLOD_PATH; |
|
|
2017 memcpy(rcd.cks[0].sum, &wf->lno, sizeof(wf->lno)); |
|
|
2018 rcd.cks[0].sum[sizeof(wf->lno)] = wf->fno; |
|
|
2019 |
|
|
2020 rcd.cks[1].type_fgs = type; |
|
|
2021 memcpy(rcd.cks[1].sum, sum, sizeof(rcd.cks[1])); |
|
|
2022 |
|
|
2023 rcd_len = sizeof(rcd) - sizeof(rcd.cks) + 2*sizeof(rcd.cks[0]); |
|
|
2024 rcd.fgs_num_cks = 2; |
|
|
2025 |
|
|
2026 if (!write_new_rcd(&rcd, rcd_len)) |
|
|
2027 return -1; |
|
|
2028 |
|
|
2029 ++white_cks; |
|
|
2030 return 1; |
|
|
2031 } |
|
|
2032 |
|
|
2033 |
|
|
2034 |
|
|
2035 /* Add the whitelist of certified non-spam and non-spammers |
|
|
2036 * and otherwise start the database */ |
|
|
2037 static void |
|
|
2038 parse_white(void) |
|
|
2039 { |
|
|
2040 int white_fd; |
|
|
2041 |
|
|
2042 white_cks = 0; |
|
|
2043 |
|
|
2044 if (!keep_white) { |
|
|
2045 memset(&dbclean_white_tbl, 0,sizeof(dbclean_white_tbl)); |
|
|
2046 dcc_wf_init(&dbclean_wf, 0); |
|
|
2047 fnm2rel_good(dbclean_wf.ascii_nm, WHITELIST_NM(grey_on), 0); |
|
|
2048 dbclean_wf.wtbl = &dbclean_white_tbl; |
|
|
2049 white_fd = open(dbclean_wf.ascii_nm, O_RDONLY, 0); |
|
|
2050 if (white_fd < 0) { |
|
|
2051 /* worry only if the file exists but can't be used */ |
|
|
2052 if (errno != ENOENT) { |
|
|
2053 dcc_error_msg("open(%s): %s", |
|
|
2054 dbclean_wf.ascii_nm, ERROR_STR()); |
|
|
2055 keep_white = 1; |
|
|
2056 } |
|
|
2057 } else { |
|
|
2058 if (0 > dcc_parse_whitefile(0, &dbclean_wf, white_fd, |
|
|
2059 white_write, 0)) |
|
|
2060 keep_white = 1; |
|
|
2061 if (0 > close(white_fd)) |
|
|
2062 dcc_error_msg("close(%s): %s", |
|
|
2063 dbclean_wf.ascii_nm, ERROR_STR()); |
|
|
2064 } |
|
|
2065 } |
|
|
2066 if (keep_white) { |
|
|
2067 /* If the whitelist was bad, purge the new database of |
|
|
2068 * the bad new whitelist. We will use the existing |
|
|
2069 * whitelist */ |
|
|
2070 write_new_flush(1); |
|
|
2071 new_db_csize = DB_PTR_BASE; |
|
|
2072 if (0 > ftruncate(new_db_fd, DB_PTR_BASE)) |
|
|
2073 dcc_logbad(EX_IOERR, "truncate(%s, %d): %s", |
|
|
2074 new_db_nm, DB_PTR_BASE, ERROR_STR()); |
|
|
2075 new_db_fsize = DB_PTR_BASE; |
|
|
2076 white_cks = 0; |
|
|
2077 } |
|
|
2078 |
|
|
2079 /* update the counts in the database file */ |
|
|
2080 write_new_hdr(1); |
|
|
2081 } |
|
|
2082 |
|
|
2083 |
|
|
2084 |
|
|
2085 /* check for conflicts in the whitelist file in the record pointed to |
|
|
2086 * by db_sts.rcd */ |
|
|
2087 static void |
|
|
2088 check_white(void) |
|
|
2089 { |
|
|
2090 static int msgs; |
|
|
2091 static int prev_lno1, prev_lno2; |
|
|
2092 static int prev_fno1, prev_fno2; |
|
|
2093 const DB_RCD_CK *rcd_ck, *prev_ck; |
|
|
2094 int lno1, lno2; |
|
|
2095 int fno1, fno2; |
|
|
2096 DCC_TGTS tgts1, tgts2; |
|
|
2097 char tgts1_buf[30], tgts2_buf[30]; |
|
|
2098 const char *fname1, *fname2; |
|
|
2099 DCC_CK_TYPES type; |
|
|
2100 DB_PTR prev; |
|
|
2101 |
|
|
2102 /* don't check if we have already complained enough */ |
|
|
2103 if (msgs > 20) |
|
|
2104 return; |
|
|
2105 |
|
|
2106 rcd_ck = db_sts.rcd.d.r->cks; |
|
|
2107 |
|
|
2108 /* it is pointless without line numbers, which are lacking only |
|
|
2109 * if we saved the old whitelist entries because the file is |
|
|
2110 * broken */ |
|
|
2111 if (DB_NUM_CKS(db_sts.rcd.d.r) != 2 |
|
|
2112 || DB_CK_TYPE(rcd_ck) != DCC_CK_FLOD_PATH) |
|
|
2113 return; |
|
|
2114 |
|
|
2115 /* conflict is impossible with a single line */ |
|
|
2116 ++rcd_ck; |
|
|
2117 prev = DB_PTR_EX(rcd_ck->prev); |
|
|
2118 if (prev == DB_PTR_NULL) |
|
|
2119 return; |
|
|
2120 |
|
|
2121 type = DB_CK_TYPE(rcd_ck); |
|
|
2122 prev_ck = db_map_rcd_ck(dcc_emsg, &db_sts.rcd2, prev, type); |
|
|
2123 if (!prev_ck) |
|
|
2124 dcc_logbad(dcc_ex_code, "%s", dcc_emsg); |
|
|
2125 |
|
|
2126 tgts1 = DB_TGTS_RCD(db_sts.rcd2.d.r); |
|
|
2127 tgts2 = DB_TGTS_RCD(db_sts.rcd.d.r); |
|
|
2128 if (tgts1 == tgts2) |
|
|
2129 return; /* no conflict */ |
|
|
2130 |
|
|
2131 memcpy(&lno1, db_sts.rcd2.d.r->cks[0].sum, sizeof(lno1)); |
|
|
2132 fno1 = db_sts.rcd2.d.r->cks[0].sum[sizeof(lno1)]; |
|
|
2133 memcpy(&lno2, db_sts.rcd.d.r->cks[0].sum, sizeof(lno2)); |
|
|
2134 fno2 = db_sts.rcd.d.r->cks[0].sum[sizeof(lno2)]; |
|
|
2135 |
|
|
2136 if (lno1 == prev_lno1 && fno1 == prev_fno1 |
|
|
2137 && lno2 == prev_lno2 && fno2 == prev_fno2) |
|
|
2138 return; |
|
|
2139 |
|
|
2140 fname1 = wf_fnm(&dbclean_wf, fno1); |
|
|
2141 fname2 = wf_fnm(&dbclean_wf, fno2); |
|
|
2142 if (fname1 == fname2) { |
|
|
2143 fname1 = ""; |
|
|
2144 } else { |
|
|
2145 fname1 = path2fnm(fname1); |
|
|
2146 } |
|
|
2147 dcc_error_msg("\"%s\" in line %d%s%s conflicts with \"%s\"" |
|
|
2148 " in line %d of %s", |
|
|
2149 dcc_tgts2str(tgts1_buf, sizeof(tgts1_buf), |
|
|
2150 tgts1, grey_on), |
|
|
2151 lno1, |
|
|
2152 *fname1 != '\0' ? " of " : "", fname1, |
|
|
2153 dcc_tgts2str(tgts2_buf, sizeof(tgts2_buf), |
|
|
2154 tgts2, grey_on), |
|
|
2155 lno2, |
|
|
2156 fname2); |
|
|
2157 ++msgs; |
|
|
2158 prev_lno1 = lno1; |
|
|
2159 prev_fno1 = fno1; |
|
|
2160 prev_lno2 = lno2; |
|
|
2161 prev_fno2 = fno2; |
|
|
2162 } |
|
|
2163 |
|
|
2164 |
|
|
2165 |
|
|
2166 /* rebuild the hash table and the totals and links within the database file |
|
|
2167 * finish with the file locked */ |
|
|
2168 static void |
|
|
2169 build_hash(void) |
|
|
2170 { |
|
|
2171 DB_PTR rcd_pos; |
|
|
2172 DB_HADDR haddr_window, haddr_lo, haddr_hi; |
|
|
2173 int pass, total_passes; |
|
|
2174 int rcd_len; |
|
|
2175 int rcd_cks, rcd_sums; |
|
|
2176 DB_PTR rcds, sums; /* passes can inflate these */ |
|
|
2177 const DB_RCD_CK *rcd_ck; |
|
|
2178 DB_HADDR guess_hash_len; |
|
|
2179 double db_rate, hash_ratio; |
|
|
2180 struct timeval db_flushed; |
|
|
2181 |
|
|
2182 db_buf_init(new_db_pagesize, 0); |
|
|
2183 |
|
|
2184 if (new_hash_len == 0) { |
|
|
2185 /* Try to choose a hash table size now so that when it |
|
|
2186 * is next time to rebuild after 24 hours of incoming |
|
|
2187 * checksums, the alpha or load factor will still be 0.9. |
|
|
2188 * We probably ran 24 hours ago, so the old hash size |
|
|
2189 * is an estimate of the size tomorrow. */ |
|
|
2190 |
|
|
2191 /* Guess the number of distinct checksums added |
|
|
2192 * tomorrow based on the current average rate */ |
|
|
2193 db_rate = db_add_rate(&new_db_parms, 1); |
|
|
2194 if (db_rate > 0.0) { |
|
|
2195 /* Increase the average rate by 10% to account |
|
|
2196 * for the 30% decrease often seen on weekends. */ |
|
|
2197 guess_hash_len = db_rate * 1.1 * 24*60*60; |
|
|
2198 |
|
|
2199 /* predict # of distinct checksums in current data */ |
|
|
2200 hash_ratio = old_db_parms.old_kept_cks; |
|
|
2201 if (hash_ratio == 0.0) { |
|
|
2202 hash_ratio = 1.0; |
|
|
2203 } else { |
|
|
2204 hash_ratio = (HADDR2LEN(old_db_parms |
|
|
2205 .old_hash_used) |
|
|
2206 / hash_ratio); |
|
|
2207 if (hash_ratio > 1.0 || hash_ratio < 0.3) |
|
|
2208 hash_ratio = 1.0; |
|
|
2209 } |
|
|
2210 guess_hash_len += (kept_cks * hash_ratio) + white_cks; |
|
|
2211 |
|
|
2212 if (db_debug) |
|
|
2213 quiet_trace_msg("hash size from old=%d" |
|
|
2214 " %d from db_rate=%.1f" |
|
|
2215 " hash_ratio=%.1f=%d/%d" |
|
|
2216 " kept=%d white=%d", |
|
|
2217 old_db_hash_used, |
|
|
2218 guess_hash_len, |
|
|
2219 db_rate, hash_ratio, |
|
|
2220 HADDR2LEN(old_db_parms |
|
|
2221 .old_hash_used), |
|
|
2222 old_db_parms.old_kept_cks, |
|
|
2223 kept_cks, white_cks); |
|
|
2224 |
|
|
2225 } else { |
|
|
2226 /* guess if we do not have a good measure |
|
|
2227 * of the recent rate */ |
|
|
2228 guess_hash_len = kept_cks+white_cks; |
|
|
2229 guess_hash_len += guess_hash_len/5; |
|
|
2230 } |
|
|
2231 |
|
|
2232 new_hash_len = old_db_hash_used; |
|
|
2233 if (new_hash_len < guess_hash_len) |
|
|
2234 new_hash_len = guess_hash_len; |
|
|
2235 |
|
|
2236 /* go for load factor 0.9 */ |
|
|
2237 new_hash_len += new_hash_len/10; |
|
|
2238 |
|
|
2239 if (new_hash_len > db_max_hash_entries) |
|
|
2240 quiet_trace_msg("default hash size %d entries" |
|
|
2241 " > maximum %d", |
|
|
2242 new_hash_len, db_max_hash_entries); |
|
|
2243 |
|
|
2244 if (grey_on) { |
|
|
2245 if (new_hash_len < MIN_HASH_ENTRIES) |
|
|
2246 new_hash_len = MIN_HASH_ENTRIES; |
|
|
2247 } else { |
|
|
2248 if (new_hash_len < DEF_HASH_ENTRIES) |
|
|
2249 new_hash_len = DEF_HASH_ENTRIES; |
|
|
2250 } |
|
|
2251 } |
|
|
2252 |
|
|
2253 /* Open and lock the new database */ |
|
|
2254 unlink_whine(new_hash_nm, 1); |
|
|
2255 new_hash_created = 1; |
|
|
2256 if (!db_open(0, -1, new_db_nm, new_hash_len, |
|
|
2257 DB_OPEN_LOCK_NOWAIT | db_mode)) { |
|
|
2258 dcc_logbad(dcc_ex_code, "could not start database %s", |
|
|
2259 new_db_nm); |
|
|
2260 } |
|
|
2261 if (db_debug) |
|
|
2262 quiet_trace_msg("%s %s", db_window_size_str, new_db_nm); |
|
|
2263 |
|
|
2264 /* guess which checksums we will keep so that we can count them */ |
|
|
2265 if (old_db_parms.nokeep_cks != 0) |
|
|
2266 db_parms.nokeep_cks = old_db_parms.nokeep_cks; |
|
|
2267 |
|
|
2268 /* add every record in the database file to the hash table and |
|
|
2269 * fix its accumulated counts and reverse links */ |
|
|
2270 comp_rcds = 0; |
|
|
2271 sums = 0; |
|
|
2272 rcds = 0; |
|
|
2273 report_progress_init(); |
|
|
2274 db_flushed = db_time; |
|
|
2275 |
|
|
2276 /* if the hash table does not fit in 75% of RAM, |
|
|
2277 * then make several passes over the data with as much of the |
|
|
2278 * hash table as fits. */ |
|
|
2279 haddr_window = db_hash_page_len*((db_buf_total*3)/4); |
|
|
2280 if (haddr_window < db_hash_len/16) |
|
|
2281 haddr_window = db_hash_len/16; |
|
|
2282 total_passes = (db_hash_len+haddr_window-1)/haddr_window; |
|
|
2283 |
|
|
2284 for (haddr_lo = 0, pass = 1; |
|
|
2285 haddr_lo < db_hash_len; |
|
|
2286 haddr_lo = haddr_hi, ++pass) { |
|
|
2287 if (haddr_lo > db_hash_len-haddr_window) |
|
|
2288 haddr_hi = MAX_HASH_ENTRIES; |
|
|
2289 else |
|
|
2290 haddr_hi = haddr_lo+haddr_window; |
|
|
2291 for (rcd_pos = DB_PTR_BASE; |
|
|
2292 rcd_pos < db_csize; |
|
|
2293 rcd_pos += rcd_len) { |
|
|
2294 /* skip reports crossing page bounardies */ |
|
|
2295 if (rcd_pos%db_pagesize > db_page_max) { |
|
|
2296 rcd_len = DB_RCD_HDR_LEN; |
|
|
2297 continue; |
|
|
2298 } |
|
|
2299 if (--progress_rpt_cnt <= 0) { |
|
|
2300 report_progress(0, " hash rebuilt", |
|
|
2301 "checksums", |
|
|
2302 sums/total_passes, kept_cks, 1); |
|
|
2303 if (db_time.tv_sec != db_flushed.tv_sec) { |
|
|
2304 db_flushed = db_time; |
|
|
2305 if (!db_flush_db(dcc_emsg)) |
|
|
2306 dcc_logbad(dcc_ex_code, |
|
|
2307 "flushing after linking" |
|
|
2308 L_HPAT": %s", |
|
|
2309 rcd_pos, dcc_emsg); |
|
|
2310 } |
|
|
2311 } |
|
|
2312 |
|
|
2313 if (!db_map_rcd(0, &db_sts.rcd, rcd_pos, &rcd_len)) { |
|
|
2314 dcc_logbad(dcc_ex_code, |
|
|
2315 "hash build failed reading" |
|
|
2316 " record at "L_HPAT, |
|
|
2317 rcd_pos); |
|
|
2318 } |
|
|
2319 |
|
|
2320 /* skip end of page padding */ |
|
|
2321 if (db_sts.rcd.d.r->fgs_num_cks == 0) |
|
|
2322 continue; |
|
|
2323 |
|
|
2324 ++rcds; |
|
|
2325 |
|
|
2326 /* count the checksums we'll link in this record */ |
|
|
2327 rcd_cks = DB_NUM_CKS(db_sts.rcd.d.r); |
|
|
2328 rcd_sums = 0; |
|
|
2329 for (rcd_ck = db_sts.rcd.d.r->cks; |
|
|
2330 rcd_ck < &db_sts.rcd.d.r->cks[rcd_cks]; |
|
|
2331 ++rcd_ck) { |
|
|
2332 if (!DB_TEST_NOKEEP(db_parms.nokeep_cks, |
|
|
2333 DB_CK_TYPE(rcd_ck))) |
|
|
2334 ++rcd_sums; |
|
|
2335 } |
|
|
2336 sums += rcd_sums; |
|
|
2337 |
|
|
2338 /* Mark the record dirty so that any new hash links |
|
|
2339 * get to the file if we are using -F. */ |
|
|
2340 db_set_flush(&db_sts.rcd, 0, rcd_len); |
|
|
2341 if (!db_link_rcd(dcc_emsg, haddr_lo, haddr_hi)) { |
|
|
2342 dcc_logbad(dcc_ex_code, |
|
|
2343 "relinking record at "L_HPAT": %s", |
|
|
2344 rcd_pos, dcc_emsg); |
|
|
2345 } |
|
|
2346 |
|
|
2347 /* check for conflicts in the whitelist file */ |
|
|
2348 if (DB_RCD_ID(db_sts.rcd.d.r) == DCC_ID_WHITE) |
|
|
2349 check_white(); |
|
|
2350 |
|
|
2351 compress_old(); |
|
|
2352 } |
|
|
2353 |
|
|
2354 if (progress_rpt_started && pass < total_passes) |
|
|
2355 quiet_trace_msg(" pass %d", pass); |
|
|
2356 } |
|
|
2357 |
|
|
2358 report_progress(1, " hash rebuilt", "checksums", |
|
|
2359 sums/total_passes, kept_cks, 1); |
|
|
2360 |
|
|
2361 db_parms.old_hash_used = db_hash_used; |
|
|
2362 db_parms.old_kept_cks = kept_cks; |
|
|
2363 db_parms.hash_used = db_hash_used; |
|
|
2364 db_parms.old_db_csize = db_csize; |
|
|
2365 if (!db_flush_parms(dcc_emsg)) |
|
|
2366 dcc_logbad(dcc_ex_code, "%s", dcc_emsg); |
|
|
2367 |
|
|
2368 quiet_trace_msg("hashed "L_DPAT" records containing "L_DPAT" checksums," |
|
|
2369 " compressed %d records", |
|
|
2370 rcds/total_passes, sums/total_passes, comp_rcds); |
|
|
2371 |
|
|
2372 /* Try to finish as much disk I/O on the new file as we can to minimize |
|
|
2373 * stalling by dccd when we close the file and hand it over. This also |
|
|
2374 * reduces system stalling hours later when dbclean runs again. */ |
|
|
2375 if (!make_clean(1)) |
|
|
2376 dcc_logbad(dcc_ex_code, "%s", dcc_emsg); |
|
|
2377 |
|
|
2378 |
|
|
2379 quiet_trace_msg("%d hash entries total, %d or %d%% used", |
|
|
2380 HADDR2LEN(db_hash_len), |
|
|
2381 HADDR2LEN(db_hash_used), |
|
|
2382 (int)((HADDR2LEN(db_hash_used)*100.0) |
|
|
2383 / HADDR2LEN(db_hash_len))); |
|
|
2384 } |
|
|
2385 |
|
|
2386 |
|
|
2387 |
|
|
2388 static u_char |
|
|
2389 write_new_db(const void *buf, int buflen, off_t pos, u_char fatal) |
|
|
2390 { |
|
|
2391 int i; |
|
|
2392 |
|
|
2393 if (pos != lseek(new_db_fd, pos, SEEK_SET)) { |
|
|
2394 if (fatal) { |
|
|
2395 dcc_logbad(EX_IOERR, "lseek(%s, 0): %s", |
|
|
2396 new_db_nm, ERROR_STR()); |
|
|
2397 } else { |
|
|
2398 dcc_error_msg("lseek(%s, 0): %s", |
|
|
2399 new_db_nm, ERROR_STR()); |
|
|
2400 } |
|
|
2401 return 0; |
|
|
2402 } |
|
|
2403 |
|
|
2404 i = write(new_db_fd, buf, buflen); |
|
|
2405 if (i == buflen) { |
|
|
2406 if (new_db_fsize < pos+buflen) |
|
|
2407 new_db_fsize = pos+buflen; |
|
|
2408 return 1; |
|
|
2409 } |
|
|
2410 |
|
|
2411 if (fatal) { |
|
|
2412 if (i < 0) |
|
|
2413 dcc_logbad(EX_IOERR, "write(%s): %s", |
|
|
2414 new_db_nm, ERROR_STR()); |
|
|
2415 else |
|
|
2416 dcc_logbad(EX_IOERR, "write(%s)=%d instead of %d", |
|
|
2417 new_db_nm, i, buflen); |
|
|
2418 } else { |
|
|
2419 if (i < 0) |
|
|
2420 dcc_error_msg("write(%s): %s", |
|
|
2421 new_db_nm, ERROR_STR()); |
|
|
2422 else |
|
|
2423 dcc_error_msg("write(%s)=%d instead of %d", |
|
|
2424 new_db_nm, i, buflen); |
|
|
2425 } |
|
|
2426 return 0; |
|
|
2427 } |
|
|
2428 |
|
|
2429 |
|
|
2430 |
|
|
2431 /* use a large buffer to encourage the file system to avoid fragmentation */ |
|
|
2432 static union { |
|
|
2433 u_char c[DB_MIN_MIN_MBYTE*(1024*1024)/4]; |
|
|
2434 DB_HDR hdr; |
|
|
2435 } write_new_db_buf; |
|
|
2436 static u_int write_new_db_buflen = 0; |
|
|
2437 static DB_PTR write_new_base; |
|
|
2438 |
|
|
2439 static u_char |
|
|
2440 write_new_flush(u_char fatal) |
|
|
2441 { |
|
|
2442 u_char result = 1; |
|
|
2443 |
|
|
2444 if (write_new_db_buflen != 0) { |
|
|
2445 if (!write_new_db(&write_new_db_buf, write_new_db_buflen, |
|
|
2446 write_new_base, fatal)) |
|
|
2447 result = 0; |
|
|
2448 } |
|
|
2449 |
|
|
2450 write_new_base = new_db_csize; |
|
|
2451 write_new_db_buflen = 0; |
|
|
2452 return result; |
|
|
2453 } |
|
|
2454 |
|
|
2455 |
|
|
2456 static u_char |
|
|
2457 write_new_buf(const void *buf, int buflen) |
|
|
2458 { |
|
|
2459 if (write_new_db_buflen + buflen > ISZ(write_new_db_buf) |
|
|
2460 && !write_new_flush(0)) |
|
|
2461 return 0; |
|
|
2462 |
|
|
2463 memcpy(&write_new_db_buf.c[write_new_db_buflen], buf, buflen); |
|
|
2464 write_new_db_buflen += buflen; |
|
|
2465 return 1; |
|
|
2466 } |
|
|
2467 |
|
|
2468 |
|
|
2469 |
|
|
2470 /* add a record to the new file */ |
|
|
2471 static u_char |
|
|
2472 write_new_rcd(const void *buf, int buflen) |
|
|
2473 { |
|
|
2474 static const u_char zeros[DB_RCD_LEN_MAX] = {0}; |
|
|
2475 DB_PTR new_page_num; |
|
|
2476 u_char result; |
|
|
2477 int pad, i; |
|
|
2478 |
|
|
2479 /* pad accross page boundaries */ |
|
|
2480 new_page_num = DB_PTR2PG_NUM(new_db_csize + buflen, new_db_pagesize); |
|
|
2481 if (new_page_num != DB_PTR2PG_NUM(new_db_csize, new_db_pagesize)) { |
|
|
2482 pad = new_page_num*new_db_pagesize - new_db_csize; |
|
|
2483 pad = (((pad + DB_RCD_HDR_LEN-1) / DB_RCD_HDR_LEN) |
|
|
2484 * DB_RCD_HDR_LEN); |
|
|
2485 do { |
|
|
2486 i = sizeof(zeros); |
|
|
2487 if (i > pad) |
|
|
2488 i = pad; |
|
|
2489 if (!write_new_buf(zeros, i)) |
|
|
2490 return 0; |
|
|
2491 pad -= i; |
|
|
2492 new_db_csize += i; |
|
|
2493 } while (pad != 0); |
|
|
2494 } |
|
|
2495 |
|
|
2496 result = write_new_buf(buf, buflen); |
|
|
2497 new_db_csize += buflen; |
|
|
2498 return result; |
|
|
2499 } |
|
|
2500 |
|
|
2501 |
|
|
2502 |
|
|
2503 /* write the magic string at the head of the database file */ |
|
|
2504 static void |
|
|
2505 write_new_hdr(u_char emptied) |
|
|
2506 { |
|
|
2507 DB_HDR *new; |
|
|
2508 struct timeval old_sn; |
|
|
2509 time_t new_rate_secs; |
|
|
2510 DCC_CK_TYPES type; |
|
|
2511 int i; |
|
|
2512 |
|
|
2513 write_new_flush(1); |
|
|
2514 |
|
|
2515 memset(&write_new_db_buf, 0, sizeof(write_new_db_buf)); |
|
|
2516 write_new_base = 0; |
|
|
2517 if (new_db_fsize > ISZ(DB_HDR) |
|
|
2518 || new_db_pagesize == 0) { |
|
|
2519 write_new_db_buflen = sizeof(DB_HDR); |
|
|
2520 } else { |
|
|
2521 write_new_db_buflen = new_db_pagesize; |
|
|
2522 if (write_new_db_buflen > ISZ(write_new_db_buf)) |
|
|
2523 write_new_db_buflen = ISZ(write_new_db_buf); |
|
|
2524 } |
|
|
2525 |
|
|
2526 new = &write_new_db_buf.hdr; |
|
|
2527 memset(new, 0, sizeof(*new)); |
|
|
2528 memcpy(new->p.version, db_version_buf, sizeof(new->p.version)); |
|
|
2529 |
|
|
2530 dcc_timeval2ts(&new->p.sn, &clean_start, 0); |
|
|
2531 if (emptied) { |
|
|
2532 new->p.cleared = clean_start.tv_sec; |
|
|
2533 } else { |
|
|
2534 new->p.cleared = old_db_parms.cleared; |
|
|
2535 switch (clean_mode) { |
|
|
2536 case NORMAL_MODE: |
|
|
2537 new->p.cleaned = clean_start.tv_sec; |
|
|
2538 new->p.cleaned_cron = clean_start.tv_sec; |
|
|
2539 break; |
|
|
2540 case NO_CRON_MODE: |
|
|
2541 new->p.cleaned = clean_start.tv_sec; |
|
|
2542 new->p.cleaned_cron = old_db_parms.cleaned_cron; |
|
|
2543 break; |
|
|
2544 case REPAIR_MODE: |
|
|
2545 case QUICK_MODE: |
|
|
2546 case HASH_MODE: |
|
|
2547 case DEL_MODE: |
|
|
2548 new->p.cleaned = old_db_parms.cleaned; |
|
|
2549 new->p.cleaned_cron = old_db_parms.cleaned_cron; |
|
|
2550 break; |
|
|
2551 } |
|
|
2552 } |
|
|
2553 |
|
|
2554 if (grey_on) |
|
|
2555 new->p.flags |= DB_PARM_FG_GREY; |
|
|
2556 if (emptied || (old_db_parms.flags & DB_PARM_FG_CLEARED)) |
|
|
2557 new->p.flags |= DB_PARM_FG_CLEARED; |
|
|
2558 if (have_expire_parms > 0 |
|
|
2559 || (have_expire_parms < 0 |
|
|
2560 && (old_db_parms.flags & DB_PARM_EXP_SET))) |
|
|
2561 new->p.flags |= DB_PARM_EXP_SET; |
|
|
2562 |
|
|
2563 new->p.nokeep_cks = (emptied || old_db_parms.nokeep_cks == 0 |
|
|
2564 ? def_nokeep_cks() |
|
|
2565 : old_db_parms.nokeep_cks); |
|
|
2566 |
|
|
2567 new->p.pagesize = new_db_pagesize; |
|
|
2568 new->p.db_csize = new_db_csize; |
|
|
2569 |
|
|
2570 /* update the traffic counts */ |
|
|
2571 if (!emptied |
|
|
2572 && old_db_parms.db_csize != 0 |
|
|
2573 && old_db_parms.db_csize >= old_db_parms.old_db_csize |
|
|
2574 && old_db_parms.hash_used != 0 |
|
|
2575 && old_db_parms.hash_used >= old_db_parms.old_hash_used) { |
|
|
2576 if (old_db_parms.rate_secs > 0 |
|
|
2577 && old_db_parms.rate_secs <= DB_MAX_RATE_SECS) { |
|
|
2578 new->p.rate_secs = old_db_parms.rate_secs; |
|
|
2579 new->p.db_added = old_db_parms.db_added; |
|
|
2580 new->p.hash_added = old_db_parms.hash_added; |
|
|
2581 } |
|
|
2582 new->p.last_rate_sec = clean_start.tv_sec; |
|
|
2583 dcc_ts2timeval(&old_sn, &old_db_parms.sn); |
|
|
2584 new_rate_secs = clean_start.tv_sec - old_sn.tv_sec; |
|
|
2585 if (new_rate_secs > 0 && new_rate_secs <= DB_MAX_RATE_SECS) { |
|
|
2586 new_rate_secs += new->p.rate_secs; |
|
|
2587 if (new_rate_secs > DB_MAX_RATE_SECS) { |
|
|
2588 double trim, new_val; |
|
|
2589 trim = DB_MAX_RATE_SECS; |
|
|
2590 trim /= new_rate_secs; |
|
|
2591 |
|
|
2592 new_val = new->p.db_added; |
|
|
2593 new_val *= trim; |
|
|
2594 new->p.db_added = new_val; |
|
|
2595 |
|
|
2596 new_val = new->p.hash_added; |
|
|
2597 new_val *= trim; |
|
|
2598 new->p.hash_added = new_val; |
|
|
2599 |
|
|
2600 new_rate_secs = DB_MAX_RATE_SECS; |
|
|
2601 } |
|
|
2602 new->p.db_added += (old_db_parms.db_csize |
|
|
2603 - old_db_parms.old_db_csize); |
|
|
2604 new->p.hash_added += (old_db_parms.hash_used |
|
|
2605 - old_db_parms.old_hash_used); |
|
|
2606 new->p.rate_secs = new_rate_secs; |
|
|
2607 } |
|
|
2608 } |
|
|
2609 |
|
|
2610 for (type = DCC_CK_TYPE_FIRST; type <= DCC_CK_TYPE_LAST; ++type) { |
|
|
2611 if (new_ex_secs[type].all != 0) { |
|
|
2612 new->p.ex_secs[type].all = new_ex_secs[type].all; |
|
|
2613 new->p.ex_secs[type].spam = new_ex_secs[type].spam; |
|
|
2614 new->p.ex_all[type] = new_ex_ts[type].all; |
|
|
2615 new->p.ex_spam[type] = new_ex_ts[type].spam; |
|
|
2616 } else { |
|
|
2617 new->p.ex_secs[type].all = def_expire_secs; |
|
|
2618 new->p.ex_secs[type].spam = (DCC_CK_LONG_TERM(type) |
|
|
2619 ? def_expire_spamsecs |
|
|
2620 : def_expire_secs); |
|
|
2621 } |
|
|
2622 } |
|
|
2623 |
|
|
2624 new_db_parms = new->p; |
|
|
2625 |
|
|
2626 for (;;) { |
|
|
2627 write_new_flush(1); |
|
|
2628 |
|
|
2629 /* ensure that the last page of the file is complete */ |
|
|
2630 if (new_db_pagesize == 0) |
|
|
2631 break; |
|
|
2632 i = new_db_fsize % new_db_pagesize; |
|
|
2633 if (i == 0) |
|
|
2634 break; |
|
|
2635 write_new_db_buflen = new_db_pagesize - i; |
|
|
2636 if (write_new_db_buflen > ISZ(write_new_db_buf)) |
|
|
2637 write_new_db_buflen = ISZ(write_new_db_buf); |
|
|
2638 memset(&write_new_db_buf, 0, write_new_db_buflen); |
|
|
2639 write_new_base = new_db_fsize; |
|
|
2640 } |
|
|
2641 } |
|
|
2642 |
|
|
2643 |
|
|
2644 |
|
|
2645 static void |
|
|
2646 unlink_whine(const char *nm, u_char enoent_ok) |
|
|
2647 { |
|
|
2648 if (0 > unlink(nm) |
|
|
2649 && (!enoent_ok || errno != ENOENT)) |
|
|
2650 dcc_error_msg("unlink(%s): %s", nm, ERROR_STR()); |
|
|
2651 } |
|
|
2652 |
|
|
2653 |
|
|
2654 |
|
|
2655 static void |
|
|
2656 rename_bail(const char *from, const char *to) |
|
|
2657 { |
|
|
2658 if (0 > rename(from, to)) |
|
|
2659 dcc_logbad(EX_IOERR, "rename(%s, %s): %s", |
|
|
2660 from, to, ERROR_STR()); |
|
|
2661 } |
|
|
2662 |
|
|
2663 |
|
|
2664 |
|
|
2665 /* try for a long time or until the server hears */ |
|
|
2666 static u_char /* 1=ok, 0=failed */ |
|
|
2667 persist_aop(DCC_AOPS aop, u_int32_t val1, |
|
|
2668 int secs) /* try for this long */ |
|
|
2669 { |
|
|
2670 return dcc_aop_persist(dcc_emsg, ctxt, |
|
|
2671 grey_on ? DCC_CLNT_FG_GREY : 0, |
|
|
2672 db_debug != 0, |
|
|
2673 aop, val1, secs, &aop_resp); |
|
|
2674 } |
|
|
2675 |
|
|
2676 |
|
|
2677 |
|
|
2678 /* tell the daemon to switch to the new database */ |
|
|
2679 static void |
|
|
2680 dccd_new_db(const char *msg) |
|
|
2681 { |
|
|
2682 /* Send a round of NOPs and ask about status to ensure the server |
|
|
2683 * has dealt with requests that arrived while we had the database |
|
|
2684 * locked and otherwise caught up. We want to try to ensure that |
|
|
2685 * the server is listening when we re-open the database so that |
|
|
2686 * it does not leave flooding off. |
|
|
2687 * On some systems with lame mmap() support including BSD/OS, the |
|
|
2688 * the daemon can stall for minutes in close(). If that or something |
|
|
2689 * else makes the daemon stall, this can appear to fail. */ |
|
|
2690 if (!persist_aop(DCC_AOP_FLOD, DCC_AOP_FLOD_LIST, RESTART_DELAY)) |
|
|
2691 dcc_error_msg("%s: %s; continuing", msg, dcc_emsg); |
|
|
2692 |
|
|
2693 dccd_unlocked = 0; |
|
|
2694 if (!persist_aop(DCC_AOP_DB_NEW, 0, RESTART_DELAY)) { |
|
|
2695 /* This cannot be a fatal error, |
|
|
2696 * lest we leave the database broken */ |
|
|
2697 dcc_error_msg("%s: %s; continuing", msg, dcc_emsg); |
|
|
2698 } |
|
|
2699 } |
|
|
2700 |
|
|
2701 |
|
|
2702 |
|
|
2703 static void |
|
|
2704 finish(void) |
|
|
2705 { |
|
|
2706 int bailing = 0; |
|
|
2707 |
|
|
2708 /* delete the new files */ |
|
|
2709 #ifndef DCC_DBCLEAN_KEEP_NEW /* for debugging */ |
|
|
2710 if (new_db_created) { |
|
|
2711 unlink_whine(new_db_nm, 0); |
|
|
2712 new_db_created = 0; |
|
|
2713 bailing = -1; |
|
|
2714 } |
|
|
2715 /* we don't really know if the new hash file was created, |
|
|
2716 * so don't worry about problems */ |
|
|
2717 if (new_hash_created) { |
|
|
2718 unlink_whine(new_hash_nm, 1); |
|
|
2719 new_hash_created = 0; |
|
|
2720 bailing = -1; |
|
|
2721 } |
|
|
2722 #endif |
|
|
2723 if (cur_db_created) { |
|
|
2724 unlink_whine(cur_db_nm, 0); |
|
|
2725 unlink_whine(cur_hash_nm, 1); |
|
|
2726 cur_db_created = 0; |
|
|
2727 bailing = -1; |
|
|
2728 } |
|
|
2729 |
|
|
2730 if (new_db_fd >= 0) { |
|
|
2731 if (0 > close(new_db_fd)) |
|
|
2732 dcc_error_msg("close(%s): %s", |
|
|
2733 new_db_nm, ERROR_STR()); |
|
|
2734 new_db_fd = -1; |
|
|
2735 } |
|
|
2736 if (old_db_fd >= 0) { |
|
|
2737 /* In most cases nothing cares about the old database now. |
|
|
2738 * We often have kept the old database open and locked until |
|
|
2739 * now. Delete it unless we are debugging */ |
|
|
2740 if (db_debug < 4 && exit_value == EX_OK) { |
|
|
2741 unlink_whine(old_db_nm, 0); |
|
|
2742 } else { |
|
|
2743 /* Push it to the disk so it won't lurk in the buffer |
|
|
2744 * cache or elsewhere to slow a system reboot */ |
|
|
2745 if (exit_value == EX_OK |
|
|
2746 && 0 > fsync(old_db_fd)) |
|
|
2747 dcc_error_msg("fsync(%s): %s", |
|
|
2748 old_db_nm, ERROR_STR()); |
|
|
2749 } |
|
|
2750 if (0 > close(old_db_fd)) |
|
|
2751 dcc_error_msg("close(%s): %s", |
|
|
2752 old_db_nm, ERROR_STR()); |
|
|
2753 old_db_fd = -1; |
|
|
2754 } |
|
|
2755 flod_unmap(0, 0); |
|
|
2756 |
|
|
2757 /* release the daemon, but if the database is still open, it's bad */ |
|
|
2758 db_close(bailing); |
|
|
2759 /* tell the daemon to switch databases */ |
|
|
2760 if (dccd_unlocked) |
|
|
2761 dccd_new_db("finish"); |
|
|
2762 |
|
|
2763 while (flods_off > 0) { |
|
|
2764 --flods_off; |
|
|
2765 if (!persist_aop(DCC_AOP_FLOD, DCC_AOP_FLOD_RESUME, |
|
|
2766 RESTART_DELAY)) |
|
|
2767 dcc_error_msg("%s", dcc_emsg); |
|
|
2768 } |
|
|
2769 |
|
|
2770 unlock_dbclean(); |
|
|
2771 } |
|
|
2772 |
|
|
2773 |
|
|
2774 |
|
|
2775 static void NRATTRIB |
|
|
2776 exit_dbclean(int v) |
|
|
2777 { |
|
|
2778 exit(exit_value = v); |
|
|
2779 } |
|
|
2780 |
|
|
2781 |
|
|
2782 |
|
|
2783 /* terminate with a signal */ |
|
|
2784 static void NRATTRIB |
|
|
2785 sigterm(int s) |
|
|
2786 { |
|
|
2787 dcc_error_msg("interrupted by signal %d", s); |
|
|
2788 exit_dbclean(s+100); |
|
|
2789 } |
