Mercurial > notdcc

comparison srvrlib/db.h @ 0:c7f6b056b673

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
First import of vendor version
author Peter Gervai <grin@grin.hu>
date Tue, 10 Mar 2009 13:49:58 +0100
parents
children
comparison
equal deleted inserted replaced
-1:000000000000 0:c7f6b056b673
1 /* Distributed Checksum Clearinghouse database definitions
2 *
3 * Copyright (c) 2008 by Rhyolite Software, LLC
4 *
5 * This agreement is not applicable to any entity which sells anti-spam
6 * solutions to others or provides an anti-spam solution as part of a
7 * security solution sold to other entities, or to a private network
8 * which employs the DCC or uses data provided by operation of the DCC
9 * but does not provide corresponding data to other users.
10 *
11 * Permission to use, copy, modify, and distribute this software without
12 * changes for any purpose with or without fee is hereby granted, provided
13 * that the above copyright notice and this permission notice appear in all
14 * copies and any distributed versions or copies are either unchanged
15 * or not called anything similar to "DCC" or "Distributed Checksum
16 * Clearinghouse".
17 *
18 * Parties not eligible to receive a license under this agreement can
19 * obtain a commercial license to use DCC by contacting Rhyolite Software
20 * at sales@rhyolite.com.
21 *
22 * A commercial license would be for Distributed Checksum and Reputation
23 * Clearinghouse software. That software includes additional features. This
24 * free license for Distributed ChecksumClearinghouse Software does not in any
25 * way grant permision to use Distributed Checksum and Reputation Clearinghouse
26 * software
27 *
28 * THE SOFTWARE IS PROVIDED "AS IS" AND RHYOLITE SOFTWARE, LLC DISCLAIMS ALL
29 * WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES
30 * OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL RHYOLITE SOFTWARE, LLC
31 * BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES
32 * OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,
33 * WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION,
34 * ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS
35 * SOFTWARE.
36 *
37 * Rhyolite Software DCC 1.3.103-1.139 $Revision$
38 */
39
40 #ifndef DB_H
41 #define DB_H
42
43 #include "srvr_defs.h"
44 #include <math.h>
45
46 extern u_char grey_on;
47
48 #define DB_DCC_NAME "dcc_db"
49 #define DB_GREY_NAME "grey_db"
50 #define DB_HASH_SUFFIX ".hash"
51 #define DB_LOCK_SUFFIX ".lock"
52
53 #define WHITELIST_NM(g) ((g) ? "grey_whitelist" : "whitelist")
54
55 #define DB_VERSION3_STR "DCC checksum database version 3"
56 #define DB_VERSION4_STR "DCC checksum database version 4"
57 #define DB_VERSION5_STR "DCC checksum database version 5"
58 /* tighten DB_EX_SEC in the next version */
59 #define DB_VERSION_STR DB_VERSION5_STR
60
61 #define HASH_MAGIC_STR "DCC hash 6"
62
63 /* hash table indeces are only 32 bits */
64 #define MAX_HASH_ENTRIES 0xffffffff
65
66 #define MIN_CLEAN_HASH_ENTRIES 1024 /* run dbclean at this size */
67 #define MIN_HASH_ENTRIES (8*MIN_CLEAN_HASH_ENTRIES)
68 #define DEF_HASH_ENTRIES (6*1024*1024)
69 #define MIN_HASH_DIVISOR ((MIN_HASH_ENTRIES*7)/8)
70
71
72 #define DB_CP3(x,v) do {u_int32_t _v = v; (x)[0] = _v>>16; \
73 (x)[1] = _v>>8; (x)[2] = _v;} while (0)
74 #define DB_CP4(x,v) do {u_int32_t _v = v; (x)[0] = _v>>24; \
75 (x)[1] = _v>>16; (x)[2] = _v>>8; (x)[3] = _v;} while (0)
76 #define DB_EX3(x) ((((u_int32_t)(x)[0])<<16) + ((x)[1]<<8) + (x)[2])
77 #define DB_EX4(x) ((((u_int32_t)(x)[0])<<24) + (((u_int32_t)(x)[1])<<16) \
78 + ((x)[2]<<8) + (x)[3])
79 /* the least significant byte should be tested first */
80 #define DB_ZERO3(x) ((x)[2] == 0 && (x)[1] == 0 && (x)[0] == 0)
81 #define DB_ZERO4(x) ((x)[3] == 0 && (x)[2] == 0 && (x)[1] == 0 && (x)[0] == 0)
82
83
84 /* a single checksum in a database record */
85 typedef u_char DB_TGTS[3]; /* a compressed count */
86 typedef u_int64_t DB_PTR; /* database record offset */
87 typedef u_int32_t DB_PTR_C; /* compressed by DB_PTR_CP() */
88 typedef struct {
89 DB_PTR_C prev; /* previous record for this checksum */
90 DB_TGTS tgts; /* accumulated reported targets */
91 DCC_CK_TYPE_B type_fgs;
92 # define DB_CK_FG_OBS 0x80 /* obsolete report of a checksum */
93 /* # define DB_CK_ 0x40 once used */
94 # define DB_CK_MASK 0x0f
95 # define DB_CK_OBS(ck) ((ck)->type_fgs & DB_CK_FG_OBS)
96 # define DB_CK_TYPE(ck) ((DCC_CK_TYPES)((ck)->type_fgs & DB_CK_MASK))
97 DCC_SUM sum;
98 } DB_RCD_CK;
99 #define DB_TGTS_CK_SET(ck,v) DB_CP3((ck)->tgts,v)
100 #define DB_TGTS_CK(ck) DB_EX3((ck)->tgts)
101
102 /* shape of a checksum database entry */
103 typedef struct {
104 DCC_TS ts; /* original server's creation date */
105 DCC_SRVR_ID srvr_id_auth; /* initial server & client auth bit */
106 # define DB_RCD_ID(r) ((r)->srvr_id_auth & ~DCC_SRVR_ID_AUTH)
107 DB_TGTS tgts_del; /* # target addresses or delete flag */
108 u_char fgs_num_cks; /* # of cksums | flags */
109 # define DB_RCD_FG_TRIM 0x80 /* some checksums deleted */
110 # define DB_RCD_FG_SUMRY 0x40 /* fake summary record */
111 # define DB_RCD_FG_DELAY 0x20 /* delayed for fake summary */
112 # define DB_RCD_TRIMMED(r) ((r)->fgs_num_cks & DB_RCD_FG_TRIM)
113 # define DB_RCD_SUMRY(r) ((r)->fgs_num_cks & DB_RCD_FG_SUMRY)
114 # define DB_RCD_DELAY(r) ((r)->fgs_num_cks & DB_RCD_FG_DELAY)
115 # define DB_NUM_CKS(r) ((r)->fgs_num_cks & DB_CK_MASK)
116 DB_RCD_CK cks[DCC_DIM_CKS];
117 } DB_RCD;
118
119 #define DB_RCD_HDR_LEN (ISZ(DB_RCD) - ISZ(DB_RCD_CK)*DCC_DIM_CKS)
120 #define DB_RCD_LEN(r) (DB_RCD_HDR_LEN + DB_NUM_CKS(r) * ISZ(DB_RCD_CK))
121 #define DB_RCD_LEN_MAX sizeof(DB_RCD)
122
123 #define DB_TGTS_RCD_SET(r,v) DB_CP3((r)->tgts_del,v)
124 #define DB_TGTS_RCD_RAW(r) DB_EX3((r)->tgts_del)
125 static inline DCC_TGTS
126 DB_TGTS_RCD(const DB_RCD *r)
127 {
128 DCC_TGTS e = DB_TGTS_RCD_RAW(r);
129 return e == DCC_TGTS_DEL ? 0 : e;
130 }
131
132 /* this allows database of up to 48 GBytes */
133 #define DB_PTR_MULT ((DB_PTR)12) /* gcd of all sizes of DB_RCD */
134 #define DB_PTR_CP(v) ((u_int32_t)((v) / DB_PTR_MULT))
135 #define DB_PTR_EX(x) ((x) * DB_PTR_MULT)
136
137 /* The kludge to speed conversion of database addresses to page numbers
138 * and offsets on 32-bit systems */
139 #define DB_PTR_SHIFT 8
140 #ifdef HAVE_64BIT_LONG
141 #define DB_PTR2PG_NUM(p,s) ((p) / (s))
142 #else
143 #define DB_PTR2PG_NUM(p,s) ((u_int32_t)((p) >> DB_PTR_SHIFT) \
144 / (s >> DB_PTR_SHIFT))
145 #endif
146
147 #define DB_PTR_NULL 0
148 #define DB_PTR_BASE ISZ(DB_HDR)
149 #define DB_PTR_MAX DB_PTR_EX((((DB_PTR)1)<<(sizeof(DB_PTR_C)*8)) -1)
150 #define DB_PTR_BAD (DB_PTR_MAX+1)
151 #define DB_PTR_IS_BAD(l) ((l) < DB_PTR_BASE || (l) >= DB_PTR_MAX)
152
153
154 typedef DCC_TS DB_SN; /* database serial number */
155
156 /* non-spam expiration */
157 #define DB_EXPIRE_SECS_DEF (24*60*60)
158 #define DB_EXPIRE_SECS_MAX DCC_MAX_SECS
159 #define DB_EXPIRE_SECS_MIN (60*60)
160 #define DB_EXPIRE_SECS_DEF_MIN (2*60*60)
161 /* spam expiration */
162 #define DB_EXPIRE_SPAMSECS_DEF (30*24*60*60)
163 #define DB_EXPIRE_SPAMSECS_DEF_MIN (1*24*60*60)
164 #define DB_EXPIRE_REP_SECS_DEF DB_EXPIRE_SECS_DEF
165 #define DB_EXPIRE_REP_SPAMSECS_DEF DB_EXPIRE_SPAMSECS_DEF
166 /* keep server-ID declarations a week longer than reputations so that they
167 * will be first to be flooded */
168 #define DB_EXPIRE_SRVR_ID_SECS (DB_EXPIRE_REP_SPAMSECS_DEF+7*24*60*60)
169
170 /* re-announce spam this often */
171 #define DCC_OLD_SPAM_SECS (DB_EXPIRE_SPAMSECS_DEF_MIN/2)
172
173
174 /* seconds to greylist or delay new mail messages
175 * RFC 2821 says SMTP clients should wait at least 30 minutes to retry,
176 * but 15 minutes seems more common than 30 minutes. Many retry after
177 * only 5 minutes, and some after only 1 (one!) second. However,
178 * many of those that retry after a few seconds keep trying for a minute
179 * or two. */
180 #define DEF_GREY_EMBARGO 270
181 #define MAX_GREY_EMBARGO (24*60*60)
182
183 #define DEF_GREY_WINDOW (7*24*60*60) /* wait as long as this */
184 #define MAX_GREY_WINDOW (10*24*60*60)
185 #define DEF_GREY_WHITE (63*24*60*60) /* remember this long */
186 #define MAX_GREY_WHITE DB_EXPIRE_SECS_MAX
187
188
189 typedef struct {
190 DCC_TS all;
191 DCC_TS spam;
192 } DB_EX_TS_TYPE;
193 typedef DB_EX_TS_TYPE DB_EX_TS[DCC_DIM_CKS];
194 typedef DCC_TS DB_SPAM_EX_TS[DCC_DIM_CKS];
195 typedef struct {
196 DCC_TGTS unused;
197 int32_t all; /* allsecs */
198 int32_t spam; /* spamsecs */
199 } DB_EX_SEC;
200 typedef DB_EX_SEC DB_EX_SECS[DCC_DIM_CKS];
201
202 #define DCC_CK_OK_GREY_CLNT(t) ((t) > DCC_CK_INVALID \
203 && t <= DCC_CK_G_TRIPLE_R_BULK)
204 #define DCC_CK_OK_GREY_FLOD(t) ((t) == DCC_CK_BODY \
205 || ((t) >= DCC_CK_G_MSG_R_TOTAL \
206 && (t) <= DCC_CK_FLOD_PATH) \
207 || ((t) == DCC_CK_IP && grey_weak_ip))
208
209 #define DEF_FLOD_THOLDS(g,t) ((g) ? 1 \
210 : t == DCC_CK_SRVR_ID ? 1 : BULK_THRESHOLD)
211
212 #define DCC_CK_OK_DCC_CLNT(g,t) ((t) > DCC_CK_INVALID \
213 && (t) <= DCC_CK_G_TRIPLE_R_BULK \
214 && ((g)|| (t) <= DCC_CK_FUZ2))
215 #define DCC_CK_OK_DB(g,t) ((t) > DCC_CK_INVALID && t <= DCC_CK_TYPE_LAST \
216 && ((g) || ((t) != DCC_CK_G_MSG_R_TOTAL \
217 && (t) != DCC_CK_G_TRIPLE_R_BULK)))
218 #define DCC_CK_OK_FLOD(g,t) ((g) ? DCC_CK_OK_GREY_FLOD(t) \
219 : ((t) > DCC_CK_INVALID \
220 && ((t) <= DCC_CK_FUZ2 \
221 || (t) == DCC_CK_FLOD_PATH \
222 || (t) == DCC_CK_SRVR_ID)))
223
224
225 typedef u_int32_t DB_NOKEEP_CKS; /* bitmask of ignored checksums */
226 #define DB_SET_NOKEEP(map,t) ((map) |= (1<<(t)))
227 #define DB_RESET_NOKEEP(map,t) ((map) &= ~(1<<(t)))
228 #define DB_TEST_NOKEEP(map,t) ((map) & (1<<(t)))
229
230 /* relative fuzziness of checksums */
231 #define DCC_CK_FUZ_LVL_NO 1 /* least fuzzy */
232 #define DCC_CK_FUZ_LVL1 2 /* somewhat fuzzy */
233 #define DCC_CK_FUZ_LVL2 3 /* fuzzier */
234 #define DCC_CK_FUZ_LVL3 4 /* reputations */
235 #define DCC_CK_FUZ_LVL_REP DCC_CK_FUZ_LVL3
236 extern const u_char *db_ck_fuzziness;
237
238
239 typedef DB_PTR DB_HOFF; /* byte offset into hash table */
240 typedef u_int32_t DB_HADDR; /* index of a hash table entry */
241 typedef u_char DB_HADDR_C[4]; /* compressed hash chain link */
242 #define DB_HADDR_CP(x,v) DB_CP4(x,v)
243 #define DB_HADDR_EX(x) DB_EX4(x)
244 #define DB_HADDR_NULL 0 /* no-answer from hashing & linking */
245 #define DB_HADDR_C_NULL(x) DB_ZERO4(x)
246 #define DB_HADDR_INVALID(h) ((h) < DB_HADDR_BASE || (h) >= db_hash_len)
247 #define DB_HADDR_C_INVALID(h) DB_HADDR_INVALID(DB_HADDR_EX(h))
248
249 typedef u_char DB_PTR_HC[4];
250 #define DB_HPTR_CP(x,v) {u_int32_t _v = DB_PTR_CP(v); \
251 (x)[0] = _v>>24; (x)[1] = _v>>16; (x)[2] = _v>>8; (x)[3] = _v;}
252 #define DB_HPTR_EX(x) DB_PTR_EX(((x)[0]<<24) + ((x)[1]<<16) \
253 + ((x)[2]<<8) + (x)[3])
254
255
256 /* shape of the magic string that starts a database */
257 typedef char DB_VERSION_BUF[64];
258 typedef struct {
259 DB_VERSION_BUF version; /* see DB_VERSION_STR */
260 DB_PTR db_csize; /* size of database contents in bytes */
261 u_int32_t pagesize; /* size of 1 DB buffer */
262 DB_SN sn; /* creation or expiration serial # */
263 time_t cleared; /* when created */
264 time_t cleaned; /* real instead of repair cleaning */
265 time_t cleaned_cron; /* cleaned by cron */
266 DB_SPAM_EX_TS ex_spam; /* recent expiration timestamps */
267 DB_SPAM_EX_TS ex_all; /* recent expiration timestamps */
268 DB_EX_SECS ex_secs; /* recent expiration thresholds */
269 DB_NOKEEP_CKS nokeep_cks; /* ignore these checksums */
270 u_int flags;
271 # define DB_PARM_FG_GREY 0x01 /* greylist database */
272 # define DB_PARM_FG_CLEARED 0x02 /* new file */
273 # define DB_PARM_EXP_SET 0x04 /* have explicit expiration durations */
274 DB_PTR old_db_csize; /* size at last cleaning */
275 DB_PTR db_added; /* bytes previously added to database */
276 DB_HADDR hash_used; /* recent of entries used */
277 DB_HADDR old_hash_used; /* entries used at last cleaning */
278 DB_HADDR hash_added; /* entries added */
279 time_t rate_secs; /* denominator of rates */
280 # define DB_MIN_RATE_SECS (12*60*60)
281 # define DB_MAX_RATE_SECS (14*24*60*60)
282 time_t last_rate_sec;
283 DB_HADDR old_kept_cks; /* reported checksums at cleaning */
284 } DB_PARMS;
285 typedef union {
286 DB_PARMS p;
287 char c[256*3];
288 } DB_HDR;
289
290 #ifdef DB_VERSION4_STR
291 typedef struct {
292 DB_VERSION_BUF version;
293 DB_PTR db_csize;
294 u_int32_t pagesize;
295 DB_SN sn;
296 time_t cleared;
297 time_t cleaned;
298 time_t cleaned_cron;
299 DB_SPAM_EX_TS ex_spam;
300 DB_EX_SECS ex_secs;
301 DB_NOKEEP_CKS nokeep_cks;
302 u_int flags;
303 DB_PTR old_db_csize;
304 DB_PTR db_added;
305 DB_HADDR hash_used;
306 DB_HADDR old_hash_used;
307 DB_HADDR hash_added;
308 time_t rate_secs;
309 time_t last_rate_sec;
310 DB_HADDR old_kept_cks;
311 } DB_V4_PARMS;
312 #endif
313 #ifdef DB_VERSION3_STR
314 typedef struct {
315 DB_VERSION_BUF version;
316 DB_PTR db_csize;
317 u_int32_t pagesize;
318 DB_SN sn;
319 DB_SPAM_EX_TS ex_spam;
320 DB_EX_SECS ex_secs;
321 DB_NOKEEP_CKS nokeep_cks;
322 DCC_TGTS unused[DCC_DIM_CKS];
323 u_int flags;
324 # define DB_PARM_V3_FG_GREY 0x01
325 # define DB_PARM_V3_FG_SELF_CLEAN 0x02
326 # define DB_PARM_V3_FG_SELF_CLEAN2 0x04
327 # define DB_PARM_V3_FG_CLEARED 0x08
328 DB_PTR old_db_csize;
329 DB_PTR db_added;
330 DB_HADDR hash_used;
331 DB_HADDR old_hash_used;
332 DB_HADDR hash_added;
333 time_t rate_secs;
334 time_t last_rate_sec;
335 DB_HADDR old_kept_cks;
336 } DB_V3_PARMS;
337 #endif
338
339 /* shape of a database hash table entry */
340 typedef struct {
341 DB_HADDR_C fwd, bak; /* hash collision chain */
342 u_char hv_type[2]; /* checksum type + some hash bits */
343 # define HE_TYPE(e) ((DCC_CK_TYPES)((e)->hv_type[0] & 0xf))
344 # define HE_IS_FREE(e) ((e)->hv_type[0] == 0)
345 # define HE_MERGE(e,t,s) ((e)->hv_type[0] = ((((s)[0])<<4)+t), \
346 (e)->hv_type[1] = (s)[1])
347 # define HE_CMP(e,t,s) ((e)->hv_type[1] == (s)[1] \
348 && (e)->hv_type[0] ==(u_char)((((s)[0])<<4)+t))
349
350 DB_PTR_HC rcd; /* record for this hash table entry */
351 } HASH_ENTRY;
352
353
354
355 typedef union {
356 HASH_ENTRY h[8]; /* this must be larger than following */
357 struct {
358 char magic[16];
359 u_int flags;
360 # define HASH_CTL_FG_CLEAN 0x01 /* consistent */
361 # define HASH_CTL_FG_NOSYNC 0x02 /* pushed to stable storage */
362 DB_HADDR free_fwd; /* hash table internal free list */
363 DB_HADDR free_bak;
364 # define FREE_HADDR_END 1
365 DB_HADDR len; /* size of file in entries */
366 DB_HADDR used; /* entries actually used */
367 DB_HADDR divisor; /* hash modulus */
368 DB_PTR db_csize; /* size of the database file */
369 time_t synced;
370 } s;
371 } HASH_CTL;
372
373
374 #define DB_HADDR_BASE ((DB_HADDR)((sizeof(HASH_CTL)+sizeof(HASH_ENTRY)-1) \
375 / sizeof(HASH_ENTRY)))
376 #define HADDR2LEN(l) ((int)((l)-DB_HADDR_BASE)) /* offset to length */
377
378
379 /* control a block of mapped memory */
380 typedef u_int16_t DB_PG_NUM;
381 typedef u_int32_t DB_PG_OFF;
382 typedef enum {
383 DB_BUF_TYPE_FREE = 0,
384 DB_BUF_TYPE_HASH,
385 DB_BUF_TYPE_DB
386 } DB_BUF_TYPE;
387 #ifdef HAVE_64BIT_LONG
388 typedef u_int64_t DB_BUF_FM;
389 #else
390 typedef u_int32_t DB_BUF_FM;
391 #endif
392 #define DB_BUF_NUM_PARTS (8*ISZ(DB_BUF_FM))
393 #define PART2BIT(part) (((DB_BUF_FM)1) << (part))
394 typedef struct db_buf {
395 struct db_buf *fwd, *bak, **hash;
396 struct db_buf *older, *newer;
397 union {
398 void *v;
399 HASH_ENTRY *h;
400 char *c;
401 } buf;
402 DB_PG_NUM pg_num;
403 int lock_cnt;
404 DB_BUF_TYPE buf_type;
405 DB_BUF_FM flush;
406 DB_BUF_FM flush_urgent;
407 struct {
408 char *lo, *hi;
409 } ranges[DB_BUF_NUM_PARTS];
410 u_char flags;
411 # define DB_BUF_FG_USE_WRITE 0x01 /* use write() */
412 # define DB_BUF_FG_EXTENSION 0x02 /* new page in file */
413 } DB_BUF;
414
415 /* context for searching for or adding a record */
416 typedef struct {
417 union { /* pointer to data in buffer */
418 void *v;
419 HASH_ENTRY *h;
420 char *c;
421 DB_RCD *r;
422 DB_PARMS *parms;
423 HASH_CTL *vals;
424 } d;
425 union { /* database address */
426 DB_HADDR haddr;
427 DB_PTR rptr;
428 } s;
429 DB_BUF *b;
430 } DB_STATE;
431
432 /* see db_close() before changing this */
433 typedef struct {
434 DB_STATE rcd; /* must be first */
435 DB_STATE rcd2;
436 DB_STATE sumrcd;
437 DB_STATE hash;
438 DB_STATE free;
439 DB_STATE tmp;
440 DB_STATE db_parms;
441 DB_STATE hash_ctl; /* hash control info; must be last */
442 } DB_STATES;
443 extern DB_STATES db_sts;
444
445 extern int db_failed_line;
446 extern const char *db_failed_file;
447 #define DB_ERROR_MSG(s) db_error_msg(__LINE__,__FILE__, "%s", s)
448 #define DB_ERROR_MSG2(s1,s2) db_error_msg(__LINE__,__FILE__, "%s: %s", s1,s2)
449
450 extern struct timeval db_time;
451 #define DB_IS_TIME(tgt,lim) DCC_IS_TIME(db_time.tv_sec,tgt,lim)
452 #define DB_ADJ_TIMER(tgt,lim,new) DCC_ADJ_TIMER(db_time.tv_sec,tgt,lim,new)
453
454 extern u_char db_minimum_map; /* this is dccd & dbclean is running */
455 extern int db_fd, db_hash_fd;
456 extern DCC_PATH db_nm, db_hash_nm;
457 extern struct timeval db_locked; /* 0 or when database was locked */
458 extern int db_debug;
459 extern DB_SN db_sn;
460
461 extern DB_HOFF db_hash_fsize; /* size of hash table file */
462 extern DB_HADDR db_hash_len; /* # of hash table entries */
463 extern DB_HADDR db_hash_divisor; /* modulus */
464 extern DB_HADDR db_hash_used; /* # of hash table entries in use */
465 extern u_int db_hash_page_len; /* # of HASH_ENTRY's per buffer */
466 extern DB_HADDR db_max_hash_entries; /* max size of hash table */
467 extern DB_PTR db_fsize; /* size of database file */
468 extern DB_PTR db_csize; /* size of database contents in bytes */
469 extern const DB_VERSION_BUF db_version_buf;
470 extern DB_PARMS db_parms;
471 extern DCC_TGTS db_tholds[DCC_DIM_CKS];
472 extern u_int db_pagesize; /* size of 1 DB buffer */
473 extern u_int db_page_max; /* only padding after this */
474 extern char db_window_size_str[]; /* size of mmap() window */
475
476 typedef struct {
477 u_int db_mmaps;
478 u_int hash_mmaps;
479 u_int adds; /* reports added */
480 } DB_STATS;
481 extern DB_STATS db_stats;
482
483
484 /* If the two files were smaller than the typical mmap() limit of a fraction
485 * of a GByte, they could be mmap()'ed directly. They are often too large.
486 *
487 * Use a modest pool of buffers to map the DB hash table and the database
488 * itself.
489 * Each access to the files could be with a single, common buffer,
490 * but that would involve many more mmap() system calls.
491 * Most of the DB hash table is expected to fit in the application's memory.
492 *
493 * Use the same modest pool of buffers to map the database itself.
494 * References to the database have a lot of locality, so the commonly used
495 * checksums and counts should remain in memory.
496 *
497 * Common operating system limits on the number of mapped segments are
498 * below 256 and so that is a bound on DB_BUF_MAX */
499 #define DB_BUF_MAX 128 /* maximum # of buffers */
500 #define DB_BUF_PARTS_MAX (DB_BUF_MAX*DB_BUF_NUM_PARTS)
501
502 /* enough buffers so max simultaneous pointers can be satisfied */
503 #define DB_BUF_MIN (sizeof(DB_STATES)/sizeof(DB_STATE) + 2)
504
505 extern int db_buf_total; /* total # of db buffers */
506 extern DB_PTR db_max_rss; /* maximum db resident set size */
507 extern DB_PTR db_max_byte;
508
509
510 extern time_t db_need_flush_secs;
511 #define DB_NEED_FLUSH_SECS 5
512 #define DB_STALE_SECS (30*60) /* limit on buffer staleness */
513 #define DB_FLUSHES (DB_STALE_SECS / DB_NEED_FLUSH_SECS)
514 #define DB_PARTS_PER_FLUSH ((DB_BUF_PARTS_MAX + DB_FLUSHES-1) / DB_FLUSHES)
515 #define DB_URGENT_NEED_FLUSH_SECS 120
516
517
518 /* fix configure script if this changes */
519 #define DB_MIN_MIN_MBYTE 32
520 #define DB_DEF_MIN_MBYTE 64 /* a reasonable tiny default */
521 #define DB_PAD_MBYTE 128 /* RAM for rate limiting blocks etc */
522 #define DB_PAD_BYTE (DB_PAD_MBYTE*1024*1024)
523 #define DB_MAX_2G_MBYTE (2048-DB_PAD_MBYTE) /* <2 GByte on 32 bit machines */
524 /* the database cannot exceed 48 GBytes because of DB_PTR_CP */
525 #define MAX_MAX_DB_MBYTE (48*1024)
526 /* fix INSTALL.html if those change */
527
528
529 /* srvr/db.c */
530 extern void db_failure(int, const char *, int, DCC_EMSG,
531 const char *, ...) PATTRIB(5,6);
532 extern void db_error_msg(int, const char *, const char *, ...) PATTRIB(3,4);
533 extern void db_set_flush(DB_STATE *, u_char, u_int);
534 #define SET_FLUSH_RCD(st,u) db_set_flush(st,u, DB_RCD_LEN((st)->d.r))
535 #define SET_FLUSH_RCD_HDR(st,u) db_set_flush(st,u, DB_RCD_HDR_LEN)
536 #define SET_FLUSH_HE(st) db_set_flush(st, 0, sizeof(HASH_ENTRY))
537 #define SET_FLUSH_HCTL(u) db_set_flush(&db_sts.hash_ctl,u, sizeof(HASH_CTL))
538 extern void rel_db_states(void);
539 extern u_char db_unload(DCC_EMSG, u_char);
540 extern u_char db_flush_db(DCC_EMSG);
541 extern u_char make_clean(u_char);
542 extern u_char db_close(int);
543 extern void db_stop(void);
544 extern u_char lock_dbclean(DCC_EMSG, const char *);
545 extern void unlock_dbclean(void);
546 extern u_int db_get_pagesize(u_int, u_int);
547 extern u_char db_buf_init(u_int, u_int);
548 typedef u_char DB_OPEN_MODES;
549 # define DB_OPEN_RDONLY 0x01
550 # define DB_OPEN_LOCK_WAIT 0x02 /* wait to get lock */
551 # define DB_OPEN_LOCK_NOWAIT 0x04 /* get lock but don't wait */
552 # define DB_OPEN_MMAP_WRITE 0x08 /* use write() instead of mmap() */
553 # define DB_OPEN_MMAP_WRITE_NOSYNC 0x10 /* if no mmap(NOSYNC) */
554 extern u_char db_open(DCC_EMSG, int, const char *, DB_HADDR, DB_OPEN_MODES);
555 extern u_char db_flush_parms(DCC_EMSG );
556 #define DB_IS_LOCKED() (db_locked.tv_sec != 0)
557 extern int db_lock(void);
558 extern u_char db_unlock(void);
559 extern void db_flush_needed(void);
560 extern DCC_TGTS db_sum_ck(DCC_TGTS, DCC_TGTS, DCC_CK_TYPES);
561 extern const char *db_ptr2str(DB_PTR);
562 extern const char *size2str(char *, u_int, double, u_char);
563 extern double db_add_rate(const DB_PARMS *, u_char);
564 extern DB_NOKEEP_CKS def_nokeep_cks(void);
565 extern void set_db_tholds(DB_NOKEEP_CKS);
566 extern u_char db_map_rcd(DCC_EMSG, DB_STATE *, DB_PTR, int *);
567 extern DB_RCD_CK *db_find_ck(DCC_EMSG, DB_RCD *, DB_PTR, DCC_CK_TYPES);
568 extern DB_RCD_CK *db_map_rcd_ck(DCC_EMSG, DB_STATE *, DB_PTR, DCC_CK_TYPES);
569 extern DB_HADDR get_db_hash_divisor(DB_HADDR);
570 extern DB_HADDR db_hash(DCC_CK_TYPES, const DCC_SUM);
571 typedef enum {
572 DB_FOUND_SYSERR=0, /* fatal error */
573 DB_FOUND_LATER, /* out of specified hash table range */
574 DB_FOUND_IT,
575 DB_FOUND_EMPTY, /* home slot empty */
576 DB_FOUND_CHAIN, /* not in chain--have last entry */
577 DB_FOUND_INTRUDER /* intruder in home slot */
578 } DB_FOUND;
579 extern DB_FOUND db_lookup(DCC_EMSG, DCC_CK_TYPES, const DCC_SUM,
580 DB_HADDR, DB_HADDR, DB_STATE *,
581 DB_STATE *, DB_RCD_CK **);
582 extern u_char db_link_rcd(DCC_EMSG, DB_HADDR, DB_HADDR);
583 extern DB_PTR db_add_rcd(DCC_EMSG, const DB_RCD *);
584
585 #endif /* DB_H */