|
0
|
1 /* Distributed Checksum Clearinghouse database definitions |
|
|
2 * |
|
|
3 * Copyright (c) 2008 by Rhyolite Software, LLC |
|
|
4 * |
|
|
5 * This agreement is not applicable to any entity which sells anti-spam |
|
|
6 * solutions to others or provides an anti-spam solution as part of a |
|
|
7 * security solution sold to other entities, or to a private network |
|
|
8 * which employs the DCC or uses data provided by operation of the DCC |
|
|
9 * but does not provide corresponding data to other users. |
|
|
10 * |
|
|
11 * Permission to use, copy, modify, and distribute this software without |
|
|
12 * changes for any purpose with or without fee is hereby granted, provided |
|
|
13 * that the above copyright notice and this permission notice appear in all |
|
|
14 * copies and any distributed versions or copies are either unchanged |
|
|
15 * or not called anything similar to "DCC" or "Distributed Checksum |
|
|
16 * Clearinghouse". |
|
|
17 * |
|
|
18 * Parties not eligible to receive a license under this agreement can |
|
|
19 * obtain a commercial license to use DCC by contacting Rhyolite Software |
|
|
20 * at sales@rhyolite.com. |
|
|
21 * |
|
|
22 * A commercial license would be for Distributed Checksum and Reputation |
|
|
23 * Clearinghouse software. That software includes additional features. This |
|
|
24 * free license for Distributed ChecksumClearinghouse Software does not in any |
|
|
25 * way grant permision to use Distributed Checksum and Reputation Clearinghouse |
|
|
26 * software |
|
|
27 * |
|
|
28 * THE SOFTWARE IS PROVIDED "AS IS" AND RHYOLITE SOFTWARE, LLC DISCLAIMS ALL |
|
|
29 * WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES |
|
|
30 * OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL RHYOLITE SOFTWARE, LLC |
|
|
31 * BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES |
|
|
32 * OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, |
|
|
33 * WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, |
|
|
34 * ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS |
|
|
35 * SOFTWARE. |
|
|
36 * |
|
|
37 * Rhyolite Software DCC 1.3.103-1.139 $Revision$ |
|
|
38 */ |
|
|
39 |
|
|
40 #ifndef DB_H |
|
|
41 #define DB_H |
|
|
42 |
|
|
43 #include "srvr_defs.h" |
|
|
44 #include <math.h> |
|
|
45 |
|
|
46 extern u_char grey_on; |
|
|
47 |
|
|
48 #define DB_DCC_NAME "dcc_db" |
|
|
49 #define DB_GREY_NAME "grey_db" |
|
|
50 #define DB_HASH_SUFFIX ".hash" |
|
|
51 #define DB_LOCK_SUFFIX ".lock" |
|
|
52 |
|
|
53 #define WHITELIST_NM(g) ((g) ? "grey_whitelist" : "whitelist") |
|
|
54 |
|
|
55 #define DB_VERSION3_STR "DCC checksum database version 3" |
|
|
56 #define DB_VERSION4_STR "DCC checksum database version 4" |
|
|
57 #define DB_VERSION5_STR "DCC checksum database version 5" |
|
|
58 /* tighten DB_EX_SEC in the next version */ |
|
|
59 #define DB_VERSION_STR DB_VERSION5_STR |
|
|
60 |
|
|
61 #define HASH_MAGIC_STR "DCC hash 6" |
|
|
62 |
|
|
63 /* hash table indeces are only 32 bits */ |
|
|
64 #define MAX_HASH_ENTRIES 0xffffffff |
|
|
65 |
|
|
66 #define MIN_CLEAN_HASH_ENTRIES 1024 /* run dbclean at this size */ |
|
|
67 #define MIN_HASH_ENTRIES (8*MIN_CLEAN_HASH_ENTRIES) |
|
|
68 #define DEF_HASH_ENTRIES (6*1024*1024) |
|
|
69 #define MIN_HASH_DIVISOR ((MIN_HASH_ENTRIES*7)/8) |
|
|
70 |
|
|
71 |
|
|
72 #define DB_CP3(x,v) do {u_int32_t _v = v; (x)[0] = _v>>16; \ |
|
|
73 (x)[1] = _v>>8; (x)[2] = _v;} while (0) |
|
|
74 #define DB_CP4(x,v) do {u_int32_t _v = v; (x)[0] = _v>>24; \ |
|
|
75 (x)[1] = _v>>16; (x)[2] = _v>>8; (x)[3] = _v;} while (0) |
|
|
76 #define DB_EX3(x) ((((u_int32_t)(x)[0])<<16) + ((x)[1]<<8) + (x)[2]) |
|
|
77 #define DB_EX4(x) ((((u_int32_t)(x)[0])<<24) + (((u_int32_t)(x)[1])<<16) \ |
|
|
78 + ((x)[2]<<8) + (x)[3]) |
|
|
79 /* the least significant byte should be tested first */ |
|
|
80 #define DB_ZERO3(x) ((x)[2] == 0 && (x)[1] == 0 && (x)[0] == 0) |
|
|
81 #define DB_ZERO4(x) ((x)[3] == 0 && (x)[2] == 0 && (x)[1] == 0 && (x)[0] == 0) |
|
|
82 |
|
|
83 |
|
|
84 /* a single checksum in a database record */ |
|
|
85 typedef u_char DB_TGTS[3]; /* a compressed count */ |
|
|
86 typedef u_int64_t DB_PTR; /* database record offset */ |
|
|
87 typedef u_int32_t DB_PTR_C; /* compressed by DB_PTR_CP() */ |
|
|
88 typedef struct { |
|
|
89 DB_PTR_C prev; /* previous record for this checksum */ |
|
|
90 DB_TGTS tgts; /* accumulated reported targets */ |
|
|
91 DCC_CK_TYPE_B type_fgs; |
|
|
92 # define DB_CK_FG_OBS 0x80 /* obsolete report of a checksum */ |
|
|
93 /* # define DB_CK_ 0x40 once used */ |
|
|
94 # define DB_CK_MASK 0x0f |
|
|
95 # define DB_CK_OBS(ck) ((ck)->type_fgs & DB_CK_FG_OBS) |
|
|
96 # define DB_CK_TYPE(ck) ((DCC_CK_TYPES)((ck)->type_fgs & DB_CK_MASK)) |
|
|
97 DCC_SUM sum; |
|
|
98 } DB_RCD_CK; |
|
|
99 #define DB_TGTS_CK_SET(ck,v) DB_CP3((ck)->tgts,v) |
|
|
100 #define DB_TGTS_CK(ck) DB_EX3((ck)->tgts) |
|
|
101 |
|
|
102 /* shape of a checksum database entry */ |
|
|
103 typedef struct { |
|
|
104 DCC_TS ts; /* original server's creation date */ |
|
|
105 DCC_SRVR_ID srvr_id_auth; /* initial server & client auth bit */ |
|
|
106 # define DB_RCD_ID(r) ((r)->srvr_id_auth & ~DCC_SRVR_ID_AUTH) |
|
|
107 DB_TGTS tgts_del; /* # target addresses or delete flag */ |
|
|
108 u_char fgs_num_cks; /* # of cksums | flags */ |
|
|
109 # define DB_RCD_FG_TRIM 0x80 /* some checksums deleted */ |
|
|
110 # define DB_RCD_FG_SUMRY 0x40 /* fake summary record */ |
|
|
111 # define DB_RCD_FG_DELAY 0x20 /* delayed for fake summary */ |
|
|
112 # define DB_RCD_TRIMMED(r) ((r)->fgs_num_cks & DB_RCD_FG_TRIM) |
|
|
113 # define DB_RCD_SUMRY(r) ((r)->fgs_num_cks & DB_RCD_FG_SUMRY) |
|
|
114 # define DB_RCD_DELAY(r) ((r)->fgs_num_cks & DB_RCD_FG_DELAY) |
|
|
115 # define DB_NUM_CKS(r) ((r)->fgs_num_cks & DB_CK_MASK) |
|
|
116 DB_RCD_CK cks[DCC_DIM_CKS]; |
|
|
117 } DB_RCD; |
|
|
118 |
|
|
119 #define DB_RCD_HDR_LEN (ISZ(DB_RCD) - ISZ(DB_RCD_CK)*DCC_DIM_CKS) |
|
|
120 #define DB_RCD_LEN(r) (DB_RCD_HDR_LEN + DB_NUM_CKS(r) * ISZ(DB_RCD_CK)) |
|
|
121 #define DB_RCD_LEN_MAX sizeof(DB_RCD) |
|
|
122 |
|
|
123 #define DB_TGTS_RCD_SET(r,v) DB_CP3((r)->tgts_del,v) |
|
|
124 #define DB_TGTS_RCD_RAW(r) DB_EX3((r)->tgts_del) |
|
|
125 static inline DCC_TGTS |
|
|
126 DB_TGTS_RCD(const DB_RCD *r) |
|
|
127 { |
|
|
128 DCC_TGTS e = DB_TGTS_RCD_RAW(r); |
|
|
129 return e == DCC_TGTS_DEL ? 0 : e; |
|
|
130 } |
|
|
131 |
|
|
132 /* this allows database of up to 48 GBytes */ |
|
|
133 #define DB_PTR_MULT ((DB_PTR)12) /* gcd of all sizes of DB_RCD */ |
|
|
134 #define DB_PTR_CP(v) ((u_int32_t)((v) / DB_PTR_MULT)) |
|
|
135 #define DB_PTR_EX(x) ((x) * DB_PTR_MULT) |
|
|
136 |
|
|
137 /* The kludge to speed conversion of database addresses to page numbers |
|
|
138 * and offsets on 32-bit systems */ |
|
|
139 #define DB_PTR_SHIFT 8 |
|
|
140 #ifdef HAVE_64BIT_LONG |
|
|
141 #define DB_PTR2PG_NUM(p,s) ((p) / (s)) |
|
|
142 #else |
|
|
143 #define DB_PTR2PG_NUM(p,s) ((u_int32_t)((p) >> DB_PTR_SHIFT) \ |
|
|
144 / (s >> DB_PTR_SHIFT)) |
|
|
145 #endif |
|
|
146 |
|
|
147 #define DB_PTR_NULL 0 |
|
|
148 #define DB_PTR_BASE ISZ(DB_HDR) |
|
|
149 #define DB_PTR_MAX DB_PTR_EX((((DB_PTR)1)<<(sizeof(DB_PTR_C)*8)) -1) |
|
|
150 #define DB_PTR_BAD (DB_PTR_MAX+1) |
|
|
151 #define DB_PTR_IS_BAD(l) ((l) < DB_PTR_BASE || (l) >= DB_PTR_MAX) |
|
|
152 |
|
|
153 |
|
|
154 typedef DCC_TS DB_SN; /* database serial number */ |
|
|
155 |
|
|
156 /* non-spam expiration */ |
|
|
157 #define DB_EXPIRE_SECS_DEF (24*60*60) |
|
|
158 #define DB_EXPIRE_SECS_MAX DCC_MAX_SECS |
|
|
159 #define DB_EXPIRE_SECS_MIN (60*60) |
|
|
160 #define DB_EXPIRE_SECS_DEF_MIN (2*60*60) |
|
|
161 /* spam expiration */ |
|
|
162 #define DB_EXPIRE_SPAMSECS_DEF (30*24*60*60) |
|
|
163 #define DB_EXPIRE_SPAMSECS_DEF_MIN (1*24*60*60) |
|
|
164 #define DB_EXPIRE_REP_SECS_DEF DB_EXPIRE_SECS_DEF |
|
|
165 #define DB_EXPIRE_REP_SPAMSECS_DEF DB_EXPIRE_SPAMSECS_DEF |
|
|
166 /* keep server-ID declarations a week longer than reputations so that they |
|
|
167 * will be first to be flooded */ |
|
|
168 #define DB_EXPIRE_SRVR_ID_SECS (DB_EXPIRE_REP_SPAMSECS_DEF+7*24*60*60) |
|
|
169 |
|
|
170 /* re-announce spam this often */ |
|
|
171 #define DCC_OLD_SPAM_SECS (DB_EXPIRE_SPAMSECS_DEF_MIN/2) |
|
|
172 |
|
|
173 |
|
|
174 /* seconds to greylist or delay new mail messages |
|
|
175 * RFC 2821 says SMTP clients should wait at least 30 minutes to retry, |
|
|
176 * but 15 minutes seems more common than 30 minutes. Many retry after |
|
|
177 * only 5 minutes, and some after only 1 (one!) second. However, |
|
|
178 * many of those that retry after a few seconds keep trying for a minute |
|
|
179 * or two. */ |
|
|
180 #define DEF_GREY_EMBARGO 270 |
|
|
181 #define MAX_GREY_EMBARGO (24*60*60) |
|
|
182 |
|
|
183 #define DEF_GREY_WINDOW (7*24*60*60) /* wait as long as this */ |
|
|
184 #define MAX_GREY_WINDOW (10*24*60*60) |
|
|
185 #define DEF_GREY_WHITE (63*24*60*60) /* remember this long */ |
|
|
186 #define MAX_GREY_WHITE DB_EXPIRE_SECS_MAX |
|
|
187 |
|
|
188 |
|
|
189 typedef struct { |
|
|
190 DCC_TS all; |
|
|
191 DCC_TS spam; |
|
|
192 } DB_EX_TS_TYPE; |
|
|
193 typedef DB_EX_TS_TYPE DB_EX_TS[DCC_DIM_CKS]; |
|
|
194 typedef DCC_TS DB_SPAM_EX_TS[DCC_DIM_CKS]; |
|
|
195 typedef struct { |
|
|
196 DCC_TGTS unused; |
|
|
197 int32_t all; /* allsecs */ |
|
|
198 int32_t spam; /* spamsecs */ |
|
|
199 } DB_EX_SEC; |
|
|
200 typedef DB_EX_SEC DB_EX_SECS[DCC_DIM_CKS]; |
|
|
201 |
|
|
202 #define DCC_CK_OK_GREY_CLNT(t) ((t) > DCC_CK_INVALID \ |
|
|
203 && t <= DCC_CK_G_TRIPLE_R_BULK) |
|
|
204 #define DCC_CK_OK_GREY_FLOD(t) ((t) == DCC_CK_BODY \ |
|
|
205 || ((t) >= DCC_CK_G_MSG_R_TOTAL \ |
|
|
206 && (t) <= DCC_CK_FLOD_PATH) \ |
|
|
207 || ((t) == DCC_CK_IP && grey_weak_ip)) |
|
|
208 |
|
|
209 #define DEF_FLOD_THOLDS(g,t) ((g) ? 1 \ |
|
|
210 : t == DCC_CK_SRVR_ID ? 1 : BULK_THRESHOLD) |
|
|
211 |
|
|
212 #define DCC_CK_OK_DCC_CLNT(g,t) ((t) > DCC_CK_INVALID \ |
|
|
213 && (t) <= DCC_CK_G_TRIPLE_R_BULK \ |
|
|
214 && ((g)|| (t) <= DCC_CK_FUZ2)) |
|
|
215 #define DCC_CK_OK_DB(g,t) ((t) > DCC_CK_INVALID && t <= DCC_CK_TYPE_LAST \ |
|
|
216 && ((g) || ((t) != DCC_CK_G_MSG_R_TOTAL \ |
|
|
217 && (t) != DCC_CK_G_TRIPLE_R_BULK))) |
|
|
218 #define DCC_CK_OK_FLOD(g,t) ((g) ? DCC_CK_OK_GREY_FLOD(t) \ |
|
|
219 : ((t) > DCC_CK_INVALID \ |
|
|
220 && ((t) <= DCC_CK_FUZ2 \ |
|
|
221 || (t) == DCC_CK_FLOD_PATH \ |
|
|
222 || (t) == DCC_CK_SRVR_ID))) |
|
|
223 |
|
|
224 |
|
|
225 typedef u_int32_t DB_NOKEEP_CKS; /* bitmask of ignored checksums */ |
|
|
226 #define DB_SET_NOKEEP(map,t) ((map) |= (1<<(t))) |
|
|
227 #define DB_RESET_NOKEEP(map,t) ((map) &= ~(1<<(t))) |
|
|
228 #define DB_TEST_NOKEEP(map,t) ((map) & (1<<(t))) |
|
|
229 |
|
|
230 /* relative fuzziness of checksums */ |
|
|
231 #define DCC_CK_FUZ_LVL_NO 1 /* least fuzzy */ |
|
|
232 #define DCC_CK_FUZ_LVL1 2 /* somewhat fuzzy */ |
|
|
233 #define DCC_CK_FUZ_LVL2 3 /* fuzzier */ |
|
|
234 #define DCC_CK_FUZ_LVL3 4 /* reputations */ |
|
|
235 #define DCC_CK_FUZ_LVL_REP DCC_CK_FUZ_LVL3 |
|
|
236 extern const u_char *db_ck_fuzziness; |
|
|
237 |
|
|
238 |
|
|
239 typedef DB_PTR DB_HOFF; /* byte offset into hash table */ |
|
|
240 typedef u_int32_t DB_HADDR; /* index of a hash table entry */ |
|
|
241 typedef u_char DB_HADDR_C[4]; /* compressed hash chain link */ |
|
|
242 #define DB_HADDR_CP(x,v) DB_CP4(x,v) |
|
|
243 #define DB_HADDR_EX(x) DB_EX4(x) |
|
|
244 #define DB_HADDR_NULL 0 /* no-answer from hashing & linking */ |
|
|
245 #define DB_HADDR_C_NULL(x) DB_ZERO4(x) |
|
|
246 #define DB_HADDR_INVALID(h) ((h) < DB_HADDR_BASE || (h) >= db_hash_len) |
|
|
247 #define DB_HADDR_C_INVALID(h) DB_HADDR_INVALID(DB_HADDR_EX(h)) |
|
|
248 |
|
|
249 typedef u_char DB_PTR_HC[4]; |
|
|
250 #define DB_HPTR_CP(x,v) {u_int32_t _v = DB_PTR_CP(v); \ |
|
|
251 (x)[0] = _v>>24; (x)[1] = _v>>16; (x)[2] = _v>>8; (x)[3] = _v;} |
|
|
252 #define DB_HPTR_EX(x) DB_PTR_EX(((x)[0]<<24) + ((x)[1]<<16) \ |
|
|
253 + ((x)[2]<<8) + (x)[3]) |
|
|
254 |
|
|
255 |
|
|
256 /* shape of the magic string that starts a database */ |
|
|
257 typedef char DB_VERSION_BUF[64]; |
|
|
258 typedef struct { |
|
|
259 DB_VERSION_BUF version; /* see DB_VERSION_STR */ |
|
|
260 DB_PTR db_csize; /* size of database contents in bytes */ |
|
|
261 u_int32_t pagesize; /* size of 1 DB buffer */ |
|
|
262 DB_SN sn; /* creation or expiration serial # */ |
|
|
263 time_t cleared; /* when created */ |
|
|
264 time_t cleaned; /* real instead of repair cleaning */ |
|
|
265 time_t cleaned_cron; /* cleaned by cron */ |
|
|
266 DB_SPAM_EX_TS ex_spam; /* recent expiration timestamps */ |
|
|
267 DB_SPAM_EX_TS ex_all; /* recent expiration timestamps */ |
|
|
268 DB_EX_SECS ex_secs; /* recent expiration thresholds */ |
|
|
269 DB_NOKEEP_CKS nokeep_cks; /* ignore these checksums */ |
|
|
270 u_int flags; |
|
|
271 # define DB_PARM_FG_GREY 0x01 /* greylist database */ |
|
|
272 # define DB_PARM_FG_CLEARED 0x02 /* new file */ |
|
|
273 # define DB_PARM_EXP_SET 0x04 /* have explicit expiration durations */ |
|
|
274 DB_PTR old_db_csize; /* size at last cleaning */ |
|
|
275 DB_PTR db_added; /* bytes previously added to database */ |
|
|
276 DB_HADDR hash_used; /* recent of entries used */ |
|
|
277 DB_HADDR old_hash_used; /* entries used at last cleaning */ |
|
|
278 DB_HADDR hash_added; /* entries added */ |
|
|
279 time_t rate_secs; /* denominator of rates */ |
|
|
280 # define DB_MIN_RATE_SECS (12*60*60) |
|
|
281 # define DB_MAX_RATE_SECS (14*24*60*60) |
|
|
282 time_t last_rate_sec; |
|
|
283 DB_HADDR old_kept_cks; /* reported checksums at cleaning */ |
|
|
284 } DB_PARMS; |
|
|
285 typedef union { |
|
|
286 DB_PARMS p; |
|
|
287 char c[256*3]; |
|
|
288 } DB_HDR; |
|
|
289 |
|
|
290 #ifdef DB_VERSION4_STR |
|
|
291 typedef struct { |
|
|
292 DB_VERSION_BUF version; |
|
|
293 DB_PTR db_csize; |
|
|
294 u_int32_t pagesize; |
|
|
295 DB_SN sn; |
|
|
296 time_t cleared; |
|
|
297 time_t cleaned; |
|
|
298 time_t cleaned_cron; |
|
|
299 DB_SPAM_EX_TS ex_spam; |
|
|
300 DB_EX_SECS ex_secs; |
|
|
301 DB_NOKEEP_CKS nokeep_cks; |
|
|
302 u_int flags; |
|
|
303 DB_PTR old_db_csize; |
|
|
304 DB_PTR db_added; |
|
|
305 DB_HADDR hash_used; |
|
|
306 DB_HADDR old_hash_used; |
|
|
307 DB_HADDR hash_added; |
|
|
308 time_t rate_secs; |
|
|
309 time_t last_rate_sec; |
|
|
310 DB_HADDR old_kept_cks; |
|
|
311 } DB_V4_PARMS; |
|
|
312 #endif |
|
|
313 #ifdef DB_VERSION3_STR |
|
|
314 typedef struct { |
|
|
315 DB_VERSION_BUF version; |
|
|
316 DB_PTR db_csize; |
|
|
317 u_int32_t pagesize; |
|
|
318 DB_SN sn; |
|
|
319 DB_SPAM_EX_TS ex_spam; |
|
|
320 DB_EX_SECS ex_secs; |
|
|
321 DB_NOKEEP_CKS nokeep_cks; |
|
|
322 DCC_TGTS unused[DCC_DIM_CKS]; |
|
|
323 u_int flags; |
|
|
324 # define DB_PARM_V3_FG_GREY 0x01 |
|
|
325 # define DB_PARM_V3_FG_SELF_CLEAN 0x02 |
|
|
326 # define DB_PARM_V3_FG_SELF_CLEAN2 0x04 |
|
|
327 # define DB_PARM_V3_FG_CLEARED 0x08 |
|
|
328 DB_PTR old_db_csize; |
|
|
329 DB_PTR db_added; |
|
|
330 DB_HADDR hash_used; |
|
|
331 DB_HADDR old_hash_used; |
|
|
332 DB_HADDR hash_added; |
|
|
333 time_t rate_secs; |
|
|
334 time_t last_rate_sec; |
|
|
335 DB_HADDR old_kept_cks; |
|
|
336 } DB_V3_PARMS; |
|
|
337 #endif |
|
|
338 |
|
|
339 /* shape of a database hash table entry */ |
|
|
340 typedef struct { |
|
|
341 DB_HADDR_C fwd, bak; /* hash collision chain */ |
|
|
342 u_char hv_type[2]; /* checksum type + some hash bits */ |
|
|
343 # define HE_TYPE(e) ((DCC_CK_TYPES)((e)->hv_type[0] & 0xf)) |
|
|
344 # define HE_IS_FREE(e) ((e)->hv_type[0] == 0) |
|
|
345 # define HE_MERGE(e,t,s) ((e)->hv_type[0] = ((((s)[0])<<4)+t), \ |
|
|
346 (e)->hv_type[1] = (s)[1]) |
|
|
347 # define HE_CMP(e,t,s) ((e)->hv_type[1] == (s)[1] \ |
|
|
348 && (e)->hv_type[0] ==(u_char)((((s)[0])<<4)+t)) |
|
|
349 |
|
|
350 DB_PTR_HC rcd; /* record for this hash table entry */ |
|
|
351 } HASH_ENTRY; |
|
|
352 |
|
|
353 |
|
|
354 |
|
|
355 typedef union { |
|
|
356 HASH_ENTRY h[8]; /* this must be larger than following */ |
|
|
357 struct { |
|
|
358 char magic[16]; |
|
|
359 u_int flags; |
|
|
360 # define HASH_CTL_FG_CLEAN 0x01 /* consistent */ |
|
|
361 # define HASH_CTL_FG_NOSYNC 0x02 /* pushed to stable storage */ |
|
|
362 DB_HADDR free_fwd; /* hash table internal free list */ |
|
|
363 DB_HADDR free_bak; |
|
|
364 # define FREE_HADDR_END 1 |
|
|
365 DB_HADDR len; /* size of file in entries */ |
|
|
366 DB_HADDR used; /* entries actually used */ |
|
|
367 DB_HADDR divisor; /* hash modulus */ |
|
|
368 DB_PTR db_csize; /* size of the database file */ |
|
|
369 time_t synced; |
|
|
370 } s; |
|
|
371 } HASH_CTL; |
|
|
372 |
|
|
373 |
|
|
374 #define DB_HADDR_BASE ((DB_HADDR)((sizeof(HASH_CTL)+sizeof(HASH_ENTRY)-1) \ |
|
|
375 / sizeof(HASH_ENTRY))) |
|
|
376 #define HADDR2LEN(l) ((int)((l)-DB_HADDR_BASE)) /* offset to length */ |
|
|
377 |
|
|
378 |
|
|
379 /* control a block of mapped memory */ |
|
|
380 typedef u_int16_t DB_PG_NUM; |
|
|
381 typedef u_int32_t DB_PG_OFF; |
|
|
382 typedef enum { |
|
|
383 DB_BUF_TYPE_FREE = 0, |
|
|
384 DB_BUF_TYPE_HASH, |
|
|
385 DB_BUF_TYPE_DB |
|
|
386 } DB_BUF_TYPE; |
|
|
387 #ifdef HAVE_64BIT_LONG |
|
|
388 typedef u_int64_t DB_BUF_FM; |
|
|
389 #else |
|
|
390 typedef u_int32_t DB_BUF_FM; |
|
|
391 #endif |
|
|
392 #define DB_BUF_NUM_PARTS (8*ISZ(DB_BUF_FM)) |
|
|
393 #define PART2BIT(part) (((DB_BUF_FM)1) << (part)) |
|
|
394 typedef struct db_buf { |
|
|
395 struct db_buf *fwd, *bak, **hash; |
|
|
396 struct db_buf *older, *newer; |
|
|
397 union { |
|
|
398 void *v; |
|
|
399 HASH_ENTRY *h; |
|
|
400 char *c; |
|
|
401 } buf; |
|
|
402 DB_PG_NUM pg_num; |
|
|
403 int lock_cnt; |
|
|
404 DB_BUF_TYPE buf_type; |
|
|
405 DB_BUF_FM flush; |
|
|
406 DB_BUF_FM flush_urgent; |
|
|
407 struct { |
|
|
408 char *lo, *hi; |
|
|
409 } ranges[DB_BUF_NUM_PARTS]; |
|
|
410 u_char flags; |
|
|
411 # define DB_BUF_FG_USE_WRITE 0x01 /* use write() */ |
|
|
412 # define DB_BUF_FG_EXTENSION 0x02 /* new page in file */ |
|
|
413 } DB_BUF; |
|
|
414 |
|
|
415 /* context for searching for or adding a record */ |
|
|
416 typedef struct { |
|
|
417 union { /* pointer to data in buffer */ |
|
|
418 void *v; |
|
|
419 HASH_ENTRY *h; |
|
|
420 char *c; |
|
|
421 DB_RCD *r; |
|
|
422 DB_PARMS *parms; |
|
|
423 HASH_CTL *vals; |
|
|
424 } d; |
|
|
425 union { /* database address */ |
|
|
426 DB_HADDR haddr; |
|
|
427 DB_PTR rptr; |
|
|
428 } s; |
|
|
429 DB_BUF *b; |
|
|
430 } DB_STATE; |
|
|
431 |
|
|
432 /* see db_close() before changing this */ |
|
|
433 typedef struct { |
|
|
434 DB_STATE rcd; /* must be first */ |
|
|
435 DB_STATE rcd2; |
|
|
436 DB_STATE sumrcd; |
|
|
437 DB_STATE hash; |
|
|
438 DB_STATE free; |
|
|
439 DB_STATE tmp; |
|
|
440 DB_STATE db_parms; |
|
|
441 DB_STATE hash_ctl; /* hash control info; must be last */ |
|
|
442 } DB_STATES; |
|
|
443 extern DB_STATES db_sts; |
|
|
444 |
|
|
445 extern int db_failed_line; |
|
|
446 extern const char *db_failed_file; |
|
|
447 #define DB_ERROR_MSG(s) db_error_msg(__LINE__,__FILE__, "%s", s) |
|
|
448 #define DB_ERROR_MSG2(s1,s2) db_error_msg(__LINE__,__FILE__, "%s: %s", s1,s2) |
|
|
449 |
|
|
450 extern struct timeval db_time; |
|
|
451 #define DB_IS_TIME(tgt,lim) DCC_IS_TIME(db_time.tv_sec,tgt,lim) |
|
|
452 #define DB_ADJ_TIMER(tgt,lim,new) DCC_ADJ_TIMER(db_time.tv_sec,tgt,lim,new) |
|
|
453 |
|
|
454 extern u_char db_minimum_map; /* this is dccd & dbclean is running */ |
|
|
455 extern int db_fd, db_hash_fd; |
|
|
456 extern DCC_PATH db_nm, db_hash_nm; |
|
|
457 extern struct timeval db_locked; /* 0 or when database was locked */ |
|
|
458 extern int db_debug; |
|
|
459 extern DB_SN db_sn; |
|
|
460 |
|
|
461 extern DB_HOFF db_hash_fsize; /* size of hash table file */ |
|
|
462 extern DB_HADDR db_hash_len; /* # of hash table entries */ |
|
|
463 extern DB_HADDR db_hash_divisor; /* modulus */ |
|
|
464 extern DB_HADDR db_hash_used; /* # of hash table entries in use */ |
|
|
465 extern u_int db_hash_page_len; /* # of HASH_ENTRY's per buffer */ |
|
|
466 extern DB_HADDR db_max_hash_entries; /* max size of hash table */ |
|
|
467 extern DB_PTR db_fsize; /* size of database file */ |
|
|
468 extern DB_PTR db_csize; /* size of database contents in bytes */ |
|
|
469 extern const DB_VERSION_BUF db_version_buf; |
|
|
470 extern DB_PARMS db_parms; |
|
|
471 extern DCC_TGTS db_tholds[DCC_DIM_CKS]; |
|
|
472 extern u_int db_pagesize; /* size of 1 DB buffer */ |
|
|
473 extern u_int db_page_max; /* only padding after this */ |
|
|
474 extern char db_window_size_str[]; /* size of mmap() window */ |
|
|
475 |
|
|
476 typedef struct { |
|
|
477 u_int db_mmaps; |
|
|
478 u_int hash_mmaps; |
|
|
479 u_int adds; /* reports added */ |
|
|
480 } DB_STATS; |
|
|
481 extern DB_STATS db_stats; |
|
|
482 |
|
|
483 |
|
|
484 /* If the two files were smaller than the typical mmap() limit of a fraction |
|
|
485 * of a GByte, they could be mmap()'ed directly. They are often too large. |
|
|
486 * |
|
|
487 * Use a modest pool of buffers to map the DB hash table and the database |
|
|
488 * itself. |
|
|
489 * Each access to the files could be with a single, common buffer, |
|
|
490 * but that would involve many more mmap() system calls. |
|
|
491 * Most of the DB hash table is expected to fit in the application's memory. |
|
|
492 * |
|
|
493 * Use the same modest pool of buffers to map the database itself. |
|
|
494 * References to the database have a lot of locality, so the commonly used |
|
|
495 * checksums and counts should remain in memory. |
|
|
496 * |
|
|
497 * Common operating system limits on the number of mapped segments are |
|
|
498 * below 256 and so that is a bound on DB_BUF_MAX */ |
|
|
499 #define DB_BUF_MAX 128 /* maximum # of buffers */ |
|
|
500 #define DB_BUF_PARTS_MAX (DB_BUF_MAX*DB_BUF_NUM_PARTS) |
|
|
501 |
|
|
502 /* enough buffers so max simultaneous pointers can be satisfied */ |
|
|
503 #define DB_BUF_MIN (sizeof(DB_STATES)/sizeof(DB_STATE) + 2) |
|
|
504 |
|
|
505 extern int db_buf_total; /* total # of db buffers */ |
|
|
506 extern DB_PTR db_max_rss; /* maximum db resident set size */ |
|
|
507 extern DB_PTR db_max_byte; |
|
|
508 |
|
|
509 |
|
|
510 extern time_t db_need_flush_secs; |
|
|
511 #define DB_NEED_FLUSH_SECS 5 |
|
|
512 #define DB_STALE_SECS (30*60) /* limit on buffer staleness */ |
|
|
513 #define DB_FLUSHES (DB_STALE_SECS / DB_NEED_FLUSH_SECS) |
|
|
514 #define DB_PARTS_PER_FLUSH ((DB_BUF_PARTS_MAX + DB_FLUSHES-1) / DB_FLUSHES) |
|
|
515 #define DB_URGENT_NEED_FLUSH_SECS 120 |
|
|
516 |
|
|
517 |
|
|
518 /* fix configure script if this changes */ |
|
|
519 #define DB_MIN_MIN_MBYTE 32 |
|
|
520 #define DB_DEF_MIN_MBYTE 64 /* a reasonable tiny default */ |
|
|
521 #define DB_PAD_MBYTE 128 /* RAM for rate limiting blocks etc */ |
|
|
522 #define DB_PAD_BYTE (DB_PAD_MBYTE*1024*1024) |
|
|
523 #define DB_MAX_2G_MBYTE (2048-DB_PAD_MBYTE) /* <2 GByte on 32 bit machines */ |
|
|
524 /* the database cannot exceed 48 GBytes because of DB_PTR_CP */ |
|
|
525 #define MAX_MAX_DB_MBYTE (48*1024) |
|
|
526 /* fix INSTALL.html if those change */ |
|
|
527 |
|
|
528 |
|
|
529 /* srvr/db.c */ |
|
|
530 extern void db_failure(int, const char *, int, DCC_EMSG, |
|
|
531 const char *, ...) PATTRIB(5,6); |
|
|
532 extern void db_error_msg(int, const char *, const char *, ...) PATTRIB(3,4); |
|
|
533 extern void db_set_flush(DB_STATE *, u_char, u_int); |
|
|
534 #define SET_FLUSH_RCD(st,u) db_set_flush(st,u, DB_RCD_LEN((st)->d.r)) |
|
|
535 #define SET_FLUSH_RCD_HDR(st,u) db_set_flush(st,u, DB_RCD_HDR_LEN) |
|
|
536 #define SET_FLUSH_HE(st) db_set_flush(st, 0, sizeof(HASH_ENTRY)) |
|
|
537 #define SET_FLUSH_HCTL(u) db_set_flush(&db_sts.hash_ctl,u, sizeof(HASH_CTL)) |
|
|
538 extern void rel_db_states(void); |
|
|
539 extern u_char db_unload(DCC_EMSG, u_char); |
|
|
540 extern u_char db_flush_db(DCC_EMSG); |
|
|
541 extern u_char make_clean(u_char); |
|
|
542 extern u_char db_close(int); |
|
|
543 extern void db_stop(void); |
|
|
544 extern u_char lock_dbclean(DCC_EMSG, const char *); |
|
|
545 extern void unlock_dbclean(void); |
|
|
546 extern u_int db_get_pagesize(u_int, u_int); |
|
|
547 extern u_char db_buf_init(u_int, u_int); |
|
|
548 typedef u_char DB_OPEN_MODES; |
|
|
549 # define DB_OPEN_RDONLY 0x01 |
|
|
550 # define DB_OPEN_LOCK_WAIT 0x02 /* wait to get lock */ |
|
|
551 # define DB_OPEN_LOCK_NOWAIT 0x04 /* get lock but don't wait */ |
|
|
552 # define DB_OPEN_MMAP_WRITE 0x08 /* use write() instead of mmap() */ |
|
|
553 # define DB_OPEN_MMAP_WRITE_NOSYNC 0x10 /* if no mmap(NOSYNC) */ |
|
|
554 extern u_char db_open(DCC_EMSG, int, const char *, DB_HADDR, DB_OPEN_MODES); |
|
|
555 extern u_char db_flush_parms(DCC_EMSG ); |
|
|
556 #define DB_IS_LOCKED() (db_locked.tv_sec != 0) |
|
|
557 extern int db_lock(void); |
|
|
558 extern u_char db_unlock(void); |
|
|
559 extern void db_flush_needed(void); |
|
|
560 extern DCC_TGTS db_sum_ck(DCC_TGTS, DCC_TGTS, DCC_CK_TYPES); |
|
|
561 extern const char *db_ptr2str(DB_PTR); |
|
|
562 extern const char *size2str(char *, u_int, double, u_char); |
|
|
563 extern double db_add_rate(const DB_PARMS *, u_char); |
|
|
564 extern DB_NOKEEP_CKS def_nokeep_cks(void); |
|
|
565 extern void set_db_tholds(DB_NOKEEP_CKS); |
|
|
566 extern u_char db_map_rcd(DCC_EMSG, DB_STATE *, DB_PTR, int *); |
|
|
567 extern DB_RCD_CK *db_find_ck(DCC_EMSG, DB_RCD *, DB_PTR, DCC_CK_TYPES); |
|
|
568 extern DB_RCD_CK *db_map_rcd_ck(DCC_EMSG, DB_STATE *, DB_PTR, DCC_CK_TYPES); |
|
|
569 extern DB_HADDR get_db_hash_divisor(DB_HADDR); |
|
|
570 extern DB_HADDR db_hash(DCC_CK_TYPES, const DCC_SUM); |
|
|
571 typedef enum { |
|
|
572 DB_FOUND_SYSERR=0, /* fatal error */ |
|
|
573 DB_FOUND_LATER, /* out of specified hash table range */ |
|
|
574 DB_FOUND_IT, |
|
|
575 DB_FOUND_EMPTY, /* home slot empty */ |
|
|
576 DB_FOUND_CHAIN, /* not in chain--have last entry */ |
|
|
577 DB_FOUND_INTRUDER /* intruder in home slot */ |
|
|
578 } DB_FOUND; |
|
|
579 extern DB_FOUND db_lookup(DCC_EMSG, DCC_CK_TYPES, const DCC_SUM, |
|
|
580 DB_HADDR, DB_HADDR, DB_STATE *, |
|
|
581 DB_STATE *, DB_RCD_CK **); |
|
|
582 extern u_char db_link_rcd(DCC_EMSG, DB_HADDR, DB_HADDR); |
|
|
583 extern DB_PTR db_add_rcd(DCC_EMSG, const DB_RCD *); |
|
|
584 |
|
|
585 #endif /* DB_H */ |
