4 * Overview storage using BerkeleyDB 2.x/3.x/4.x
6 * 2004-02-17 : Need to track search cursors, since it's possible that
7 * ovdb_closesearch does not get called. We now close
8 * any cursors still open in ovdb_close, or they'd be in
9 * the database indefinitely causing deadlocks.
10 * 2002-08-13 : Change BOOL to bool, remove portability to < 2.4.
11 * 2002-08-11 : Cleaned up use of sprintf and fixed a bunch of warnings.
12 * 2002-02-28 : Update getartinfo for the overview API change in 2.4. This
13 * breaks compatibility with INN 2.3.x....
14 * 2000-12-12 : Add support for BerkeleyDB DB_SYSTEM_MEM option, controlled
15 * : by ovdb.conf 'useshm' and 'shmkey'
16 * 2000-11-27 : Update for DB 3.2.x compatibility
17 * 2000-11-13 : New 'readserver' feature
18 * 2000-10-10 : ovdb_search now closes the cursor right after the last
20 * 2000-10-05 : artnum member of struct datakey changed from ARTNUM to u_int32_t.
21 * OS's where sizeof(long)==8 will have to rebuild their databases
23 * 2000-10-05 : from Dan Riley: struct datakey needs to be zero'd, for
24 * 64-bit OSs where the struct has internal padding bytes.
25 * 2000-09-29 : ovdb_expiregroup can now fix incorrect counts; use new
26 * inn/version.h so can have same ovdb.c for 2.3.0, 2.3.1, and 2.4
27 * 2000-09-28 : low mark in ovdb_expiregroup still wasn't right
28 * 2000-09-27 : Further improvements to ovdb_expiregroup: restructured the
29 * loop; now updates groupinfo as it goes along rather than
30 * counting records at the end, which prevents a possible
32 * 2000-09-19 : *lo wasn't being set in ovdb_expiregroup
33 * 2000-09-15 : added ovdb_check_user(); tweaked some error msgs; fixed an
34 * improper use of RENEW
35 * 2000-08-28: New major release: version 2.00 (beta)
36 * + "groupsbyname" and "groupstats" databases replaced with "groupinfo".
37 * + ovdb_recover, ovdb_upgrade, and dbprocs are now deprecated; their
38 * functionality is now in ovdb_init and ovdb_monitor.
39 * + ovdb_init can upgrade a database from the old version of ovdb to
40 * work with this version.
41 * + Rewrote ovdb_expiregroup(); it can now re-write OV data rather
42 * than simply deleting old keys (which can leave 'holes' that result
43 * in inefficient disk-space use).
44 * + Add "nocompact" to ovdb.conf, which controls whether ovdb_expiregroup()
46 * + No longer needs the BerkeleyDB tools db_archive, db_checkpoint, and
47 * db_deadlock. That functionality is now in ovdb_monitor.
48 * + ovdb_open() won't succeed if ovdb_monitor is not running. This will
49 * prevent the problems that happen if the database is not regularly
50 * checkpointed and deadlock-tested.
51 * + Internal group IDs (32-bit ints) are now reused.
52 * + Add "maxlocks" to ovdb.conf, which will set the DB lk_max parameter.
53 * + Pull "test" code out into ovdb_stat. ovdb_stat will also provide
54 * functionality similar to the BerkeleyDB "db_stat" command.
55 * + Update docs: write man pages for the new ovdb_* commands; update
58 * 2000-07-11 : fix possible alignment problem; add test code
59 * 2000-07-07 : bugfix: timestamp handling
60 * 2000-06-10 : Modified groupnum() interface; fix ovdb_add() to return false
61 * for certain groupnum() errors
62 * 2000-06-08 : Added BerkeleyDB 3.1.x compatibility
63 * 2000-04-09 : Tweak some default parameters; store aliased group info
64 * 2000-03-29 : Add DB_RMW flag to the 'get' of get-modify-put sequences
65 * 2000-02-17 : Update expire behavior to be consistent with current
67 * 2000-01-13 : Fix to make compatible with unmodified nnrpd/article.c
68 * 2000-01-04 : Added data versioning
69 * 1999-12-20 : Added BerkeleyDB 3.x compatibility
70 * 1999-12-06 : First Release -- H. Kehoe <hakehoe@avalon.net>
75 #include "portable/socket.h"
76 #include "portable/time.h"
84 #ifdef HAVE_SYS_SELECT_H
85 # include <sys/select.h>
90 #include "inn/innconf.h"
91 #include "inn/messages.h"
97 #include "ovinterface.h"
99 #include "ovdb-private.h"
101 #ifdef HAVE_UNIX_DOMAIN_SOCKETS
105 #ifndef USE_BERKELEY_DB
107 /* Provide stub functions if we don't have db */
109 bool ovdb_open(int mode UNUSED)
111 syslog(L_FATAL, "OVDB: ovdb support not enabled");
115 bool ovdb_groupstats(char *group UNUSED, int *lo UNUSED, int *hi UNUSED, int *count UNUSED, int *flag UNUSED)
118 bool ovdb_groupadd(char *group UNUSED, ARTNUM lo UNUSED, ARTNUM hi UNUSED, char *flag UNUSED)
121 bool ovdb_groupdel(char *group UNUSED)
124 bool ovdb_add(char *group UNUSED, ARTNUM artnum UNUSED, TOKEN token UNUSED, char *data UNUSED, int len UNUSED, time_t arrived UNUSED, time_t expires UNUSED)
127 bool ovdb_cancel(TOKEN token UNUSED)
130 void *ovdb_opensearch(char *group UNUSED, int low UNUSED, int high UNUSED)
133 bool ovdb_search(void *handle UNUSED, ARTNUM *artnum UNUSED, char **data UNUSED, int *len UNUSED, TOKEN *token UNUSED, time_t *arrived UNUSED)
136 void ovdb_closesearch(void *handle UNUSED) { }
138 bool ovdb_getartinfo(char *group UNUSED, ARTNUM artnum UNUSED, TOKEN *token UNUSED)
141 bool ovdb_expiregroup(char *group UNUSED, int *lo UNUSED, struct history *h UNUSED)
144 bool ovdb_ctl(OVCTLTYPE type UNUSED, void *val UNUSED)
147 void ovdb_close(void) { }
149 #else /* USE_BERKELEY_DB */
151 #define EXPIREGROUP_TXN_SIZE 100
152 #define DELETE_TXN_SIZE 500
154 struct ovdb_conf ovdb_conf;
155 DB_ENV *OVDBenv = NULL;
156 int ovdb_errmode = OVDB_ERR_SYSLOG;
159 static bool Cutofflow;
160 static DB **dbs = NULL;
161 static int oneatatime = 0;
162 static int current_db = -1;
163 static time_t eo_start = 0;
164 static int clientmode = 0;
166 static DB *groupinfo = NULL;
167 static DB *groupaliases = NULL;
169 #define OVDBtxn_nosync 1
170 #define OVDBnumdbfiles 2
171 #define OVDBpagesize 3
172 #define OVDBcachesize 4
174 #define OVDBmaxlocks 6
175 #define OVDBnocompact 7
176 #define OVDBreadserver 8
177 #define OVDBnumrsprocs 9
178 #define OVDBmaxrsconn 10
179 #define OVDBuseshm 11
180 #define OVDBshmkey 12
182 static CONFTOKEN toks[] = {
183 { OVDBtxn_nosync, "txn_nosync" },
184 { OVDBnumdbfiles, "numdbfiles" },
185 { OVDBpagesize, "pagesize" },
186 { OVDBcachesize, "cachesize" },
187 { OVDBminkey, "minkey" },
188 { OVDBmaxlocks, "maxlocks" },
189 { OVDBnocompact, "nocompact" },
190 { OVDBreadserver, "readserver" },
191 { OVDBnumrsprocs, "numrsprocs" },
192 { OVDBmaxrsconn, "maxrsconn" },
193 { OVDBuseshm, "useshm" },
194 { OVDBshmkey, "shmkey" },
198 #define _PATH_OVDBCONF "ovdb.conf"
200 /*********** readserver functions ***********/
202 static int clientfd = -1;
204 /* read client send and recieve functions. */
207 csend(void *data, int n)
213 status = xwrite(clientfd, data, n);
215 syswarn("OVDB: rc: cant write");
219 static int crecv(void *data, int n)
227 r = read(clientfd, (char *)data + p, n - p);
229 if(r < 0 && errno == EINTR)
231 syslog(LOG_ERR, "OVDB: rc: cant read: %m");
240 /* Attempt to connect to the readserver. If anything fails, we
241 return -1 so that ovdb_open can open the database directly. */
243 static int client_connect()
248 #ifdef HAVE_UNIX_DOMAIN_SOCKETS
249 struct sockaddr_un sa;
251 struct sockaddr_in sa;
253 char banner[sizeof(OVDB_SERVER_BANNER)];
255 struct timeval timeout;
257 #ifdef HAVE_UNIX_DOMAIN_SOCKETS
258 clientfd = socket(AF_UNIX, SOCK_STREAM, 0);
260 clientfd = socket(AF_INET, SOCK_STREAM, 0);
263 syslog(LOG_ERR, "OVDB: rc: socket: %m");
267 #ifdef HAVE_UNIX_DOMAIN_SOCKETS
268 sa.sun_family = AF_UNIX;
269 path = concatpath(innconf->pathrun, OVDB_SERVER_SOCKET);
270 strlcpy(sa.sun_path, path, sizeof(sa.sun_path));
273 sa.sin_family = AF_INET;
275 sa.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
276 bind(clientfd, (struct sockaddr *) &sa, sizeof sa);
277 sa.sin_port = htons(OVDB_SERVER_PORT);
279 if((r = connect(clientfd, (struct sockaddr *) &sa, sizeof sa)) != 0) {
280 syslog(LOG_ERR, "OVDB: rc: cant connect to server: %m");
286 while(p < sizeof(OVDB_SERVER_BANNER)) {
288 FD_SET(clientfd, &fds);
292 r = select(clientfd+1, &fds, NULL, NULL, &timeout);
297 syslog(LOG_ERR, "OVDB: rc: select: %m");
303 syslog(LOG_ERR, "OVDB: rc: timeout waiting for server");
309 r = read(clientfd, banner + p, sizeof(OVDB_SERVER_BANNER) - p);
311 if(r < 0 && errno == EINTR)
313 syslog(LOG_ERR, "OVDB: rc: cant read: %m");
321 if(memcmp(banner, OVDB_SERVER_BANNER, sizeof(OVDB_SERVER_BANNER))) {
322 syslog(LOG_ERR, "OVDB: rc: unknown reply from server");
331 client_disconnect(void)
335 if (clientfd != -1) {
337 csend(&rs, sizeof(rs));
343 /*********** internal functions ***********/
345 #if DB_VERSION_MAJOR == 2
346 char *db_strerror(int err)
350 return "Recovery Needed";
352 return strerror(err);
355 #endif /* DB_VERSION_MAJOR == 2 */
358 static bool conf_bool_val(char *str, bool *value)
360 if(strcasecmp(str, "on") == 0
361 || strcasecmp(str, "true") == 0
362 || strcasecmp(str, "yes") == 0) {
366 if(strcasecmp(str, "off") == 0
367 || strcasecmp(str, "false") == 0
368 || strcasecmp(str, "no") == 0) {
375 static bool conf_long_val(char *str, long *value)
380 v = strtol(str, NULL, 10);
381 if(v == 0 && errno != 0) {
388 void read_ovdb_conf(void)
390 static int confread = 0;
402 ovdb_conf.home = innconf->pathoverview;
403 ovdb_conf.txn_nosync = 1;
404 ovdb_conf.numdbfiles = 32;
405 ovdb_conf.pagesize = 8192;
406 ovdb_conf.cachesize = 8000 * 1024;
407 ovdb_conf.minkey = 0;
408 ovdb_conf.maxlocks = 4000;
409 ovdb_conf.nocompact = 1;
410 ovdb_conf.readserver = 0;
411 ovdb_conf.numrsprocs = 5;
412 ovdb_conf.maxrsconn = 0;
413 ovdb_conf.useshm = 0;
414 ovdb_conf.shmkey = 6400;
416 path = concatpath(innconf->pathetc, _PATH_OVDBCONF);
421 while(!done && (tok = CONFgettoken(toks, f))) {
424 tok = CONFgettoken(0, f);
429 if(conf_bool_val(tok->name, &b)) {
430 ovdb_conf.txn_nosync = b;
434 tok = CONFgettoken(0, f);
439 if(conf_long_val(tok->name, &l) && l > 0) {
440 ovdb_conf.numdbfiles = l;
444 tok = CONFgettoken(0, f);
449 if(conf_long_val(tok->name, &l) && l > 0) {
450 ovdb_conf.pagesize = l;
454 tok = CONFgettoken(0, f);
459 if(conf_long_val(tok->name, &l) && l > 0) {
460 ovdb_conf.cachesize = l * 1024;
464 tok = CONFgettoken(0, f);
469 if(conf_long_val(tok->name, &l) && l > 1) {
470 ovdb_conf.minkey = l;
474 tok = CONFgettoken(0, f);
479 if(conf_long_val(tok->name, &l) && l > 0) {
480 ovdb_conf.maxlocks = l;
484 tok = CONFgettoken(0, f);
489 if(conf_long_val(tok->name, &l) && l >= 0) {
490 ovdb_conf.nocompact = l;
494 tok = CONFgettoken(0, f);
499 if(conf_bool_val(tok->name, &b)) {
500 ovdb_conf.readserver = b;
504 tok = CONFgettoken(0, f);
509 if(conf_long_val(tok->name, &l) && l > 0) {
510 ovdb_conf.numrsprocs = l;
514 tok = CONFgettoken(0, f);
519 if(conf_long_val(tok->name, &l) && l >= 0) {
520 ovdb_conf.maxrsconn = l;
524 tok = CONFgettoken(0, f);
529 if(conf_bool_val(tok->name, &b)) {
530 ovdb_conf.useshm = b;
534 tok = CONFgettoken(0, f);
539 if(conf_long_val(tok->name, &l) && l >= 0) {
540 ovdb_conf.shmkey = l;
548 /* If user did not specify minkey, choose one based on pagesize */
549 if(ovdb_conf.minkey == 0) {
550 ovdb_conf.minkey = ovdb_conf.pagesize / 2048 - 1;
551 if(ovdb_conf.minkey < 2)
552 ovdb_conf.minkey = 2;
559 /* Function that db will use to report errors */
560 #if DB_VERSION_MAJOR > 4 || (DB_VERSION_MAJOR == 4 && DB_VERSION_MINOR >= 3)
561 static void OVDBerror(const DB_ENV *dbenv UNUSED, const char *db_errpfx UNUSED, const char *buffer)
563 static void OVDBerror(const char *db_errpfx UNUSED, char *buffer)
566 switch(ovdb_errmode) {
567 case OVDB_ERR_SYSLOG:
568 syslog(L_ERROR, "OVDB: %s", buffer);
570 case OVDB_ERR_STDERR:
571 fprintf(stderr, "OVDB: %s\n", buffer);
576 static u_int32_t _db_flags = 0;
577 #if DB_VERSION_MAJOR == 2
578 static DB_INFO _dbinfo;
581 static int open_db_file(int which)
585 #if DB_VERSION_MAJOR > 4 || (DB_VERSION_MAJOR == 4 && DB_VERSION_MINOR >= 1)
589 if(dbs[which] != NULL)
592 snprintf(name, sizeof(name), "ov%05d", which);
594 #if DB_VERSION_MAJOR == 2
595 ret = db_open(name, DB_BTREE, _db_flags, 0666, OVDBenv, &_dbinfo,
602 ret = db_create(&(dbs[which]), OVDBenv, 0);
606 if(ovdb_conf.minkey > 0)
607 (dbs[which])->set_bt_minkey(dbs[which], ovdb_conf.minkey);
608 if(ovdb_conf.pagesize > 0)
609 (dbs[which])->set_pagesize(dbs[which], ovdb_conf.pagesize);
611 #if DB_VERSION_MAJOR > 4 || (DB_VERSION_MAJOR == 4 && DB_VERSION_MINOR >= 1)
612 TXN_START(t_open_db_file, tid);
613 ret = (dbs[which])->open(dbs[which], tid, name, NULL, DB_BTREE, _db_flags,
616 TXN_COMMIT(t_open_db_file, tid);
618 ret = (dbs[which])->open(dbs[which], name, NULL, DB_BTREE, _db_flags,
622 (dbs[which])->close(dbs[which], 0);
630 static void close_db_file(int which)
632 if(which == -1 || dbs[which] == NULL)
635 dbs[which]->close(dbs[which], 0);
639 static int which_db(char *group)
644 grouphash = Hash(group, strlen(group));
645 memcpy(&i, &grouphash, sizeof(i));
646 return i % ovdb_conf.numdbfiles;
649 static DB *get_db_bynum(int which)
652 if(which >= ovdb_conf.numdbfiles)
655 if(which != current_db && current_db != -1)
656 close_db_file(current_db);
658 ret = open_db_file(which);
660 syslog(L_ERROR, "OVDB: open_db_file failed: %s", db_strerror(ret));
668 int ovdb_getgroupinfo(char *group, struct groupinfo *gi, int ignoredeleted, DB_TXN *tid, int getflags)
673 if(group == NULL) /* just in case */
676 memset(&key, 0, sizeof key);
677 memset(&val, 0, sizeof val);
680 key.size = strlen(group);
682 val.ulen = sizeof(struct groupinfo);
683 val.flags = DB_DBT_USERMEM;
685 ret = groupinfo->get(groupinfo, tid, &key, &val, getflags);
689 if(val.size != sizeof(struct groupinfo)) {
690 syslog(L_ERROR, "OVDB: wrong size for %s groupinfo (%u)",
695 if(ignoredeleted && (gi->status & GROUPINFO_DELETED))
701 #define GROUPID_MAX_FREELIST 10240
702 #define GROUPID_MIN_FREELIST 100
704 /* allocate a new group ID and return in gno */
705 /* must be used in a transaction */
706 static int groupid_new(group_id_t *gno, DB_TXN *tid)
710 group_id_t newgno, *freelist, one;
712 memset(&key, 0, sizeof key);
713 memset(&val, 0, sizeof val);
715 key.data = (char *) "!groupid_freelist";
716 key.size = sizeof("!groupid_freelist");
718 ret = groupinfo->get(groupinfo, tid, &key, &val, DB_RMW);
720 if(ret == DB_NOTFOUND) {
721 val.size = sizeof(group_id_t);
729 if(val.size % sizeof(group_id_t)) {
730 syslog(L_ERROR, "OVDB: invalid size (%d) for !groupid_freelist",
735 n = val.size / sizeof(group_id_t);
736 freelist = xmalloc(n * sizeof(group_id_t));
737 memcpy(freelist, val.data, val.size);
738 if(n <= GROUPID_MIN_FREELIST ) {
739 newgno = freelist[n-1];
743 newgno = freelist[0];
744 val.data = &(freelist[1]);
745 val.size -= sizeof(group_id_t);
748 ret = groupinfo->put(groupinfo, tid, &key, &val, 0);
759 /* mark given group ID as "unused" */
760 /* must be used in a transaction */
761 static int groupid_free(group_id_t gno, DB_TXN *tid)
765 group_id_t *freelist;
767 memset(&key, 0, sizeof key);
768 memset(&val, 0, sizeof val);
770 key.data = (char *) "!groupid_freelist";
771 key.size = sizeof("!groupid_freelist");
773 ret = groupinfo->get(groupinfo, tid, &key, &val, DB_RMW);
778 if(val.size % sizeof(group_id_t)) {
779 syslog(L_ERROR, "OVDB: invalid size (%d) for !groupid_freelist",
784 n = val.size / sizeof(group_id_t);
785 if(n > GROUPID_MAX_FREELIST)
787 freelist = xmalloc((n + 1) * sizeof(group_id_t));
788 memcpy(freelist, val.data, val.size);
790 if(gno >= freelist[n-1]) { /* shouldn't happen */
794 for(i = 0; i < n-1; i++) {
795 if(gno == freelist[i]) { /* already on freelist */
801 freelist[n] = freelist[n-1];
804 val.size += sizeof(group_id_t);
806 ret = groupinfo->put(groupinfo, tid, &key, &val, 0);
812 /* Must be called outside of a transaction because it makes its own
814 static int delete_all_records(int whichdb, group_id_t gno)
824 memset(&key, 0, sizeof key);
825 memset(&val, 0, sizeof val);
826 memset(&dk, 0, sizeof dk);
828 db = get_db_bynum(whichdb);
836 TXN_START(t_del, tid);
838 /* get a cursor to traverse the ov records and delete them */
839 ret = db->cursor(db, tid, &dbcursor, 0);
845 TXN_RETRY(t_del, tid);
847 TXN_ABORT(t_del, tid);
848 syslog(L_ERROR, "OVDB: delete_all_records: db->cursor: %s", db_strerror(ret));
853 key.size = sizeof dk;
854 val.flags = DB_DBT_PARTIAL;
856 for(count = 0; count < DELETE_TXN_SIZE; count++) {
857 ret = dbcursor->c_get(dbcursor, &key, &val,
858 count ? DB_NEXT : DB_SET_RANGE);
864 dbcursor->c_close(dbcursor);
865 TXN_COMMIT(t_del, tid);
868 dbcursor->c_close(dbcursor);
869 TXN_RETRY(t_del, tid);
871 warn("OVDB: delete_all_records: DBcursor->c_get: %s",
873 dbcursor->c_close(dbcursor);
874 TXN_ABORT(t_del, tid);
878 if(key.size == sizeof dk
879 && memcmp(key.data, &gno, sizeof gno)) {
883 ret = dbcursor->c_del(dbcursor, 0);
889 dbcursor->c_close(dbcursor);
890 TXN_RETRY(t_del, tid);
892 warn("OVDB: delete_all_records: DBcursor->c_del: %s",
894 dbcursor->c_close(dbcursor);
895 TXN_ABORT(t_del, tid);
899 dbcursor->c_close(dbcursor);
900 TXN_COMMIT(t_del, tid);
901 if(count < DELETE_TXN_SIZE) {
908 /* Make a temporary groupinfo key using the given db number and group ID.
909 Must be called in a transaction */
911 mk_temp_groupinfo(int whichdb, group_id_t gno, DB_TXN *tid)
913 char keystr[1 + sizeof gno];
918 memset(&key, 0, sizeof key);
919 memset(&val, 0, sizeof val);
920 memset(&gi, 0, sizeof gi);
923 memcpy(keystr + 1, &gno, sizeof gno);
925 gi.current_db = whichdb;
926 gi.current_gid = gno;
927 gi.status = GROUPINFO_DELETED;
930 key.size = sizeof keystr;
932 val.size = sizeof gi;
934 ret = groupinfo->put(groupinfo, tid, &key, &val, 0);
939 syslog(L_ERROR, "OVDB: mk_temp_groupinfo: groupinfo->put: %s", db_strerror(ret));
947 /* Delete a temporary groupinfo key created by mk_temp_groupid, then
949 Must NOT be called within a transaction. */
951 rm_temp_groupinfo(group_id_t gno)
953 char keystr[1 + sizeof gno];
958 memset(&key, 0, sizeof key);
961 memcpy(keystr + 1, &gno, sizeof gno);
964 key.size = sizeof keystr;
966 TXN_START(t_tmp, tid);
968 ret = groupinfo->del(groupinfo, tid, &key, 0);
974 TXN_RETRY(t_tmp, tid);
976 TXN_ABORT(t_tmp, tid);
977 syslog(L_ERROR, "OVDB: rm_temp_groupinfo: groupinfo->del: %s", db_strerror(ret));
981 switch(groupid_free(gno, tid)) {
985 TXN_RETRY(t_tmp, tid);
987 TXN_ABORT(t_tmp, tid);
988 syslog(L_ERROR, "OVDB: rm_temp_groupinfo: groupid_free: %s", db_strerror(ret));
992 TXN_COMMIT(t_tmp, tid);
996 /* This function deletes overview records for deleted or forgotton groups */
997 /* argument: 0 = process deleted groups 1 = process forgotton groups */
998 static bool delete_old_stuff(int forgotton)
1003 struct groupinfo gi;
1004 char **dellist = NULL;
1005 size_t *dellistsz = NULL;
1006 int listlen, listcount;
1009 TXN_START(t_dellist, tid);
1010 if (dellist != NULL) {
1011 for (i = 0; i < listcount; ++i)
1015 if (dellistsz != NULL)
1019 dellist = xmalloc(listlen * sizeof(char *));
1020 dellistsz = xmalloc(listlen * sizeof(size_t));
1022 memset(&key, 0, sizeof key);
1023 memset(&val, 0, sizeof val);
1026 val.ulen = val.dlen = sizeof gi;
1027 val.flags = DB_DBT_USERMEM | DB_DBT_PARTIAL;
1029 /* get a cursor to traverse all of the groupinfo records */
1030 ret = groupinfo->cursor(groupinfo, tid, &cursor, 0);
1032 syslog(L_ERROR, "OVDB: delete_old_stuff: groupinfo->cursor: %s", db_strerror(ret));
1038 while((ret = cursor->c_get(cursor, &key, &val, DB_NEXT)) == 0) {
1039 if(key.size == sizeof("!groupid_freelist") &&
1040 !strcmp("!groupid_freelist", key.data))
1042 if(val.size != sizeof(struct groupinfo)) {
1043 syslog(L_ERROR, "OVDB: delete_old_stuff: wrong size for groupinfo record");
1046 if((!forgotton && (gi.status & GROUPINFO_DELETED))
1047 || (forgotton && (gi.expired < eo_start))) {
1048 dellist[listcount] = xmalloc(key.size);
1049 memcpy(dellist[listcount], key.data, key.size);
1050 dellistsz[listcount] = key.size;
1052 if(listcount >= listlen) {
1054 dellist = xrealloc(dellist, listlen * sizeof(char *));
1055 dellistsz = xrealloc(dellistsz, listlen * sizeof(size_t));
1059 cursor->c_close(cursor);
1063 TXN_COMMIT(t_dellist, tid);
1066 TXN_RETRY(t_dellist, tid);
1068 TXN_ABORT(t_dellist, tid);
1069 syslog(L_ERROR, "OVDB: delete_old_stuff: cursor->c_get: %s", db_strerror(ret));
1070 for (i = 0; i < listcount; ++i)
1077 for(i = 0; i < listcount; i++) {
1078 TXN_START(t_dos, tid);
1080 /* Can't use ovdb_getgroupinfo here */
1081 key.data = dellist[i];
1082 key.size = dellistsz[i];
1084 val.ulen = sizeof(struct groupinfo);
1085 val.flags = DB_DBT_USERMEM;
1087 ret = groupinfo->get(groupinfo, tid, &key, &val, 0);
1094 TXN_RETRY(t_dos, tid);
1096 TXN_ABORT(t_dos, tid);
1099 syslog(L_ERROR, "OVDB: delete_old_stuff: groupinfo->get: %s\n", db_strerror(ret));
1100 TXN_ABORT(t_dos, tid);
1103 if(val.size != sizeof(struct groupinfo)) {
1104 TXN_ABORT(t_dos, tid);
1108 /* This if() is true if this ISN'T a key created by mk_temp_groupinfo */
1109 if(*((char *)key.data) != 0 || key.size != 1 + sizeof(group_id_t)) {
1111 switch(mk_temp_groupinfo(gi.current_db, gi.current_gid, tid)) {
1115 TXN_RETRY(t_dos, tid);
1117 TXN_ABORT(t_dos, tid);
1120 if(gi.status & GROUPINFO_MOVING) {
1121 switch(mk_temp_groupinfo(gi.new_db, gi.new_gid, tid)) {
1125 TXN_RETRY(t_dos, tid);
1127 TXN_ABORT(t_dos, tid);
1132 /* delete the corresponding groupaliases record (if exists) */
1133 switch(groupaliases->del(groupaliases, tid, &key, 0)) {
1138 TXN_RETRY(t_dos, tid);
1140 TXN_ABORT(t_dos, tid);
1144 switch(groupinfo->del(groupinfo, tid, &key, 0)) {
1149 TXN_RETRY(t_dos, tid);
1151 TXN_ABORT(t_dos, tid);
1156 TXN_COMMIT(t_dos, tid);
1158 if(delete_all_records(gi.current_db, gi.current_gid) == 0) {
1159 rm_temp_groupinfo(gi.current_gid);
1161 if(gi.status & GROUPINFO_MOVING) {
1162 if(delete_all_records(gi.new_db, gi.new_gid) == 0) {
1163 rm_temp_groupinfo(gi.new_gid);
1168 for(i = 0; i < listcount; i++)
1175 static int count_records(struct groupinfo *gi)
1182 u_int32_t artnum, newlow = 0;
1184 memset(&key, 0, sizeof key);
1185 memset(&val, 0, sizeof val);
1186 memset(&dk, 0, sizeof dk);
1188 db = get_db_bynum(gi->current_db);
1192 dk.groupnum = gi->current_gid;
1195 key.size = key.ulen = sizeof dk;
1196 key.flags = DB_DBT_USERMEM;
1197 val.flags = DB_DBT_PARTIAL;
1201 ret = db->cursor(db, NULL, &cursor, 0);
1205 ret = cursor->c_get(cursor, &key, &val, DB_SET_RANGE);
1212 cursor->c_close(cursor);
1215 while(ret == 0 && key.size == sizeof(dk) && dk.groupnum == gi->current_gid) {
1216 artnum = ntohl(dk.artnum);
1217 if(newlow == 0 || newlow > artnum)
1219 if(artnum > gi->high)
1223 ret = cursor->c_get(cursor, &key, &val, DB_NEXT);
1225 cursor->c_close(cursor);
1227 gi->low = gi->high + 1;
1231 if(ret == DB_NOTFOUND)
1238 * Locking: OVopen() calls ovdb_getlock(OVDB_LOCK_NORMAL). This
1239 * aquires a read (shared) lock on the lockfile. Multiple processes
1240 * can have this file locked at the same time. That way, if there
1241 * are any processes that are using the database, the lock file will
1242 * have one or more shared locks on it.
1244 * ovdb_init, when starting, calls ovdb_getlock(OVDB_LOCK_EXCLUSIVE).
1245 * This tries to get a write (exclusive) lock, which will fail if
1246 * anyone has a shared lock. This way, ovdb_init can tell if there
1247 * are any processes using the database. If not, and the excl. lock
1248 * succeeds, ovdb_init is free to do DB_RUNRECOVER.
1250 * ovdb_getlock() in the "normal" lock mode calls ovdb_check_monitor,
1251 * which looks for the OVDB_MONITOR_PIDFILE. If said file does not
1252 * exist, or the PID in it does not exist, it will fail. This will
1253 * prevent OVopen() from opening the database if ovdb_monitor is not running.
1255 * The OVDB_LOCK_ADMIN mode is used by ovdb_monitor to get a shared lock
1256 * without testing the pidfile.
1258 static int lockfd = -1;
1259 bool ovdb_getlock(int mode)
1262 char *lockfn = concatpath(innconf->pathrun, OVDB_LOCKFN);
1263 lockfd = open(lockfn,
1264 mode == OVDB_LOCK_NORMAL ? O_RDWR : O_CREAT|O_RDWR, 0660);
1268 syslog(L_FATAL, "OVDB: can not open database unless ovdb_monitor is running.");
1271 close_on_exec(lockfd, true);
1276 if(mode == OVDB_LOCK_NORMAL) {
1277 if(!ovdb_check_pidfile(OVDB_MONITOR_PIDFILE)) {
1278 syslog(L_FATAL, "OVDB: can not open database unless ovdb_monitor is running.");
1282 if(mode == OVDB_LOCK_EXCLUSIVE) {
1283 if(!inn_lock_file(lockfd, INN_LOCK_WRITE, false)) {
1290 if(!inn_lock_file(lockfd, INN_LOCK_READ, false)) {
1299 bool ovdb_releaselock(void)
1304 r = inn_lock_file(lockfd, INN_LOCK_UNLOCK, false);
1310 bool ovdb_check_pidfile(char *file)
1314 char *pidfn = concatpath(innconf->pathrun, file);
1316 f = open(pidfn, O_RDONLY);
1319 syslog(L_FATAL, "OVDB: can't open %s: %m", pidfn);
1323 memset(buf, 0, SMBUF);
1324 if(read(f, buf, SMBUF-1) < 0) {
1325 syslog(L_FATAL, "OVDB: can't read from %s: %m", pidfn);
1336 if(kill(pid, 0) < 0 && errno == ESRCH) {
1342 /* make sure the effective uid is that of NEWSUSER */
1343 bool ovdb_check_user(void)
1346 static int result = -1;
1349 p = getpwnam(NEWSUSER);
1351 syslog(L_FATAL, "OVDB: getpwnam(" NEWSUSER ") failed: %m");
1354 result = (p->pw_uid == geteuid());
1359 static int check_version(void)
1366 #if DB_VERSION_MAJOR == 2
1368 memset(&dbinfo, 0, sizeof dbinfo);
1370 ret = db_open("version", DB_BTREE, _db_flags, 0666, OVDBenv, &dbinfo,
1373 syslog(L_FATAL, "OVDB: db_open failed: %s", db_strerror(ret));
1377 /* open version db */
1378 ret = db_create(&vdb, OVDBenv, 0);
1380 syslog(L_FATAL, "OVDB: open: db_create: %s", db_strerror(ret));
1383 #if DB_VERSION_MAJOR > 4 || (DB_VERSION_MAJOR == 4 && DB_VERSION_MINOR >= 1)
1384 ret = vdb->open(vdb, NULL, "version", NULL, DB_BTREE, _db_flags, 0666);
1386 ret = vdb->open(vdb, "version", NULL, DB_BTREE, _db_flags, 0666);
1390 syslog(L_FATAL, "OVDB: open: version->open: %s", db_strerror(ret));
1394 memset(&key, 0, sizeof key);
1395 memset(&val, 0, sizeof val);
1396 key.data = (char *) "dataversion";
1397 key.size = sizeof("dataversion");
1398 ret = vdb->get(vdb, NULL, &key, &val, 0);
1400 if(ret != DB_NOTFOUND) {
1401 syslog(L_FATAL, "OVDB: open: can't retrieve version: %s", db_strerror(ret));
1406 if(ret == DB_NOTFOUND || val.size != sizeof dv) {
1408 if(!(OVDBmode & OV_WRITE)) {
1413 val.size = sizeof dv;
1414 ret = vdb->put(vdb, NULL, &key, &val, 0);
1416 syslog(L_FATAL, "OVDB: open: can't store version: %s", db_strerror(ret));
1421 memcpy(&dv, val.data, sizeof dv);
1423 if(dv > DATA_VERSION) {
1424 syslog(L_FATAL, "OVDB: can't open database: unknown version %d", dv);
1428 if(dv < DATA_VERSION) {
1429 syslog(L_FATAL, "OVDB: database is an old version, please run ovdb_init");
1434 /* The version db also stores some config values, which will override the
1435 corresponding ovdb.conf setting. */
1437 key.data = (char *) "numdbfiles";
1438 key.size = sizeof("numdbfiles");
1439 ret = vdb->get(vdb, NULL, &key, &val, 0);
1441 if(ret != DB_NOTFOUND) {
1442 syslog(L_FATAL, "OVDB: open: can't retrieve numdbfiles: %s", db_strerror(ret));
1447 if(ret == DB_NOTFOUND || val.size != sizeof(ovdb_conf.numdbfiles)) {
1448 if(!(OVDBmode & OV_WRITE)) {
1452 val.data = &(ovdb_conf.numdbfiles);
1453 val.size = sizeof(ovdb_conf.numdbfiles);
1454 ret = vdb->put(vdb, NULL, &key, &val, 0);
1456 syslog(L_FATAL, "OVDB: open: can't store numdbfiles: %s", db_strerror(ret));
1461 memcpy(&(ovdb_conf.numdbfiles), val.data, sizeof(ovdb_conf.numdbfiles));
1469 int ovdb_open_berkeleydb(int mode, int flags)
1472 u_int32_t ai_flags = DB_INIT_LOCK|DB_INIT_LOG|DB_INIT_MPOOL|DB_INIT_TXN;
1478 return 0; /* already opened */
1480 if(OVDBmode & OV_WRITE) {
1481 _db_flags |= DB_CREATE;
1482 ai_flags |= DB_CREATE;
1484 _db_flags |= DB_RDONLY;
1486 if(flags & OVDB_RECOVER)
1487 ai_flags |= DB_RECOVER;
1489 #if DB_VERSION_MAJOR == 2 || (DB_VERSION_MAJOR == 3 && DB_VERSION_MINOR < 2)
1490 if(ovdb_conf.txn_nosync)
1491 ai_flags |= DB_TXN_NOSYNC;
1494 #if DB_VERSION_MAJOR == 2
1496 OVDBenv = xcalloc(1, sizeof(DB_ENV));
1498 OVDBenv->db_errcall = OVDBerror;
1499 OVDBenv->mp_size = ovdb_conf.cachesize;
1500 OVDBenv->lk_max = ovdb_conf.maxlocks;
1502 /* initialize environment */
1503 ret = db_appinit(ovdb_conf.home, NULL, OVDBenv, ai_flags);
1507 syslog(L_FATAL, "OVDB: db_appinit failed: %s", db_strerror(ret));
1511 ret = db_env_create(&OVDBenv, 0);
1513 syslog(L_FATAL, "OVDB: db_env_create: %s", db_strerror(ret));
1517 if((flags & (OVDB_UPGRADE | OVDB_RECOVER)) == (OVDB_UPGRADE | OVDB_RECOVER))
1518 ai_flags |= DB_PRIVATE;
1519 if(!(ai_flags & DB_PRIVATE)) {
1520 if(ovdb_conf.useshm)
1521 ai_flags |= DB_SYSTEM_MEM;
1522 #if DB_VERSION_MAJOR >= 4 || (DB_VERSION_MAJOR == 3 && DB_VERSION_MINOR > 0)
1523 OVDBenv->set_shm_key(OVDBenv, ovdb_conf.shmkey);
1527 OVDBenv->set_errcall(OVDBenv, OVDBerror);
1528 OVDBenv->set_cachesize(OVDBenv, 0, ovdb_conf.cachesize, 1);
1529 #if DB_VERSION_MAJOR >= 4
1530 OVDBenv->set_lk_max_locks(OVDBenv, ovdb_conf.maxlocks);
1531 OVDBenv->set_lk_max_lockers(OVDBenv, ovdb_conf.maxlocks);
1532 OVDBenv->set_lk_max_objects(OVDBenv, ovdb_conf.maxlocks);
1534 OVDBenv->set_lk_max(OVDBenv, ovdb_conf.maxlocks);
1537 #if DB_VERSION_MAJOR >= 4 || (DB_VERSION_MAJOR == 3 && DB_VERSION_MINOR >= 2)
1538 if(ovdb_conf.txn_nosync)
1539 OVDBenv->set_flags(OVDBenv, DB_TXN_NOSYNC, 1);
1542 if((flags & (OVDB_UPGRADE | OVDB_RECOVER)) != OVDB_UPGRADE) {
1543 #if DB_VERSION_MAJOR == 3 && DB_VERSION_MINOR == 0
1544 ret = OVDBenv->open(OVDBenv, ovdb_conf.home, NULL, ai_flags, 0666);
1546 ret = OVDBenv->open(OVDBenv, ovdb_conf.home, ai_flags, 0666);
1549 OVDBenv->close(OVDBenv, 0);
1551 syslog(L_FATAL, "OVDB: OVDBenv->open: %s", db_strerror(ret));
1555 #endif /* DB_VERSION_MAJOR == 2 */
1560 bool ovdb_open(int mode)
1563 #if DB_VERSION_MAJOR == 2
1569 if(OVDBenv != NULL || clientmode) {
1570 syslog(L_ERROR, "OVDB: ovdb_open called more than once");
1575 if(ovdb_conf.readserver && mode == OV_READ)
1578 if(mode & OVDB_SERVER)
1582 if(client_connect() == 0)
1586 if(!ovdb_check_user()) {
1587 syslog(L_FATAL, "OVDB: must be running as " NEWSUSER " to access overview.");
1590 if(!ovdb_getlock(OVDB_LOCK_NORMAL)) {
1591 syslog(L_FATAL, "OVDB: ovdb_getlock failed: %m");
1595 if(ovdb_open_berkeleydb(mode, 0) != 0)
1598 if(check_version() != 0)
1601 if(mode & OV_WRITE || mode & OVDB_SERVER) {
1607 #if DB_VERSION_MAJOR == 2
1608 memset(&_dbinfo, 0, sizeof _dbinfo);
1609 _dbinfo.db_pagesize = ovdb_conf.pagesize;
1610 _dbinfo.bt_minkey = ovdb_conf.minkey;
1613 dbs = xcalloc(ovdb_conf.numdbfiles, sizeof(DB *));
1616 for(i = 0; i < ovdb_conf.numdbfiles; i++) {
1617 ret = open_db_file(i);
1619 syslog(L_FATAL, "OVDB: open_db_file failed: %s", db_strerror(ret));
1625 #if DB_VERSION_MAJOR == 2
1626 memset(&dbinfo, 0, sizeof dbinfo);
1628 ret = db_open("groupinfo", DB_BTREE, _db_flags, 0666, OVDBenv,
1629 &dbinfo, &groupinfo);
1631 syslog(L_FATAL, "OVDB: db_open failed: %s", db_strerror(ret));
1635 ret = db_open("groupaliases", DB_HASH, _db_flags, 0666, OVDBenv,
1636 &dbinfo, &groupaliases);
1638 syslog(L_FATAL, "OVDB: db_open failed: %s", db_strerror(ret));
1642 ret = db_create(&groupinfo, OVDBenv, 0);
1644 syslog(L_FATAL, "OVDB: open: db_create: %s", db_strerror(ret));
1647 #if DB_VERSION_MAJOR > 4 || (DB_VERSION_MAJOR == 4 && DB_VERSION_MINOR >= 1)
1648 TXN_START(t_open_groupinfo, tid);
1649 ret = groupinfo->open(groupinfo, tid, "groupinfo", NULL, DB_BTREE,
1652 TXN_COMMIT(t_open_groupinfo, tid);
1654 ret = groupinfo->open(groupinfo, "groupinfo", NULL, DB_BTREE,
1658 groupinfo->close(groupinfo, 0);
1659 syslog(L_FATAL, "OVDB: open: groupinfo->open: %s", db_strerror(ret));
1662 ret = db_create(&groupaliases, OVDBenv, 0);
1664 syslog(L_FATAL, "OVDB: open: db_create: %s", db_strerror(ret));
1667 #if DB_VERSION_MAJOR > 4 || (DB_VERSION_MAJOR == 4 && DB_VERSION_MINOR >= 1)
1668 TXN_START(t_open_groupaliases, tid);
1669 ret = groupaliases->open(groupaliases, tid, "groupaliases", NULL, DB_HASH,
1672 TXN_COMMIT(t_open_groupaliases, tid);
1674 ret = groupaliases->open(groupaliases, "groupaliases", NULL, DB_HASH,
1678 groupaliases->close(groupaliases, 0);
1679 syslog(L_FATAL, "OVDB: open: groupaliases->open: %s", db_strerror(ret));
1690 ovdb_groupstats(char *group, int *lo, int *hi, int *count, int *flag)
1693 struct groupinfo gi;
1697 struct rs_groupstats repl;
1699 rs.what = CMD_GROUPSTATS;
1700 rs.grouplen = strlen(group)+1;
1702 if (csend(&rs, sizeof(rs)) < 0)
1704 if (csend(group, rs.grouplen) < 0)
1706 crecv(&repl, sizeof(repl));
1708 if(repl.status != CMD_GROUPSTATS)
1711 /* we don't use the alias yet, but the OV API will be extended
1712 at some point so that the alias is returned also */
1713 if(repl.aliaslen != 0) {
1714 char *buf = xmalloc(repl.aliaslen);
1715 crecv(buf, repl.aliaslen);
1724 *count = repl.count;
1730 ret = ovdb_getgroupinfo(group, &gi, true, NULL, 0);
1738 syslog(L_ERROR, "OVDB: ovdb_getgroupinfo failed: %s", db_strerror(ret));
1753 bool ovdb_groupadd(char *group, ARTNUM lo, ARTNUM hi, char *flag)
1756 struct groupinfo gi;
1761 memset(&key, 0, sizeof key);
1762 memset(&val, 0, sizeof val);
1764 TXN_START(t_groupadd, tid);
1770 ret = ovdb_getgroupinfo(group, &gi, false, tid, DB_RMW);
1778 TXN_RETRY(t_groupadd, tid);
1780 TXN_ABORT(t_groupadd, tid);
1781 syslog(L_ERROR, "OVDB: ovdb_getgroupinfo failed: %s", db_strerror(ret));
1785 if(!new && (gi.status & GROUPINFO_DELETED)
1786 && !(gi.status & GROUPINFO_EXPIRING)
1787 && !(gi.status & GROUPINFO_MOVING)) {
1789 char g[MAXHEADERSIZE];
1791 strlcpy(g, group, sizeof(g));
1794 key.size = s + sizeof(int);
1797 memcpy(g+s, &c, sizeof(int));
1798 ret = groupinfo->get(groupinfo, tid, &key, &val, 0);
1800 if(ret == TRYAGAIN) {
1801 TXN_RETRY(t_groupadd, tid);
1804 val.size = sizeof(gi);
1805 ret = groupinfo->put(groupinfo, tid, &key, &val, 0);
1811 TXN_RETRY(t_groupadd, tid);
1813 TXN_ABORT(t_groupadd, tid);
1814 syslog(L_ERROR, "OVDB: groupinfo->put: %s", db_strerror(ret));
1818 key.size = strlen(group);
1819 ret = groupinfo->del(groupinfo, tid, &key, 0);
1825 TXN_RETRY(t_groupadd, tid);
1827 TXN_ABORT(t_groupadd, tid);
1828 syslog(L_ERROR, "OVDB: groupinfo->del: %s", db_strerror(ret));
1835 ret = groupid_new(&gi.current_gid, tid);
1841 TXN_RETRY(t_groupadd, tid);
1843 TXN_ABORT(t_groupadd, tid);
1844 syslog(L_ERROR, "OVDB: groupid_new: %s", db_strerror(ret));
1847 gi.low = lo ? lo : 1;
1851 gi.expired = time(NULL);
1852 gi.expiregrouppid = 0;
1853 gi.current_db = gi.new_db = which_db(group);
1854 gi.new_gid = gi.current_gid;
1861 key.size = strlen(group);
1863 val.size = sizeof gi;
1865 ret = groupinfo->put(groupinfo, tid, &key, &val, 0);
1870 TXN_RETRY(t_groupadd, tid);
1872 TXN_ABORT(t_groupadd, tid);
1873 syslog(L_ERROR, "OVDB: groupadd: groupinfo->put: %s", db_strerror(ret));
1879 key.size = strlen(group);
1880 val.data = flag + 1;
1881 val.size = strcspn(flag, "\n") - 1;
1883 switch(ret = groupaliases->put(groupaliases, tid, &key, &val, 0)) {
1887 TXN_RETRY(t_groupadd, tid);
1889 TXN_ABORT(t_groupadd, tid);
1890 syslog(L_ERROR, "OVDB: groupadd: groupaliases->put: %s", db_strerror(ret));
1895 TXN_COMMIT(t_groupadd, tid);
1899 bool ovdb_groupdel(char *group)
1902 struct groupinfo gi;
1906 memset(&key, 0, sizeof key);
1907 memset(&val, 0, sizeof val);
1909 /* We only need to set the deleted flag in groupinfo to prevent readers
1910 from seeing this group. The actual overview records aren't deleted
1911 now, since that could take a significant amount of time (and innd
1912 is who normally calls this function). The expireover run will
1913 clean up the deleted groups. */
1915 TXN_START(t_groupdel, tid);
1920 ret = ovdb_getgroupinfo(group, &gi, true, tid, DB_RMW);
1928 TXN_RETRY(t_groupdel, tid);
1930 syslog(L_ERROR, "OVDB: ovdb_getgroupinfo failed: %s", db_strerror(ret));
1931 TXN_ABORT(t_groupdel, tid);
1935 gi.status |= GROUPINFO_DELETED;
1938 key.size = strlen(group);
1940 val.size = sizeof gi;
1942 switch(ret = groupinfo->put(groupinfo, tid, &key, &val, 0)) {
1946 TXN_RETRY(t_groupdel, tid);
1948 TXN_ABORT(t_groupdel, tid);
1949 syslog(L_ERROR, "OVDB: groupadd: groupinfo->put: %s", db_strerror(ret));
1953 switch(ret = groupaliases->del(groupaliases, tid, &key, 0)) {
1958 TXN_RETRY(t_groupdel, tid);
1960 syslog(L_ERROR, "OVDB: groupdel: groupaliases->del: %s", db_strerror(ret));
1961 TXN_ABORT(t_groupdel, tid);
1965 TXN_COMMIT(t_groupdel, tid);
1969 bool ovdb_add(char *group, ARTNUM artnum, TOKEN token, char *data, int len, time_t arrived, time_t expires)
1971 static size_t databuflen = 0;
1972 static char *databuf;
1976 struct groupinfo gi;
1980 memset(&dk, 0, sizeof dk);
1982 if(databuflen == 0) {
1983 databuflen = BIG_BUFFER;
1984 databuf = xmalloc(databuflen);
1986 if(databuflen < len + sizeof(struct ovdata)) {
1987 databuflen = len + sizeof(struct ovdata);
1988 databuf = xrealloc(databuf, databuflen);
1991 /* hmm... BerkeleyDB needs something like a 'struct iovec' so that we don't
1992 have to make a new buffer and copy everything in to it */
1994 ((struct ovdata *)databuf)->token = token;
1995 ((struct ovdata *)databuf)->arrived = arrived;
1996 ((struct ovdata *)databuf)->expires = expires;
1997 memcpy(databuf + sizeof(struct ovdata), data, len);
1998 len += sizeof(struct ovdata);
2000 memset(&key, 0, sizeof key);
2001 memset(&val, 0, sizeof val);
2003 /* start the transaction */
2004 TXN_START(t_add, tid);
2009 /* first, retrieve groupinfo */
2010 ret = ovdb_getgroupinfo(group, &gi, true, tid, DB_RMW);
2016 TXN_ABORT(t_add, tid);
2019 TXN_RETRY(t_add, tid);
2021 TXN_ABORT(t_add, tid);
2022 syslog(L_ERROR, "OVDB: add: ovdb_getgroupinfo: %s", db_strerror(ret));
2026 /* adjust groupinfo */
2027 if(Cutofflow && gi.low > artnum) {
2028 TXN_ABORT(t_add, tid);
2031 if(gi.low == 0 || gi.low > artnum)
2033 if(gi.high < artnum)
2037 /* store groupinfo */
2039 key.size = strlen(group);
2041 val.size = sizeof gi;
2043 ret = groupinfo->put(groupinfo, tid, &key, &val, 0);
2049 TXN_RETRY(t_add, tid);
2051 TXN_ABORT(t_add, tid);
2052 syslog(L_ERROR, "OVDB: add: groupinfo->put: %s", db_strerror(ret));
2056 /* store overview */
2057 db = get_db_bynum(gi.current_db);
2059 TXN_ABORT(t_add, tid);
2062 dk.groupnum = gi.current_gid;
2063 dk.artnum = htonl((u_int32_t)artnum);
2066 key.size = sizeof dk;
2070 ret = db->put(db, tid, &key, &val, 0);
2076 TXN_RETRY(t_add, tid);
2078 TXN_ABORT(t_add, tid);
2079 syslog(L_ERROR, "OVDB: add: db->put: %s", db_strerror(ret));
2083 if(artnum < gi.high && gi.status & GROUPINFO_MOVING) {
2084 /* If the GROUPINFO_MOVING flag is set, then expireover
2085 is writing overview records under a new groupid.
2086 If this overview record is not at the highmark,
2087 we need to also store it under the new groupid */
2088 db = get_db_bynum(gi.new_db);
2090 TXN_ABORT(t_add, tid);
2093 dk.groupnum = gi.new_gid;
2095 ret = db->put(db, tid, &key, &val, 0);
2101 TXN_RETRY(t_add, tid);
2103 TXN_ABORT(t_add, tid);
2104 syslog(L_ERROR, "OVDB: add: db->put: %s", db_strerror(ret));
2109 TXN_COMMIT(t_add, tid);
2113 bool ovdb_cancel(TOKEN token UNUSED)
2126 /* Even though nnrpd only does one search at a time, a read server process could
2127 do many concurrent searches; hence we must keep track of an arbitrary number of
2129 static struct ovdbsearch **searches = NULL;
2130 static int nsearches = 0;
2131 static int maxsearches = 0;
2134 ovdb_opensearch(char *group, int low, int high)
2137 struct ovdbsearch *s;
2138 struct groupinfo gi;
2143 struct rs_opensrch repl;
2145 rs.what = CMD_OPENSRCH;
2146 rs.grouplen = strlen(group)+1;
2150 if (csend(&rs, sizeof(rs)) < 0)
2152 if (csend(group, rs.grouplen) < 0)
2154 crecv(&repl, sizeof(repl));
2156 if(repl.status != CMD_OPENSRCH)
2162 ret = ovdb_getgroupinfo(group, &gi, true, NULL, 0);
2170 syslog(L_ERROR, "OVDB: ovdb_getgroupinfo failed: %s", db_strerror(ret));
2174 s = xmalloc(sizeof(struct ovdbsearch));
2175 db = get_db_bynum(gi.current_db);
2181 ret = db->cursor(db, NULL, &(s->cursor), 0);
2183 syslog(L_ERROR, "OVDB: opensearch: s->db->cursor: %s", db_strerror(ret));
2188 s->gid = gi.current_gid;
2193 if(searches == NULL) {
2196 searches = xmalloc(sizeof(struct ovdbsearch *) * maxsearches);
2198 if(nsearches == maxsearches) {
2200 searches = xrealloc(searches, sizeof(struct ovdbsearch *) * maxsearches);
2202 searches[nsearches] = s;
2209 ovdb_search(void *handle, ARTNUM *artnum, char **data, int *len,
2210 TOKEN *token, time_t *arrived)
2212 struct ovdbsearch *s = (struct ovdbsearch *)handle;
2221 struct rs_srch repl;
2222 static char *databuf;
2223 static int buflen = 0;
2228 if (csend(&rs, sizeof(rs)) < 0)
2230 if (crecv(&repl, sizeof(repl)) < 0)
2233 if(repl.status != CMD_SRCH)
2235 if(repl.len > buflen) {
2237 buflen = repl.len + 512;
2238 databuf = xmalloc(buflen);
2240 buflen = repl.len + 512;
2241 databuf = xrealloc(databuf, buflen);
2244 crecv(databuf, repl.len);
2247 *artnum = repl.artnum;
2249 *token = repl.token;
2251 *arrived = repl.arrived;
2261 flags = DB_SET_RANGE;
2262 memset(&dk, 0, sizeof dk);
2263 dk.groupnum = s->gid;
2264 dk.artnum = htonl(s->firstart);
2274 syslog(L_ERROR, "OVDB: OVsearch called again after false return");
2278 memset(&key, 0, sizeof key);
2279 memset(&val, 0, sizeof val);
2281 key.size = key.ulen = sizeof(struct datakey);
2282 key.flags = DB_DBT_USERMEM;
2285 /* caller doesn't need data, so we don't have to retrieve it all */
2286 val.flags |= DB_DBT_PARTIAL;
2288 if(token || arrived)
2289 val.dlen = sizeof(struct ovdata);
2292 switch(ret = s->cursor->c_get(s->cursor, &key, &val, flags)) {
2297 s->cursor->c_close(s->cursor);
2301 syslog(L_ERROR, "OVDB: search: c_get: %s", db_strerror(ret));
2303 s->cursor->c_close(s->cursor);
2308 if(key.size != sizeof(struct datakey)) {
2310 s->cursor->c_close(s->cursor);
2315 if(dk.groupnum != s->gid || ntohl(dk.artnum) > s->lastart) {
2317 s->cursor->c_close(s->cursor);
2322 if( ((len || data) && val.size <= sizeof(struct ovdata))
2323 || ((token || arrived) && val.size < sizeof(struct ovdata)) ) {
2324 syslog(L_ERROR, "OVDB: search: bad value length");
2326 s->cursor->c_close(s->cursor);
2331 if(ntohl(dk.artnum) == s->lastart) {
2333 s->cursor->c_close(s->cursor);
2338 *artnum = ntohl(dk.artnum);
2340 if(token || arrived)
2341 memcpy(&ovd, val.data, sizeof(struct ovdata));
2345 *arrived = ovd.arrived;
2348 *len = val.size - sizeof(struct ovdata);
2350 *data = (char *)val.data + sizeof(struct ovdata);
2355 void ovdb_closesearch(void *handle)
2361 rs.what = CMD_CLOSESRCH;
2363 csend(&rs, sizeof(rs));
2364 /* no reply is sent for a CMD_CLOSESRCH */
2366 struct ovdbsearch *s = (struct ovdbsearch *)handle;
2369 s->cursor->c_close(s->cursor);
2371 for(i = 0; i < nsearches; i++) {
2372 if(s == searches[i]) {
2377 for( ; i < nsearches; i++) {
2378 searches[i] = searches[i+1];
2385 bool ovdb_getartinfo(char *group, ARTNUM artnum, TOKEN *token)
2388 group_id_t cgid = 0;
2393 struct groupinfo gi;
2398 struct rs_artinfo repl;
2400 rs.what = CMD_ARTINFO;
2401 rs.grouplen = strlen(group)+1;
2404 if (csend(&rs, sizeof(rs)) < 0)
2406 if (csend(group, rs.grouplen) < 0)
2408 crecv(&repl, sizeof(repl));
2410 if(repl.status != CMD_ARTINFO)
2414 *token = repl.token;
2420 ret = ovdb_getgroupinfo(group, &gi, true, NULL, 0);
2428 syslog(L_ERROR, "OVDB: ovdb_getgroupinfo failed: %s", db_strerror(ret));
2433 /* This was our second groupinfo retrieval; because the article
2434 retrieval came up empty. If the group ID hasn't changed
2435 since the first groupinfo retrieval, we can assume the article
2436 is definitely not there. Otherwise, we'll try to retrieve
2437 it the article again. */
2438 if(cdb == gi.current_db && cgid == gi.current_gid)
2442 db = get_db_bynum(gi.current_db);
2446 memset(&dk, 0, sizeof dk);
2447 dk.groupnum = gi.current_gid;
2448 dk.artnum = htonl((u_int32_t)artnum);
2450 memset(&key, 0, sizeof key);
2451 memset(&val, 0, sizeof val);
2454 key.size = sizeof dk;
2456 /* caller doesn't need data, so we don't have to retrieve it all */
2457 val.flags = DB_DBT_PARTIAL;
2460 val.dlen = sizeof(struct ovdata);
2462 switch(ret = db->get(db, NULL, &key, &val, 0)) {
2467 syslog(L_ERROR, "OVDB: getartinfo: db->get: %s", db_strerror(ret));
2471 if(ret == DB_NOTFOUND) {
2472 /* If the group is being moved (i.e., its group ID is going
2473 to change), there's a chance the article is now under the
2474 new ID. So we'll grab the groupinfo again to check for
2476 if(!pass && (gi.status & GROUPINFO_MOVING)) {
2477 cdb = gi.current_db;
2478 cgid = gi.current_gid;
2487 if(token && val.size < sizeof(struct ovdata)) {
2488 syslog(L_ERROR, "OVDB: getartinfo: data too short");
2493 memcpy(&ovd, val.data, sizeof(struct ovdata));
2499 bool ovdb_expiregroup(char *group, int *lo, struct history *h)
2501 DB *db, *ndb = NULL;
2502 DBT key, val, nkey, gkey, gval;
2505 int ret = 0, delete, old_db = 0, cleanup;
2506 struct groupinfo gi;
2508 struct datakey dk, ndk;
2509 group_id_t old_gid = 0;
2511 u_int32_t artnum = 0, currentart, lowest;
2512 int i, compact, done, currentcount, newcount;
2515 eo_start = time(NULL);
2516 delete_old_stuff(0); /* remove deleted groups first */
2519 /* Special case: when called with NULL group, we're to clean out
2520 * records for forgotton groups (groups removed from the active file
2521 * but not from overview).
2522 * This happens at the end of the expireover run, and only if all
2523 * of the groups in the active file have been processed.
2524 * delete_old_stuff(1) will remove groups that are in ovdb but
2525 * have not been processed during this run.
2529 return delete_old_stuff(1);
2531 memset(&key, 0, sizeof key);
2532 memset(&nkey, 0, sizeof nkey);
2533 memset(&val, 0, sizeof val);
2534 memset(&dk, 0, sizeof dk);
2535 memset(&ndk, 0, sizeof ndk);
2537 TXN_START(t_expgroup_1, tid);
2544 ret = ovdb_getgroupinfo(group, &gi, true, tid, DB_RMW);
2550 TXN_RETRY(t_expgroup_1, tid);
2552 syslog(L_ERROR, "OVDB: expiregroup: ovdb_getgroupinfo failed: %s", db_strerror(ret));
2554 TXN_ABORT(t_expgroup_1, tid);
2558 if(gi.status & GROUPINFO_EXPIRING) {
2559 /* is there another expireover working on this group? */
2560 ret = kill(gi.expiregrouppid, 0);
2565 TXN_ABORT(t_expgroup_1, tid);
2569 /* a previous expireover run must've died. We'll clean
2571 if(gi.status & GROUPINFO_MOVING) {
2574 old_gid = gi.new_gid;
2576 ret = mk_temp_groupinfo(old_db, old_gid, tid);
2581 TXN_RETRY(t_expgroup_1, tid);
2583 TXN_ABORT(t_expgroup_1, tid);
2586 gi.status &= ~GROUPINFO_MOVING;
2590 if(gi.count < ovdb_conf.nocompact || ovdb_conf.nocompact == 0)
2598 db = get_db_bynum(gi.current_db);
2600 TXN_ABORT(t_expgroup_1, tid);
2604 gi.status |= GROUPINFO_EXPIRING;
2605 gi.expiregrouppid = getpid();
2607 gi.status |= GROUPINFO_MOVING;
2608 gi.new_db = gi.current_db;
2610 ret = groupid_new(&gi.new_gid, tid);
2616 TXN_RETRY(t_expgroup_1, tid);
2618 TXN_ABORT(t_expgroup_1, tid);
2619 syslog(L_ERROR, "OVDB: expiregroup: groupid_new: %s", db_strerror(ret));
2625 key.size = strlen(group);
2627 val.size = sizeof gi;
2629 ret = groupinfo->put(groupinfo, tid, &key, &val, 0);
2635 TXN_RETRY(t_expgroup_1, tid);
2637 TXN_ABORT(t_expgroup_1, tid);
2638 syslog(L_ERROR, "OVDB: expiregroup: groupinfo->put: %s", db_strerror(ret));
2641 TXN_COMMIT(t_expgroup_1, tid);
2644 if(delete_all_records(old_db, old_gid) == 0) {
2645 rm_temp_groupinfo(old_gid);
2650 * The following loop iterates over the OV records for the group in
2651 * "batches", to limit transaction sizes.
2656 * process EXPIREGROUP_TXN_SIZE records
2657 * write updated groupinfo
2658 * commit transaction
2662 lowest = currentcount = 0;
2664 memset(&gkey, 0, sizeof gkey);
2665 memset(&gval, 0, sizeof gval);
2667 gkey.size = strlen(group);
2669 gval.size = sizeof gi;
2672 TXN_START(t_expgroup_loop, tid);
2678 ret = ovdb_getgroupinfo(group, &gi, false, tid, DB_RMW);
2684 TXN_RETRY(t_expgroup_loop, tid);
2686 TXN_ABORT(t_expgroup_loop, tid);
2687 syslog(L_ERROR, "OVDB: expiregroup: ovdb_getgroupinfo: %s", db_strerror(ret));
2691 ret = db->cursor(db, tid, &cursor, 0);
2697 TXN_RETRY(t_expgroup_loop, tid);
2699 TXN_ABORT(t_expgroup_loop, tid);
2700 syslog(L_ERROR, "OVDB: expiregroup: db->cursor: %s", db_strerror(ret));
2704 dk.groupnum = gi.current_gid;
2705 dk.artnum = htonl(currentart);
2707 key.size = key.ulen = sizeof dk;
2708 key.flags = DB_DBT_USERMEM;
2710 for(i=0; i < EXPIREGROUP_TXN_SIZE; i++) {
2711 ret = cursor->c_get(cursor, &key, &val,
2712 i == 0 ? DB_SET_RANGE : DB_NEXT);
2719 cursor->c_close(cursor);
2720 TXN_RETRY(t_expgroup_loop, tid);
2722 cursor->c_close(cursor);
2723 TXN_ABORT(t_expgroup_loop, tid);
2724 syslog(L_ERROR, "OVDB: expiregroup: c_get: %s", db_strerror(ret));
2728 /* stop if: there are no more keys, an unknown key is reached,
2729 or reach a different group */
2731 if(ret == DB_NOTFOUND
2732 || key.size != sizeof dk
2733 || dk.groupnum != gi.current_gid) {
2738 artnum = ntohl(dk.artnum);
2741 if(val.size < sizeof ovd) {
2742 delete = 1; /* must be corrupt, just delete it */
2744 memcpy(&ovd, val.data, sizeof ovd);
2747 if (!SMprobe(EXPENSIVESTAT, &ovd.token, NULL) || OVstatall) {
2748 if((ah = SMretrieve(ovd.token, RETR_STAT)) == NULL) {
2753 if (!OVhisthasmsgid(h, (char *)val.data + sizeof(ovd))) {
2757 if (!delete && innconf->groupbaseexpiry &&
2758 OVgroupbasedexpire(ovd.token, group,
2759 (char *)val.data + sizeof(ovd),
2760 val.size - sizeof(ovd),
2761 ovd.arrived, ovd.expires)) {
2768 switch(ret = cursor->c_del(cursor, 0)) {
2774 cursor->c_close(cursor);
2775 TXN_RETRY(t_expgroup_loop, tid);
2777 cursor->c_close(cursor);
2778 TXN_ABORT(t_expgroup_loop, tid);
2779 syslog(L_ERROR, "OVDB: expiregroup: c_del: %s", db_strerror(ret));
2787 ndk.groupnum = gi.new_gid;
2788 ndk.artnum = dk.artnum;
2790 nkey.size = sizeof ndk;
2792 switch(ret = ndb->put(ndb, tid, &nkey, &val, 0)) {
2796 cursor->c_close(cursor);
2797 TXN_RETRY(t_expgroup_loop, tid);
2799 cursor->c_close(cursor);
2800 TXN_ABORT(t_expgroup_loop, tid);
2801 syslog(L_ERROR, "OVDB: expiregroup: ndb->put: %s", db_strerror(ret));
2806 if(lowest != -1 && (lowest == 0 || artnum < lowest))
2810 /* end of for loop */
2812 if(cursor->c_close(cursor) == TRYAGAIN) {
2813 TXN_RETRY(t_expgroup_loop, tid);
2816 if(lowest != 0 && lowest != -1)
2821 old_db = gi.current_db;
2822 gi.current_db = gi.new_db;
2823 old_gid = gi.current_gid;
2824 gi.current_gid = gi.new_gid;
2825 gi.status &= ~GROUPINFO_MOVING;
2827 ret = mk_temp_groupinfo(old_db, old_gid, tid);
2832 TXN_RETRY(t_expgroup_loop, tid);
2834 TXN_ABORT(t_expgroup_loop, tid);
2839 gi.status &= ~GROUPINFO_EXPIRING;
2840 gi.expired = time(NULL);
2841 if(gi.count == 0 && lowest == 0)
2845 ret = groupinfo->put(groupinfo, tid, &gkey, &gval, 0);
2851 TXN_RETRY(t_expgroup_loop, tid);
2853 TXN_ABORT(t_expgroup_loop, tid);
2854 syslog(L_ERROR, "OVDB: expiregroup: groupinfo->put: %s", db_strerror(ret));
2857 TXN_COMMIT(t_expgroup_loop, tid);
2859 currentcount += newcount;
2866 currentart = artnum+1;
2870 if(delete_all_records(old_db, old_gid) == 0) {
2871 rm_temp_groupinfo(old_gid);
2875 if(currentcount != gi.count) {
2876 syslog(L_NOTICE, "OVDB: expiregroup: recounting %s", group);
2878 TXN_START(t_expgroup_recount, tid);
2882 switch(ret = ovdb_getgroupinfo(group, &gi, false, tid, DB_RMW)) {
2886 TXN_RETRY(t_expgroup_recount, tid);
2888 TXN_ABORT(t_expgroup_recount, tid);
2889 syslog(L_ERROR, "OVDB: expiregroup: ovdb_getgroupinfo: %s", db_strerror(ret));
2893 if(count_records(&gi) != 0) {
2894 TXN_ABORT(t_expgroup_recount, tid);
2898 ret = groupinfo->put(groupinfo, tid, &gkey, &gval, 0);
2904 TXN_RETRY(t_expgroup_recount, tid);
2906 TXN_ABORT(t_expgroup_recount, tid);
2907 syslog(L_ERROR, "OVDB: expiregroup: groupinfo->put: %s", db_strerror(ret));
2910 TXN_COMMIT(t_expgroup_recount, tid);
2918 bool ovdb_ctl(OVCTLTYPE type, void *val)
2921 OVSORTTYPE *sorttype;
2930 sorttype = (OVSORTTYPE *)val;
2931 *sorttype = OVNEWSGROUP;
2934 Cutofflow = *(bool *)val;
2936 case OVSTATICSEARCH:
2942 boolval = (bool *)val;
2950 void ovdb_close_berkeleydb(void)
2953 /* close db environment */
2954 #if DB_VERSION_MAJOR == 2
2955 db_appexit(OVDBenv);
2958 OVDBenv->close(OVDBenv, 0);
2964 void ovdb_close(void)
2969 client_disconnect();
2973 while(searches != NULL && nsearches) {
2974 ovdb_closesearch(searches[0]);
2976 if(searches != NULL) {
2982 /* close databases */
2983 for(i = 0; i < ovdb_conf.numdbfiles; i++)
2990 groupinfo->close(groupinfo, 0);
2994 groupaliases->close(groupaliases, 0);
2995 groupaliases = NULL;
2998 ovdb_close_berkeleydb();
3003 #endif /* USE_BERKELEY_DB */