2 Unix SMB/CIFS implementation.
3 Samba database functions
4 Copyright (C) Andrew Tridgell 1999-2000
5 Copyright (C) Luke Kenneth Casson Leighton 2000
6 Copyright (C) Paul `Rusty' Russell 2000
7 Copyright (C) Jeremy Allison 2000-2003
9 This program is free software; you can redistribute it and/or modify
10 it under the terms of the GNU General Public License as published by
11 the Free Software Foundation; either version 2 of the License, or
12 (at your option) any later version.
14 This program is distributed in the hope that it will be useful,
15 but WITHOUT ANY WARRANTY; without even the implied warranty of
16 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 GNU General Public License for more details.
19 You should have received a copy of the GNU General Public License
20 along with this program; if not, write to the Free Software
21 Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
25 /* NOTE: If you use tdbs under valgrind, and in particular if you run
26 * tdbtorture, you may get spurious "uninitialized value" warnings. I
27 * think this is because valgrind doesn't understand that the mmap'd
28 * area may be written to by other processes. Memory can, from the
29 * point of view of the grinded process, spontaneously become
32 * I can think of a few solutions. [mbp 20030311]
34 * 1 - Write suppressions for Valgrind so that it doesn't complain
35 * about this. Probably the most reasonable but people need to
36 * remember to use them.
38 * 2 - Use IO not mmap when running under valgrind. Not so nice.
40 * 3 - Use the special valgrind macros to mark memory as valid at the
41 * right time. Probably too hard -- the process just doesn't know.
71 #define TDB_MAGIC_FOOD "TDB file\n"
72 #define TDB_VERSION (0x26011967 + 6)
73 #define TDB_MAGIC (0x26011999U)
74 #define TDB_FREE_MAGIC (~TDB_MAGIC)
75 #define TDB_DEAD_MAGIC (0xFEE1DEAD)
76 #define TDB_ALIGNMENT 4
77 #define MIN_REC_SIZE (2*sizeof(struct list_struct) + TDB_ALIGNMENT)
78 #define DEFAULT_HASH_SIZE 131
79 #define TDB_PAGE_SIZE 0x2000
80 #define FREELIST_TOP (sizeof(struct tdb_header))
81 #define TDB_ALIGN(x,a) (((x) + (a)-1) & ~((a)-1))
82 #define TDB_BYTEREV(x) (((((x)&0xff)<<24)|((x)&0xFF00)<<8)|(((x)>>8)&0xFF00)|((x)>>24))
83 #define TDB_DEAD(r) ((r)->magic == TDB_DEAD_MAGIC)
84 #define TDB_BAD_MAGIC(r) ((r)->magic != TDB_MAGIC && !TDB_DEAD(r))
85 #define TDB_HASH_TOP(hash) (FREELIST_TOP + (BUCKET(hash)+1)*sizeof(tdb_off))
87 /* NB assumes there is a local variable called "tdb" that is the
88 * current context, also takes doubly-parenthesized print-style
90 #define TDB_LOG(x) (tdb->log_fn?((tdb->log_fn x),0) : 0)
101 #define MAP_FAILED ((void *)-1)
104 /* free memory if the pointer is valid and zero the pointer */
106 #define SAFE_FREE(x) do { if ((x) != NULL) {free((x)); (x)=NULL;} } while(0)
109 #define BUCKET(hash) ((hash) % tdb->header.hash_size)
112 /* all contexts, to ensure no double-opens (fcntl locks don't nest!) */
113 static TDB_CONTEXT *tdbs = NULL;
115 static int tdb_munmap(TDB_CONTEXT *tdb)
117 if (tdb->flags & TDB_INTERNAL)
122 int ret = munmap(tdb->map_ptr, tdb->map_size);
131 static void tdb_mmap(TDB_CONTEXT *tdb)
133 if (tdb->flags & TDB_INTERNAL)
137 if (!(tdb->flags & TDB_NOMMAP)) {
138 tdb->map_ptr = mmap(NULL, tdb->map_size,
139 PROT_READ|(tdb->read_only? 0:PROT_WRITE),
140 MAP_SHARED|MAP_FILE, tdb->fd, 0);
143 * NB. When mmap fails it returns MAP_FAILED *NOT* NULL !!!!
146 if (tdb->map_ptr == MAP_FAILED) {
148 TDB_LOG((tdb, 2, "tdb_mmap failed for size %d (%s)\n",
149 tdb->map_size, strerror(errno)));
159 /* Endian conversion: we only ever deal with 4 byte quantities */
160 static void *convert(void *buf, u32 size)
163 for (i = 0; i < size / 4; i++)
164 p[i] = TDB_BYTEREV(p[i]);
167 #define DOCONV() (tdb->flags & TDB_CONVERT)
168 #define CONVERT(x) (DOCONV() ? convert(&x, sizeof(x)) : &x)
170 /* the body of the database is made of one list_struct for the free space
171 plus a separate data list for each hash value */
173 tdb_off next; /* offset of the next record in the list */
174 tdb_len rec_len; /* total byte length of record */
175 tdb_len key_len; /* byte length of key */
176 tdb_len data_len; /* byte length of data */
177 u32 full_hash; /* the full 32 bit hash of the key */
178 u32 magic; /* try to catch errors */
179 /* the following union is implied:
181 char record[rec_len];
186 u32 totalsize; (tailer)
191 /***************************************************************
192 Allow a caller to set a "alarm" flag that tdb can check to abort
193 a blocking lock on SIGALRM.
194 ***************************************************************/
196 static sig_atomic_t *palarm_fired;
198 void tdb_set_lock_alarm(sig_atomic_t *palarm)
200 palarm_fired = palarm;
203 /* a byte range locking function - return 0 on success
204 this functions locks/unlocks 1 byte at the specified offset.
206 On error, errno is also set so that errors are passed back properly
207 through tdb_open(). */
208 static int tdb_brlock(TDB_CONTEXT *tdb, tdb_off offset,
209 int rw_type, int lck_type, int probe)
214 if (tdb->flags & TDB_NOLOCK)
216 if ((rw_type == F_WRLCK) && (tdb->read_only)) {
222 fl.l_whence = SEEK_SET;
228 ret = fcntl(tdb->fd,lck_type,&fl);
229 if (ret == -1 && errno == EINTR && palarm_fired && *palarm_fired)
231 } while (ret == -1 && errno == EINTR);
234 if (!probe && lck_type != F_SETLK) {
235 /* Ensure error code is set for log fun to examine. */
236 if (errno == EINTR && palarm_fired && *palarm_fired)
237 tdb->ecode = TDB_ERR_LOCK_TIMEOUT;
239 tdb->ecode = TDB_ERR_LOCK;
240 TDB_LOG((tdb, 5,"tdb_brlock failed (fd=%d) at offset %d rw_type=%d lck_type=%d\n",
241 tdb->fd, offset, rw_type, lck_type));
243 /* Was it an alarm timeout ? */
244 if (errno == EINTR && palarm_fired && *palarm_fired)
245 return TDB_ERRCODE(TDB_ERR_LOCK_TIMEOUT, -1);
246 /* Otherwise - generic lock error. */
247 /* errno set by fcntl */
248 return TDB_ERRCODE(TDB_ERR_LOCK, -1);
253 /* lock a list in the database. list -1 is the alloc list */
254 static int tdb_lock(TDB_CONTEXT *tdb, int list, int ltype)
256 if (list < -1 || list >= (int)tdb->header.hash_size) {
257 TDB_LOG((tdb, 0,"tdb_lock: invalid list %d for ltype=%d\n",
261 if (tdb->flags & TDB_NOLOCK)
264 /* Since fcntl locks don't nest, we do a lock for the first one,
265 and simply bump the count for future ones */
266 if (tdb->locked[list+1].count == 0) {
267 if (!tdb->read_only && tdb->header.rwlocks) {
268 if (tdb_spinlock(tdb, list, ltype)) {
269 TDB_LOG((tdb, 0, "tdb_lock spinlock failed on list ltype=%d\n",
273 } else if (tdb_brlock(tdb,FREELIST_TOP+4*list,ltype,F_SETLKW, 0)) {
274 TDB_LOG((tdb, 0,"tdb_lock failed on list %d ltype=%d (%s)\n",
275 list, ltype, strerror(errno)));
278 tdb->locked[list+1].ltype = ltype;
280 tdb->locked[list+1].count++;
284 /* unlock the database: returns void because it's too late for errors. */
285 /* changed to return int it may be interesting to know there
286 has been an error --simo */
287 static int tdb_unlock(TDB_CONTEXT *tdb, int list, int ltype)
291 if (tdb->flags & TDB_NOLOCK)
295 if (list < -1 || list >= (int)tdb->header.hash_size) {
296 TDB_LOG((tdb, 0, "tdb_unlock: list %d invalid (%d)\n", list, tdb->header.hash_size));
300 if (tdb->locked[list+1].count==0) {
301 TDB_LOG((tdb, 0, "tdb_unlock: count is 0\n"));
305 if (tdb->locked[list+1].count == 1) {
306 /* Down to last nested lock: unlock underneath */
307 if (!tdb->read_only && tdb->header.rwlocks) {
308 ret = tdb_spinunlock(tdb, list, ltype);
310 ret = tdb_brlock(tdb, FREELIST_TOP+4*list, F_UNLCK, F_SETLKW, 0);
315 tdb->locked[list+1].count--;
318 TDB_LOG((tdb, 0,"tdb_unlock: An error occurred unlocking!\n"));
322 /* This is based on the hash algorithm from gdbm */
323 static u32 tdb_hash(TDB_DATA *key)
325 u32 value; /* Used to compute the hash value. */
326 u32 i; /* Used to cycle through random values. */
328 /* Set the initial value from the key size. */
329 for (value = 0x238F13AF * key->dsize, i=0; i < key->dsize; i++)
330 value = (value + (key->dptr[i] << (i*5 % 24)));
332 return (1103515243 * value + 12345);
335 /* check for an out of bounds access - if it is out of bounds then
336 see if the database has been expanded by someone else and expand
338 note that "len" is the minimum length needed for the db
340 static int tdb_oob(TDB_CONTEXT *tdb, tdb_off len, int probe)
343 if (len <= tdb->map_size)
345 if (tdb->flags & TDB_INTERNAL) {
347 /* Ensure ecode is set for log fn. */
348 tdb->ecode = TDB_ERR_IO;
349 TDB_LOG((tdb, 0,"tdb_oob len %d beyond internal malloc size %d\n",
350 (int)len, (int)tdb->map_size));
352 return TDB_ERRCODE(TDB_ERR_IO, -1);
355 if (fstat(tdb->fd, &st) == -1)
356 return TDB_ERRCODE(TDB_ERR_IO, -1);
358 if (st.st_size < (size_t)len) {
360 /* Ensure ecode is set for log fn. */
361 tdb->ecode = TDB_ERR_IO;
362 TDB_LOG((tdb, 0,"tdb_oob len %d beyond eof at %d\n",
363 (int)len, (int)st.st_size));
365 return TDB_ERRCODE(TDB_ERR_IO, -1);
368 /* Unmap, update size, remap */
369 if (tdb_munmap(tdb) == -1)
370 return TDB_ERRCODE(TDB_ERR_IO, -1);
371 tdb->map_size = st.st_size;
376 /* write a lump of data at a specified offset */
377 static int tdb_write(TDB_CONTEXT *tdb, tdb_off off, void *buf, tdb_len len)
379 if (tdb_oob(tdb, off + len, 0) != 0)
383 memcpy(off + (char *)tdb->map_ptr, buf, len);
385 else if (pwrite(tdb->fd, buf, len, off) != (ssize_t)len) {
387 else if (lseek(tdb->fd, off, SEEK_SET) != off
388 || write(tdb->fd, buf, len) != (ssize_t)len) {
390 /* Ensure ecode is set for log fn. */
391 tdb->ecode = TDB_ERR_IO;
392 TDB_LOG((tdb, 0,"tdb_write failed at %d len=%d (%s)\n",
393 off, len, strerror(errno)));
394 return TDB_ERRCODE(TDB_ERR_IO, -1);
399 /* read a lump of data at a specified offset, maybe convert */
400 static int tdb_read(TDB_CONTEXT *tdb,tdb_off off,void *buf,tdb_len len,int cv)
402 if (tdb_oob(tdb, off + len, 0) != 0)
406 memcpy(buf, off + (char *)tdb->map_ptr, len);
408 else if (pread(tdb->fd, buf, len, off) != (ssize_t)len) {
410 else if (lseek(tdb->fd, off, SEEK_SET) != off
411 || read(tdb->fd, buf, len) != (ssize_t)len) {
413 /* Ensure ecode is set for log fn. */
414 tdb->ecode = TDB_ERR_IO;
415 TDB_LOG((tdb, 0,"tdb_read failed at %d len=%d (%s)\n",
416 off, len, strerror(errno)));
417 return TDB_ERRCODE(TDB_ERR_IO, -1);
424 /* read a lump of data, allocating the space for it */
425 static char *tdb_alloc_read(TDB_CONTEXT *tdb, tdb_off offset, tdb_len len)
429 if (!(buf = malloc(len))) {
430 /* Ensure ecode is set for log fn. */
431 tdb->ecode = TDB_ERR_OOM;
432 TDB_LOG((tdb, 0,"tdb_alloc_read malloc failed len=%d (%s)\n",
433 len, strerror(errno)));
434 return TDB_ERRCODE(TDB_ERR_OOM, buf);
436 if (tdb_read(tdb, offset, buf, len, 0) == -1) {
443 /* read/write a tdb_off */
444 static int ofs_read(TDB_CONTEXT *tdb, tdb_off offset, tdb_off *d)
446 return tdb_read(tdb, offset, (char*)d, sizeof(*d), DOCONV());
448 static int ofs_write(TDB_CONTEXT *tdb, tdb_off offset, tdb_off *d)
451 return tdb_write(tdb, offset, CONVERT(off), sizeof(*d));
454 /* read/write a record */
455 static int rec_read(TDB_CONTEXT *tdb, tdb_off offset, struct list_struct *rec)
457 if (tdb_read(tdb, offset, rec, sizeof(*rec),DOCONV()) == -1)
459 if (TDB_BAD_MAGIC(rec)) {
460 /* Ensure ecode is set for log fn. */
461 tdb->ecode = TDB_ERR_CORRUPT;
462 TDB_LOG((tdb, 0,"rec_read bad magic 0x%x at offset=%d\n", rec->magic, offset));
463 return TDB_ERRCODE(TDB_ERR_CORRUPT, -1);
465 return tdb_oob(tdb, rec->next+sizeof(*rec), 0);
467 static int rec_write(TDB_CONTEXT *tdb, tdb_off offset, struct list_struct *rec)
469 struct list_struct r = *rec;
470 return tdb_write(tdb, offset, CONVERT(r), sizeof(r));
473 /* read a freelist record and check for simple errors */
474 static int rec_free_read(TDB_CONTEXT *tdb, tdb_off off, struct list_struct *rec)
476 if (tdb_read(tdb, off, rec, sizeof(*rec),DOCONV()) == -1)
479 if (rec->magic == TDB_MAGIC) {
480 /* this happens when a app is showdown while deleting a record - we should
481 not completely fail when this happens */
482 TDB_LOG((tdb, 0,"rec_free_read non-free magic at offset=%d - fixing\n",
484 rec->magic = TDB_FREE_MAGIC;
485 if (tdb_write(tdb, off, rec, sizeof(*rec)) == -1)
489 if (rec->magic != TDB_FREE_MAGIC) {
490 /* Ensure ecode is set for log fn. */
491 tdb->ecode = TDB_ERR_CORRUPT;
492 TDB_LOG((tdb, 0,"rec_free_read bad magic 0x%x at offset=%d\n",
494 return TDB_ERRCODE(TDB_ERR_CORRUPT, -1);
496 if (tdb_oob(tdb, rec->next+sizeof(*rec), 0) != 0)
501 /* update a record tailer (must hold allocation lock) */
502 static int update_tailer(TDB_CONTEXT *tdb, tdb_off offset,
503 const struct list_struct *rec)
507 /* Offset of tailer from record header */
508 totalsize = sizeof(*rec) + rec->rec_len;
509 return ofs_write(tdb, offset + totalsize - sizeof(tdb_off),
513 static tdb_off tdb_dump_record(TDB_CONTEXT *tdb, tdb_off offset)
515 struct list_struct rec;
516 tdb_off tailer_ofs, tailer;
518 if (tdb_read(tdb, offset, (char *)&rec, sizeof(rec), DOCONV()) == -1) {
519 printf("ERROR: failed to read record at %u\n", offset);
523 printf(" rec: offset=%u next=%d rec_len=%d key_len=%d data_len=%d full_hash=0x%x magic=0x%x\n",
524 offset, rec.next, rec.rec_len, rec.key_len, rec.data_len, rec.full_hash, rec.magic);
526 tailer_ofs = offset + sizeof(rec) + rec.rec_len - sizeof(tdb_off);
527 if (ofs_read(tdb, tailer_ofs, &tailer) == -1) {
528 printf("ERROR: failed to read tailer at %u\n", tailer_ofs);
532 if (tailer != rec.rec_len + sizeof(rec)) {
533 printf("ERROR: tailer does not match record! tailer=%u totalsize=%u\n",
534 (unsigned)tailer, (unsigned)(rec.rec_len + sizeof(rec)));
539 static int tdb_dump_chain(TDB_CONTEXT *tdb, int i)
541 tdb_off rec_ptr, top;
543 top = TDB_HASH_TOP(i);
545 if (tdb_lock(tdb, i, F_WRLCK) != 0)
548 if (ofs_read(tdb, top, &rec_ptr) == -1)
549 return tdb_unlock(tdb, i, F_WRLCK);
552 printf("hash=%d\n", i);
555 rec_ptr = tdb_dump_record(tdb, rec_ptr);
558 return tdb_unlock(tdb, i, F_WRLCK);
561 void tdb_dump_all(TDB_CONTEXT *tdb)
564 for (i=0;i<tdb->header.hash_size;i++) {
565 tdb_dump_chain(tdb, i);
567 printf("freelist:\n");
568 tdb_dump_chain(tdb, -1);
571 int tdb_printfreelist(TDB_CONTEXT *tdb)
575 tdb_off offset, rec_ptr;
576 struct list_struct rec;
578 if ((ret = tdb_lock(tdb, -1, F_WRLCK)) != 0)
581 offset = FREELIST_TOP;
583 /* read in the freelist top */
584 if (ofs_read(tdb, offset, &rec_ptr) == -1) {
585 tdb_unlock(tdb, -1, F_WRLCK);
589 printf("freelist top=[0x%08x]\n", rec_ptr );
591 if (tdb_read(tdb, rec_ptr, (char *)&rec, sizeof(rec), DOCONV()) == -1) {
592 tdb_unlock(tdb, -1, F_WRLCK);
596 if (rec.magic != TDB_FREE_MAGIC) {
597 printf("bad magic 0x%08x in free list\n", rec.magic);
598 tdb_unlock(tdb, -1, F_WRLCK);
602 printf("entry offset=[0x%08x], rec.rec_len = [0x%08x (%d)]\n", rec.next, rec.rec_len, rec.rec_len );
603 total_free += rec.rec_len;
605 /* move to the next record */
608 printf("total rec_len = [0x%08x (%d)]\n", (int)total_free,
611 return tdb_unlock(tdb, -1, F_WRLCK);
614 /* Remove an element from the freelist. Must have alloc lock. */
615 static int remove_from_freelist(TDB_CONTEXT *tdb, tdb_off off, tdb_off next)
619 /* read in the freelist top */
620 last_ptr = FREELIST_TOP;
621 while (ofs_read(tdb, last_ptr, &i) != -1 && i != 0) {
623 /* We've found it! */
624 return ofs_write(tdb, last_ptr, &next);
626 /* Follow chain (next offset is at start of record) */
629 TDB_LOG((tdb, 0,"remove_from_freelist: not on list at off=%d\n", off));
630 return TDB_ERRCODE(TDB_ERR_CORRUPT, -1);
633 /* Add an element into the freelist. Merge adjacent records if
635 static int tdb_free(TDB_CONTEXT *tdb, tdb_off offset, struct list_struct *rec)
639 /* Allocation and tailer lock */
640 if (tdb_lock(tdb, -1, F_WRLCK) != 0)
643 /* set an initial tailer, so if we fail we don't leave a bogus record */
644 if (update_tailer(tdb, offset, rec) != 0) {
645 TDB_LOG((tdb, 0, "tdb_free: upfate_tailer failed!\n"));
649 /* Look right first (I'm an Australian, dammit) */
650 right = offset + sizeof(*rec) + rec->rec_len;
651 if (right + sizeof(*rec) <= tdb->map_size) {
652 struct list_struct r;
654 if (tdb_read(tdb, right, &r, sizeof(r), DOCONV()) == -1) {
655 TDB_LOG((tdb, 0, "tdb_free: right read failed at %u\n", right));
659 /* If it's free, expand to include it. */
660 if (r.magic == TDB_FREE_MAGIC) {
661 if (remove_from_freelist(tdb, right, r.next) == -1) {
662 TDB_LOG((tdb, 0, "tdb_free: right free failed at %u\n", right));
665 rec->rec_len += sizeof(r) + r.rec_len;
671 left = offset - sizeof(tdb_off);
672 if (left > TDB_HASH_TOP(tdb->header.hash_size-1)) {
673 struct list_struct l;
676 /* Read in tailer and jump back to header */
677 if (ofs_read(tdb, left, &leftsize) == -1) {
678 TDB_LOG((tdb, 0, "tdb_free: left offset read failed at %u\n", left));
681 left = offset - leftsize;
683 /* Now read in record */
684 if (tdb_read(tdb, left, &l, sizeof(l), DOCONV()) == -1) {
685 TDB_LOG((tdb, 0, "tdb_free: left read failed at %u (%u)\n", left, leftsize));
689 /* If it's free, expand to include it. */
690 if (l.magic == TDB_FREE_MAGIC) {
691 if (remove_from_freelist(tdb, left, l.next) == -1) {
692 TDB_LOG((tdb, 0, "tdb_free: left free failed at %u\n", left));
696 rec->rec_len += leftsize;
702 if (update_tailer(tdb, offset, rec) == -1) {
703 TDB_LOG((tdb, 0, "tdb_free: update_tailer failed at %u\n", offset));
707 /* Now, prepend to free list */
708 rec->magic = TDB_FREE_MAGIC;
710 if (ofs_read(tdb, FREELIST_TOP, &rec->next) == -1 ||
711 rec_write(tdb, offset, rec) == -1 ||
712 ofs_write(tdb, FREELIST_TOP, &offset) == -1) {
713 TDB_LOG((tdb, 0, "tdb_free record write failed at offset=%d\n", offset));
717 /* And we're done. */
718 tdb_unlock(tdb, -1, F_WRLCK);
722 tdb_unlock(tdb, -1, F_WRLCK);
727 /* expand a file. we prefer to use ftruncate, as that is what posix
728 says to use for mmap expansion */
729 static int expand_file(TDB_CONTEXT *tdb, tdb_off size, tdb_off addition)
732 #if HAVE_FTRUNCATE_EXTEND
733 if (ftruncate(tdb->fd, size+addition) != 0) {
734 TDB_LOG((tdb, 0, "expand_file ftruncate to %d failed (%s)\n",
735 size+addition, strerror(errno)));
742 if (pwrite(tdb->fd, &b, 1, (size+addition) - 1) != 1) {
744 if (lseek(tdb->fd, (size+addition) - 1, SEEK_SET) != (size+addition) - 1 ||
745 write(tdb->fd, &b, 1) != 1) {
747 TDB_LOG((tdb, 0, "expand_file to %d failed (%s)\n",
748 size+addition, strerror(errno)));
753 /* now fill the file with something. This ensures that the file isn't sparse, which would be
754 very bad if we ran out of disk. This must be done with write, not via mmap */
755 memset(buf, 0x42, sizeof(buf));
757 int n = addition>sizeof(buf)?sizeof(buf):addition;
759 int ret = pwrite(tdb->fd, buf, n, size);
762 if (lseek(tdb->fd, size, SEEK_SET) != size)
764 ret = write(tdb->fd, buf, n);
767 TDB_LOG((tdb, 0, "expand_file write of %d failed (%s)\n",
768 n, strerror(errno)));
778 /* expand the database at least size bytes by expanding the underlying
779 file and doing the mmap again if necessary */
780 static int tdb_expand(TDB_CONTEXT *tdb, tdb_off size)
782 struct list_struct rec;
785 if (tdb_lock(tdb, -1, F_WRLCK) == -1) {
786 TDB_LOG((tdb, 0, "lock failed in tdb_expand\n"));
790 /* must know about any previous expansions by another process */
791 tdb_oob(tdb, tdb->map_size + 1, 1);
793 /* always make room for at least 10 more records, and round
794 the database up to a multiple of TDB_PAGE_SIZE */
795 size = TDB_ALIGN(tdb->map_size + size*10, TDB_PAGE_SIZE) - tdb->map_size;
797 if (!(tdb->flags & TDB_INTERNAL))
801 * We must ensure the file is unmapped before doing this
802 * to ensure consistency with systems like OpenBSD where
803 * writes and mmaps are not consistent.
806 /* expand the file itself */
807 if (!(tdb->flags & TDB_INTERNAL)) {
808 if (expand_file(tdb, tdb->map_size, size) != 0)
812 tdb->map_size += size;
814 if (tdb->flags & TDB_INTERNAL)
815 tdb->map_ptr = realloc(tdb->map_ptr, tdb->map_size);
818 * We must ensure the file is remapped before adding the space
819 * to ensure consistency with systems like OpenBSD where
820 * writes and mmaps are not consistent.
823 /* We're ok if the mmap fails as we'll fallback to read/write */
827 /* form a new freelist record */
828 memset(&rec,'\0',sizeof(rec));
829 rec.rec_len = size - sizeof(rec);
831 /* link it into the free list */
832 offset = tdb->map_size - size;
833 if (tdb_free(tdb, offset, &rec) == -1)
836 tdb_unlock(tdb, -1, F_WRLCK);
839 tdb_unlock(tdb, -1, F_WRLCK);
843 /* allocate some space from the free list. The offset returned points
844 to a unconnected list_struct within the database with room for at
845 least length bytes of total data
847 0 is returned if the space could not be allocated
849 static tdb_off tdb_allocate(TDB_CONTEXT *tdb, tdb_len length,
850 struct list_struct *rec)
852 tdb_off rec_ptr, last_ptr, newrec_ptr;
853 struct list_struct newrec;
855 if (tdb_lock(tdb, -1, F_WRLCK) == -1)
858 /* Extra bytes required for tailer */
859 length += sizeof(tdb_off);
862 last_ptr = FREELIST_TOP;
864 /* read in the freelist top */
865 if (ofs_read(tdb, FREELIST_TOP, &rec_ptr) == -1)
868 /* keep looking until we find a freelist record big enough */
870 if (rec_free_read(tdb, rec_ptr, rec) == -1)
873 if (rec->rec_len >= length) {
874 /* found it - now possibly split it up */
875 if (rec->rec_len > length + MIN_REC_SIZE) {
876 /* Length of left piece */
877 length = TDB_ALIGN(length, TDB_ALIGNMENT);
879 /* Right piece to go on free list */
880 newrec.rec_len = rec->rec_len
881 - (sizeof(*rec) + length);
882 newrec_ptr = rec_ptr + sizeof(*rec) + length;
884 /* And left record is shortened */
885 rec->rec_len = length;
889 /* Remove allocated record from the free list */
890 if (ofs_write(tdb, last_ptr, &rec->next) == -1)
893 /* Update header: do this before we drop alloc
894 lock, otherwise tdb_free() might try to
895 merge with us, thinking we're free.
896 (Thanks Jeremy Allison). */
897 rec->magic = TDB_MAGIC;
898 if (rec_write(tdb, rec_ptr, rec) == -1)
901 /* Did we create new block? */
903 /* Update allocated record tailer (we
905 if (update_tailer(tdb, rec_ptr, rec) == -1)
908 /* Free new record */
909 if (tdb_free(tdb, newrec_ptr, &newrec) == -1)
913 /* all done - return the new record offset */
914 tdb_unlock(tdb, -1, F_WRLCK);
917 /* move to the next record */
921 /* we didn't find enough space. See if we can expand the
922 database and if we can then try again */
923 if (tdb_expand(tdb, length + sizeof(*rec)) == 0)
926 tdb_unlock(tdb, -1, F_WRLCK);
930 /* initialise a new database with a specified hash size */
931 static int tdb_new_database(TDB_CONTEXT *tdb, int hash_size)
933 struct tdb_header *newdb;
936 /* We make it up in memory, then write it out if not internal */
937 size = sizeof(struct tdb_header) + (hash_size+1)*sizeof(tdb_off);
938 if (!(newdb = calloc(size, 1)))
939 return TDB_ERRCODE(TDB_ERR_OOM, -1);
941 /* Fill in the header */
942 newdb->version = TDB_VERSION;
943 newdb->hash_size = hash_size;
945 newdb->rwlocks = size;
947 if (tdb->flags & TDB_INTERNAL) {
948 tdb->map_size = size;
949 tdb->map_ptr = (char *)newdb;
950 memcpy(&tdb->header, newdb, sizeof(tdb->header));
951 /* Convert the `ondisk' version if asked. */
955 if (lseek(tdb->fd, 0, SEEK_SET) == -1)
958 if (ftruncate(tdb->fd, 0) == -1)
961 /* This creates an endian-converted header, as if read from disk */
963 memcpy(&tdb->header, newdb, sizeof(tdb->header));
964 /* Don't endian-convert the magic food! */
965 memcpy(newdb->magic_food, TDB_MAGIC_FOOD, strlen(TDB_MAGIC_FOOD)+1);
966 if (write(tdb->fd, newdb, size) != size)
969 ret = tdb_create_rwlocks(tdb->fd, hash_size);
976 /* Returns 0 on fail. On success, return offset of record, and fills
978 static tdb_off tdb_find(TDB_CONTEXT *tdb, TDB_DATA key, u32 hash,
979 struct list_struct *r)
983 /* read in the hash top */
984 if (ofs_read(tdb, TDB_HASH_TOP(hash), &rec_ptr) == -1)
987 /* keep looking until we find the right record */
989 if (rec_read(tdb, rec_ptr, r) == -1)
992 if (!TDB_DEAD(r) && hash==r->full_hash && key.dsize==r->key_len) {
994 /* a very likely hit - read the key */
995 k = tdb_alloc_read(tdb, rec_ptr + sizeof(*r),
1000 if (memcmp(key.dptr, k, key.dsize) == 0) {
1008 return TDB_ERRCODE(TDB_ERR_NOEXIST, 0);
1011 /* If they do lockkeys, check that this hash is one they locked */
1012 static int tdb_keylocked(TDB_CONTEXT *tdb, u32 hash)
1015 if (!tdb->lockedkeys)
1017 for (i = 0; i < tdb->lockedkeys[0]; i++)
1018 if (tdb->lockedkeys[i+1] == hash)
1020 return TDB_ERRCODE(TDB_ERR_NOLOCK, 0);
1023 /* As tdb_find, but if you succeed, keep the lock */
1024 static tdb_off tdb_find_lock(TDB_CONTEXT *tdb, TDB_DATA key, int locktype,
1025 struct list_struct *rec)
1029 hash = tdb_hash(&key);
1030 if (!tdb_keylocked(tdb, hash))
1032 if (tdb_lock(tdb, BUCKET(hash), locktype) == -1)
1034 if (!(rec_ptr = tdb_find(tdb, key, hash, rec)))
1035 tdb_unlock(tdb, BUCKET(hash), locktype);
1039 enum TDB_ERROR tdb_error(TDB_CONTEXT *tdb)
1044 static struct tdb_errname {
1045 enum TDB_ERROR ecode; const char *estring;
1046 } emap[] = { {TDB_SUCCESS, "Success"},
1047 {TDB_ERR_CORRUPT, "Corrupt database"},
1048 {TDB_ERR_IO, "IO Error"},
1049 {TDB_ERR_LOCK, "Locking error"},
1050 {TDB_ERR_OOM, "Out of memory"},
1051 {TDB_ERR_EXISTS, "Record exists"},
1052 {TDB_ERR_NOLOCK, "Lock exists on other keys"},
1053 {TDB_ERR_NOEXIST, "Record does not exist"} };
1055 /* Error string for the last tdb error */
1056 const char *tdb_errorstr(TDB_CONTEXT *tdb)
1059 for (i = 0; i < sizeof(emap) / sizeof(struct tdb_errname); i++)
1060 if (tdb->ecode == emap[i].ecode)
1061 return emap[i].estring;
1062 return "Invalid error code";
1065 /* update an entry in place - this only works if the new data size
1066 is <= the old data size and the key exists.
1067 on failure return -1.
1070 static int tdb_update(TDB_CONTEXT *tdb, TDB_DATA key, TDB_DATA dbuf)
1072 struct list_struct rec;
1076 if (!(rec_ptr = tdb_find(tdb, key, tdb_hash(&key), &rec)))
1079 /* must be long enough key, data and tailer */
1080 if (rec.rec_len < key.dsize + dbuf.dsize + sizeof(tdb_off)) {
1081 tdb->ecode = TDB_SUCCESS; /* Not really an error */
1085 if (tdb_write(tdb, rec_ptr + sizeof(rec) + rec.key_len,
1086 dbuf.dptr, dbuf.dsize) == -1)
1089 if (dbuf.dsize != rec.data_len) {
1091 rec.data_len = dbuf.dsize;
1092 return rec_write(tdb, rec_ptr, &rec);
1098 /* find an entry in the database given a key */
1099 /* If an entry doesn't exist tdb_err will be set to
1100 * TDB_ERR_NOEXIST. If a key has no data attached
1101 * tdb_err will not be set. Both will return a
1102 * zero pptr and zero dsize.
1105 TDB_DATA tdb_fetch(TDB_CONTEXT *tdb, TDB_DATA key)
1108 struct list_struct rec;
1111 /* find which hash bucket it is in */
1112 if (!(rec_ptr = tdb_find_lock(tdb,key,F_RDLCK,&rec)))
1116 ret.dptr = tdb_alloc_read(tdb, rec_ptr + sizeof(rec) + rec.key_len,
1120 ret.dsize = rec.data_len;
1121 tdb_unlock(tdb, BUCKET(rec.full_hash), F_RDLCK);
1125 /* check if an entry in the database exists
1127 note that 1 is returned if the key is found and 0 is returned if not found
1128 this doesn't match the conventions in the rest of this module, but is
1129 compatible with gdbm
1131 int tdb_exists(TDB_CONTEXT *tdb, TDB_DATA key)
1133 struct list_struct rec;
1135 if (tdb_find_lock(tdb, key, F_RDLCK, &rec) == 0)
1137 tdb_unlock(tdb, BUCKET(rec.full_hash), F_RDLCK);
1141 /* record lock stops delete underneath */
1142 static int lock_record(TDB_CONTEXT *tdb, tdb_off off)
1144 return off ? tdb_brlock(tdb, off, F_RDLCK, F_SETLKW, 0) : 0;
1147 Write locks override our own fcntl readlocks, so check it here.
1148 Note this is meant to be F_SETLK, *not* F_SETLKW, as it's not
1149 an error to fail to get the lock here.
1152 static int write_lock_record(TDB_CONTEXT *tdb, tdb_off off)
1154 struct tdb_traverse_lock *i;
1155 for (i = &tdb->travlocks; i; i = i->next)
1158 return tdb_brlock(tdb, off, F_WRLCK, F_SETLK, 1);
1162 Note this is meant to be F_SETLK, *not* F_SETLKW, as it's not
1163 an error to fail to get the lock here.
1166 static int write_unlock_record(TDB_CONTEXT *tdb, tdb_off off)
1168 return tdb_brlock(tdb, off, F_UNLCK, F_SETLK, 0);
1170 /* fcntl locks don't stack: avoid unlocking someone else's */
1171 static int unlock_record(TDB_CONTEXT *tdb, tdb_off off)
1173 struct tdb_traverse_lock *i;
1178 for (i = &tdb->travlocks; i; i = i->next)
1181 return (count == 1 ? tdb_brlock(tdb, off, F_UNLCK, F_SETLKW, 0) : 0);
1184 /* actually delete an entry in the database given the offset */
1185 static int do_delete(TDB_CONTEXT *tdb, tdb_off rec_ptr, struct list_struct*rec)
1187 tdb_off last_ptr, i;
1188 struct list_struct lastrec;
1190 if (tdb->read_only) return -1;
1192 if (write_lock_record(tdb, rec_ptr) == -1) {
1193 /* Someone traversing here: mark it as dead */
1194 rec->magic = TDB_DEAD_MAGIC;
1195 return rec_write(tdb, rec_ptr, rec);
1197 if (write_unlock_record(tdb, rec_ptr) != 0)
1200 /* find previous record in hash chain */
1201 if (ofs_read(tdb, TDB_HASH_TOP(rec->full_hash), &i) == -1)
1203 for (last_ptr = 0; i != rec_ptr; last_ptr = i, i = lastrec.next)
1204 if (rec_read(tdb, i, &lastrec) == -1)
1207 /* unlink it: next ptr is at start of record. */
1209 last_ptr = TDB_HASH_TOP(rec->full_hash);
1210 if (ofs_write(tdb, last_ptr, &rec->next) == -1)
1213 /* recover the space */
1214 if (tdb_free(tdb, rec_ptr, rec) == -1)
1219 /* Uses traverse lock: 0 = finish, -1 = error, other = record offset */
1220 static int tdb_next_lock(TDB_CONTEXT *tdb, struct tdb_traverse_lock *tlock,
1221 struct list_struct *rec)
1223 int want_next = (tlock->off != 0);
1225 /* No traversal allows if you've called tdb_lockkeys() */
1226 if (tdb->lockedkeys)
1227 return TDB_ERRCODE(TDB_ERR_NOLOCK, -1);
1229 /* Lock each chain from the start one. */
1230 for (; tlock->hash < tdb->header.hash_size; tlock->hash++) {
1231 if (tdb_lock(tdb, tlock->hash, F_WRLCK) == -1)
1234 /* No previous record? Start at top of chain. */
1236 if (ofs_read(tdb, TDB_HASH_TOP(tlock->hash),
1240 /* Otherwise unlock the previous record. */
1241 if (unlock_record(tdb, tlock->off) != 0)
1246 /* We have offset of old record: grab next */
1247 if (rec_read(tdb, tlock->off, rec) == -1)
1249 tlock->off = rec->next;
1252 /* Iterate through chain */
1253 while( tlock->off) {
1255 if (rec_read(tdb, tlock->off, rec) == -1)
1257 if (!TDB_DEAD(rec)) {
1258 /* Woohoo: we found one! */
1259 if (lock_record(tdb, tlock->off) != 0)
1263 /* Try to clean dead ones from old traverses */
1264 current = tlock->off;
1265 tlock->off = rec->next;
1266 if (!tdb->read_only &&
1267 do_delete(tdb, current, rec) != 0)
1270 tdb_unlock(tdb, tlock->hash, F_WRLCK);
1273 /* We finished iteration without finding anything */
1274 return TDB_ERRCODE(TDB_SUCCESS, 0);
1278 if (tdb_unlock(tdb, tlock->hash, F_WRLCK) != 0)
1279 TDB_LOG((tdb, 0, "tdb_next_lock: On error unlock failed!\n"));
1283 /* traverse the entire database - calling fn(tdb, key, data) on each element.
1284 return -1 on error or the record count traversed
1285 if fn is NULL then it is not called
1286 a non-zero return value from fn() indicates that the traversal should stop
1288 int tdb_traverse(TDB_CONTEXT *tdb, tdb_traverse_func fn, void *state)
1291 struct list_struct rec;
1292 struct tdb_traverse_lock tl = { NULL, 0, 0 };
1295 /* This was in the initializaton, above, but the IRIX compiler
1296 * did not like it. crh
1298 tl.next = tdb->travlocks.next;
1300 /* fcntl locks don't stack: beware traverse inside traverse */
1301 tdb->travlocks.next = &tl;
1303 /* tdb_next_lock places locks on the record returned, and its chain */
1304 while ((ret = tdb_next_lock(tdb, &tl, &rec)) > 0) {
1306 /* now read the full record */
1307 key.dptr = tdb_alloc_read(tdb, tl.off + sizeof(rec),
1308 rec.key_len + rec.data_len);
1311 if (tdb_unlock(tdb, tl.hash, F_WRLCK) != 0)
1313 if (unlock_record(tdb, tl.off) != 0)
1314 TDB_LOG((tdb, 0, "tdb_traverse: key.dptr == NULL and unlock_record failed!\n"));
1317 key.dsize = rec.key_len;
1318 dbuf.dptr = key.dptr + rec.key_len;
1319 dbuf.dsize = rec.data_len;
1321 /* Drop chain lock, call out */
1322 if (tdb_unlock(tdb, tl.hash, F_WRLCK) != 0) {
1326 if (fn && fn(tdb, key, dbuf, state)) {
1327 /* They want us to terminate traversal */
1329 if (unlock_record(tdb, tl.off) != 0) {
1330 TDB_LOG((tdb, 0, "tdb_traverse: unlock_record failed!\n"));;
1333 tdb->travlocks.next = tl.next;
1334 SAFE_FREE(key.dptr);
1337 SAFE_FREE(key.dptr);
1340 tdb->travlocks.next = tl.next;
1347 /* find the first entry in the database and return its key */
1348 TDB_DATA tdb_firstkey(TDB_CONTEXT *tdb)
1351 struct list_struct rec;
1353 /* release any old lock */
1354 if (unlock_record(tdb, tdb->travlocks.off) != 0)
1356 tdb->travlocks.off = tdb->travlocks.hash = 0;
1358 if (tdb_next_lock(tdb, &tdb->travlocks, &rec) <= 0)
1360 /* now read the key */
1361 key.dsize = rec.key_len;
1362 key.dptr =tdb_alloc_read(tdb,tdb->travlocks.off+sizeof(rec),key.dsize);
1363 if (tdb_unlock(tdb, BUCKET(tdb->travlocks.hash), F_WRLCK) != 0)
1364 TDB_LOG((tdb, 0, "tdb_firstkey: error occurred while tdb_unlocking!\n"));
1368 /* find the next entry in the database, returning its key */
1369 TDB_DATA tdb_nextkey(TDB_CONTEXT *tdb, TDB_DATA oldkey)
1372 TDB_DATA key = tdb_null;
1373 struct list_struct rec;
1376 /* Is locked key the old key? If so, traverse will be reliable. */
1377 if (tdb->travlocks.off) {
1378 if (tdb_lock(tdb,tdb->travlocks.hash,F_WRLCK))
1380 if (rec_read(tdb, tdb->travlocks.off, &rec) == -1
1381 || !(k = tdb_alloc_read(tdb,tdb->travlocks.off+sizeof(rec),
1383 || memcmp(k, oldkey.dptr, oldkey.dsize) != 0) {
1384 /* No, it wasn't: unlock it and start from scratch */
1385 if (unlock_record(tdb, tdb->travlocks.off) != 0)
1387 if (tdb_unlock(tdb, tdb->travlocks.hash, F_WRLCK) != 0)
1389 tdb->travlocks.off = 0;
1395 if (!tdb->travlocks.off) {
1396 /* No previous element: do normal find, and lock record */
1397 tdb->travlocks.off = tdb_find_lock(tdb, oldkey, F_WRLCK, &rec);
1398 if (!tdb->travlocks.off)
1400 tdb->travlocks.hash = BUCKET(rec.full_hash);
1401 if (lock_record(tdb, tdb->travlocks.off) != 0) {
1402 TDB_LOG((tdb, 0, "tdb_nextkey: lock_record failed (%s)!\n", strerror(errno)));
1406 oldhash = tdb->travlocks.hash;
1408 /* Grab next record: locks chain and returned record,
1409 unlocks old record */
1410 if (tdb_next_lock(tdb, &tdb->travlocks, &rec) > 0) {
1411 key.dsize = rec.key_len;
1412 key.dptr = tdb_alloc_read(tdb, tdb->travlocks.off+sizeof(rec),
1414 /* Unlock the chain of this new record */
1415 if (tdb_unlock(tdb, tdb->travlocks.hash, F_WRLCK) != 0)
1416 TDB_LOG((tdb, 0, "tdb_nextkey: WARNING tdb_unlock failed!\n"));
1418 /* Unlock the chain of old record */
1419 if (tdb_unlock(tdb, BUCKET(oldhash), F_WRLCK) != 0)
1420 TDB_LOG((tdb, 0, "tdb_nextkey: WARNING tdb_unlock failed!\n"));
1424 /* delete an entry in the database given a key */
1425 int tdb_delete(TDB_CONTEXT *tdb, TDB_DATA key)
1428 struct list_struct rec;
1431 if (!(rec_ptr = tdb_find_lock(tdb, key, F_WRLCK, &rec)))
1433 ret = do_delete(tdb, rec_ptr, &rec);
1434 if (tdb_unlock(tdb, BUCKET(rec.full_hash), F_WRLCK) != 0)
1435 TDB_LOG((tdb, 0, "tdb_delete: WARNING tdb_unlock failed!\n"));
1439 /* store an element in the database, replacing any existing element
1442 return 0 on success, -1 on failure
1444 int tdb_store(TDB_CONTEXT *tdb, TDB_DATA key, TDB_DATA dbuf, int flag)
1446 struct list_struct rec;
1452 /* find which hash bucket it is in */
1453 hash = tdb_hash(&key);
1454 if (!tdb_keylocked(tdb, hash))
1456 if (tdb_lock(tdb, BUCKET(hash), F_WRLCK) == -1)
1459 /* check for it existing, on insert. */
1460 if (flag == TDB_INSERT) {
1461 if (tdb_exists(tdb, key)) {
1462 tdb->ecode = TDB_ERR_EXISTS;
1466 /* first try in-place update, on modify or replace. */
1467 if (tdb_update(tdb, key, dbuf) == 0)
1469 if (flag == TDB_MODIFY && tdb->ecode == TDB_ERR_NOEXIST)
1472 /* reset the error code potentially set by the tdb_update() */
1473 tdb->ecode = TDB_SUCCESS;
1475 /* delete any existing record - if it doesn't exist we don't
1476 care. Doing this first reduces fragmentation, and avoids
1477 coalescing with `allocated' block before it's updated. */
1478 if (flag != TDB_INSERT)
1479 tdb_delete(tdb, key);
1481 /* Copy key+value *before* allocating free space in case malloc
1482 fails and we are left with a dead spot in the tdb. */
1484 if (!(p = (char *)malloc(key.dsize + dbuf.dsize))) {
1485 tdb->ecode = TDB_ERR_OOM;
1489 memcpy(p, key.dptr, key.dsize);
1491 memcpy(p+key.dsize, dbuf.dptr, dbuf.dsize);
1493 /* now we're into insert / modify / replace of a record which
1494 * we know could not be optimised by an in-place store (for
1495 * various reasons). */
1496 if (!(rec_ptr = tdb_allocate(tdb, key.dsize + dbuf.dsize, &rec)))
1499 /* Read hash top into next ptr */
1500 if (ofs_read(tdb, TDB_HASH_TOP(hash), &rec.next) == -1)
1503 rec.key_len = key.dsize;
1504 rec.data_len = dbuf.dsize;
1505 rec.full_hash = hash;
1506 rec.magic = TDB_MAGIC;
1508 /* write out and point the top of the hash chain at it */
1509 if (rec_write(tdb, rec_ptr, &rec) == -1
1510 || tdb_write(tdb, rec_ptr+sizeof(rec), p, key.dsize+dbuf.dsize)==-1
1511 || ofs_write(tdb, TDB_HASH_TOP(hash), &rec_ptr) == -1) {
1512 /* Need to tdb_unallocate() here */
1517 tdb_unlock(tdb, BUCKET(hash), F_WRLCK);
1524 /* Attempt to append data to an entry in place - this only works if the new data size
1525 is <= the old data size and the key exists.
1526 on failure return -1. Record must be locked before calling.
1528 static int tdb_append_inplace(TDB_CONTEXT *tdb, TDB_DATA key, TDB_DATA new_dbuf)
1530 struct list_struct rec;
1534 if (!(rec_ptr = tdb_find(tdb, key, tdb_hash(&key), &rec)))
1537 /* Append of 0 is always ok. */
1538 if (new_dbuf.dsize == 0)
1541 /* must be long enough for key, old data + new data and tailer */
1542 if (rec.rec_len < key.dsize + rec.data_len + new_dbuf.dsize + sizeof(tdb_off)) {
1544 tdb->ecode = TDB_SUCCESS; /* Not really an error */
1548 if (tdb_write(tdb, rec_ptr + sizeof(rec) + rec.key_len + rec.data_len,
1549 new_dbuf.dptr, new_dbuf.dsize) == -1)
1553 rec.data_len += new_dbuf.dsize;
1554 return rec_write(tdb, rec_ptr, &rec);
1557 /* Append to an entry. Create if not exist. */
1559 int tdb_append(TDB_CONTEXT *tdb, TDB_DATA key, TDB_DATA new_dbuf)
1561 struct list_struct rec;
1566 size_t new_data_size = 0;
1568 /* find which hash bucket it is in */
1569 hash = tdb_hash(&key);
1570 if (!tdb_keylocked(tdb, hash))
1572 if (tdb_lock(tdb, BUCKET(hash), F_WRLCK) == -1)
1575 /* first try in-place. */
1576 if (tdb_append_inplace(tdb, key, new_dbuf) == 0)
1579 /* reset the error code potentially set by the tdb_append_inplace() */
1580 tdb->ecode = TDB_SUCCESS;
1583 if (!(rec_ptr = tdb_find(tdb, key, hash, &rec))) {
1584 if (tdb->ecode != TDB_ERR_NOEXIST)
1587 /* Not found - create. */
1589 ret = tdb_store(tdb, key, new_dbuf, TDB_INSERT);
1593 new_data_size = rec.data_len + new_dbuf.dsize;
1595 /* Copy key+old_value+value *before* allocating free space in case malloc
1596 fails and we are left with a dead spot in the tdb. */
1598 if (!(p = (char *)malloc(key.dsize + new_data_size))) {
1599 tdb->ecode = TDB_ERR_OOM;
1603 /* Copy the key in place. */
1604 memcpy(p, key.dptr, key.dsize);
1606 /* Now read the old data into place. */
1608 tdb_read(tdb, rec_ptr + sizeof(rec) + rec.key_len, p + key.dsize, rec.data_len, 0) == -1)
1611 /* Finally append the new data. */
1613 memcpy(p+key.dsize+rec.data_len, new_dbuf.dptr, new_dbuf.dsize);
1615 /* delete any existing record - if it doesn't exist we don't
1616 care. Doing this first reduces fragmentation, and avoids
1617 coalescing with `allocated' block before it's updated. */
1619 tdb_delete(tdb, key);
1621 if (!(rec_ptr = tdb_allocate(tdb, key.dsize + new_data_size, &rec)))
1624 /* Read hash top into next ptr */
1625 if (ofs_read(tdb, TDB_HASH_TOP(hash), &rec.next) == -1)
1628 rec.key_len = key.dsize;
1629 rec.data_len = new_data_size;
1630 rec.full_hash = hash;
1631 rec.magic = TDB_MAGIC;
1633 /* write out and point the top of the hash chain at it */
1634 if (rec_write(tdb, rec_ptr, &rec) == -1
1635 || tdb_write(tdb, rec_ptr+sizeof(rec), p, key.dsize+new_data_size)==-1
1636 || ofs_write(tdb, TDB_HASH_TOP(hash), &rec_ptr) == -1) {
1637 /* Need to tdb_unallocate() here */
1643 tdb_unlock(tdb, BUCKET(hash), F_WRLCK);
1651 static int tdb_already_open(dev_t device,
1656 for (i = tdbs; i; i = i->next) {
1657 if (i->device == device && i->inode == ino) {
1665 /* open the database, creating it if necessary
1667 The open_flags and mode are passed straight to the open call on the
1668 database file. A flags value of O_WRONLY is invalid. The hash size
1669 is advisory, use zero for a default value.
1671 Return is NULL on error, in which case errno is also set. Don't
1672 try to call tdb_error or tdb_errname, just do strerror(errno).
1674 @param name may be NULL for internal databases. */
1675 TDB_CONTEXT *tdb_open(const char *name, int hash_size, int tdb_flags,
1676 int open_flags, mode_t mode)
1678 return tdb_open_ex(name, hash_size, tdb_flags, open_flags, mode, NULL);
1682 TDB_CONTEXT *tdb_open_ex(const char *name, int hash_size, int tdb_flags,
1683 int open_flags, mode_t mode,
1684 tdb_log_func log_fn)
1688 int rev = 0, locked;
1692 if (!(tdb = calloc(1, sizeof *tdb))) {
1693 /* Can't log this */
1699 tdb->map_ptr = NULL;
1700 tdb->lockedkeys = NULL;
1701 tdb->flags = tdb_flags;
1702 tdb->open_flags = open_flags;
1703 tdb->log_fn = log_fn;
1705 if ((open_flags & O_ACCMODE) == O_WRONLY) {
1706 TDB_LOG((tdb, 0, "tdb_open_ex: can't open tdb %s write-only\n",
1713 hash_size = DEFAULT_HASH_SIZE;
1714 if ((open_flags & O_ACCMODE) == O_RDONLY) {
1716 /* read only databases don't do locking or clear if first */
1717 tdb->flags |= TDB_NOLOCK;
1718 tdb->flags &= ~TDB_CLEAR_IF_FIRST;
1721 /* internal databases don't mmap or lock, and start off cleared */
1722 if (tdb->flags & TDB_INTERNAL) {
1723 tdb->flags |= (TDB_NOLOCK | TDB_NOMMAP);
1724 tdb->flags &= ~TDB_CLEAR_IF_FIRST;
1725 if (tdb_new_database(tdb, hash_size) != 0) {
1726 TDB_LOG((tdb, 0, "tdb_open_ex: tdb_new_database failed!"));
1732 if ((tdb->fd = open(name, open_flags, mode)) == -1) {
1733 TDB_LOG((tdb, 5, "tdb_open_ex: could not open file %s: %s\n",
1734 name, strerror(errno)));
1735 goto fail; /* errno set by open(2) */
1738 /* ensure there is only one process initialising at once */
1739 if (tdb_brlock(tdb, GLOBAL_LOCK, F_WRLCK, F_SETLKW, 0) == -1) {
1740 TDB_LOG((tdb, 0, "tdb_open_ex: failed to get global lock on %s: %s\n",
1741 name, strerror(errno)));
1742 goto fail; /* errno set by tdb_brlock */
1745 /* we need to zero database if we are the only one with it open */
1746 if ((locked = (tdb_brlock(tdb, ACTIVE_LOCK, F_WRLCK, F_SETLK, 0) == 0))
1747 && (tdb_flags & TDB_CLEAR_IF_FIRST)) {
1748 open_flags |= O_CREAT;
1749 if (ftruncate(tdb->fd, 0) == -1) {
1750 TDB_LOG((tdb, 0, "tdb_open_ex: "
1751 "failed to truncate %s: %s\n",
1752 name, strerror(errno)));
1753 goto fail; /* errno set by ftruncate */
1757 if (read(tdb->fd, &tdb->header, sizeof(tdb->header)) != sizeof(tdb->header)
1758 || strcmp(tdb->header.magic_food, TDB_MAGIC_FOOD) != 0
1759 || (tdb->header.version != TDB_VERSION
1760 && !(rev = (tdb->header.version==TDB_BYTEREV(TDB_VERSION))))) {
1761 /* its not a valid database - possibly initialise it */
1762 if (!(open_flags & O_CREAT) || tdb_new_database(tdb, hash_size) == -1) {
1763 errno = EIO; /* ie bad format or something */
1766 rev = (tdb->flags & TDB_CONVERT);
1768 vp = (unsigned char *)&tdb->header.version;
1769 vertest = (((u32)vp[0]) << 24) | (((u32)vp[1]) << 16) |
1770 (((u32)vp[2]) << 8) | (u32)vp[3];
1771 tdb->flags |= (vertest==TDB_VERSION) ? TDB_BIGENDIAN : 0;
1773 tdb->flags &= ~TDB_CONVERT;
1775 tdb->flags |= TDB_CONVERT;
1776 convert(&tdb->header, sizeof(tdb->header));
1778 if (fstat(tdb->fd, &st) == -1)
1781 /* Is it already in the open list? If so, fail. */
1782 if (tdb_already_open(st.st_dev, st.st_ino)) {
1783 TDB_LOG((tdb, 2, "tdb_open_ex: "
1784 "%s (%d,%d) is already open in this process\n",
1785 name, st.st_dev, st.st_ino));
1790 if (!(tdb->name = (char *)strdup(name))) {
1795 tdb->map_size = st.st_size;
1796 tdb->device = st.st_dev;
1797 tdb->inode = st.st_ino;
1798 tdb->locked = calloc(tdb->header.hash_size+1, sizeof(tdb->locked[0]));
1800 TDB_LOG((tdb, 2, "tdb_open_ex: "
1801 "failed to allocate lock structure for %s\n",
1808 if (!tdb->read_only)
1809 if (tdb_clear_spinlocks(tdb) != 0) {
1810 TDB_LOG((tdb, 0, "tdb_open_ex: "
1811 "failed to clear spinlock\n"));
1814 if (tdb_brlock(tdb, ACTIVE_LOCK, F_UNLCK, F_SETLK, 0) == -1) {
1815 TDB_LOG((tdb, 0, "tdb_open_ex: "
1816 "failed to take ACTIVE_LOCK on %s: %s\n",
1817 name, strerror(errno)));
1821 /* leave this lock in place to indicate it's in use */
1822 if (tdb_brlock(tdb, ACTIVE_LOCK, F_RDLCK, F_SETLKW, 0) == -1)
1826 /* Internal (memory-only) databases skip all the code above to
1827 * do with disk files, and resume here by releasing their
1828 * global lock and hooking into the active list. */
1829 if (tdb_brlock(tdb, GLOBAL_LOCK, F_UNLCK, F_SETLKW, 0) == -1)
1836 { int save_errno = errno;
1842 if (tdb->flags & TDB_INTERNAL)
1843 SAFE_FREE(tdb->map_ptr);
1847 SAFE_FREE(tdb->name);
1849 if (close(tdb->fd) != 0)
1850 TDB_LOG((tdb, 5, "tdb_open_ex: failed to close tdb->fd on error!\n"));
1851 SAFE_FREE(tdb->locked);
1861 * @returns -1 for error; 0 for success.
1863 int tdb_close(TDB_CONTEXT *tdb)
1869 if (tdb->flags & TDB_INTERNAL)
1870 SAFE_FREE(tdb->map_ptr);
1874 SAFE_FREE(tdb->name);
1876 ret = close(tdb->fd);
1877 SAFE_FREE(tdb->locked);
1878 SAFE_FREE(tdb->lockedkeys);
1880 /* Remove from contexts list */
1881 for (i = &tdbs; *i; i = &(*i)->next) {
1888 memset(tdb, 0, sizeof(*tdb));
1894 /* lock/unlock entire database */
1895 int tdb_lockall(TDB_CONTEXT *tdb)
1899 /* There are no locks on read-only dbs */
1901 return TDB_ERRCODE(TDB_ERR_LOCK, -1);
1902 if (tdb->lockedkeys)
1903 return TDB_ERRCODE(TDB_ERR_NOLOCK, -1);
1904 for (i = 0; i < tdb->header.hash_size; i++)
1905 if (tdb_lock(tdb, i, F_WRLCK))
1908 /* If error, release locks we have... */
1909 if (i < tdb->header.hash_size) {
1912 for ( j = 0; j < i; j++)
1913 tdb_unlock(tdb, j, F_WRLCK);
1914 return TDB_ERRCODE(TDB_ERR_NOLOCK, -1);
1919 void tdb_unlockall(TDB_CONTEXT *tdb)
1922 for (i=0; i < tdb->header.hash_size; i++)
1923 tdb_unlock(tdb, i, F_WRLCK);
1926 int tdb_lockkeys(TDB_CONTEXT *tdb, u32 number, TDB_DATA keys[])
1930 /* Can't lock more keys if already locked */
1931 if (tdb->lockedkeys)
1932 return TDB_ERRCODE(TDB_ERR_NOLOCK, -1);
1933 if (!(tdb->lockedkeys = malloc(sizeof(u32) * (number+1))))
1934 return TDB_ERRCODE(TDB_ERR_OOM, -1);
1935 /* First number in array is # keys */
1936 tdb->lockedkeys[0] = number;
1938 /* Insertion sort by bucket */
1939 for (i = 0; i < number; i++) {
1940 hash = tdb_hash(&keys[i]);
1941 for (j = 0; j < i && BUCKET(tdb->lockedkeys[j+1]) < BUCKET(hash); j++);
1942 memmove(&tdb->lockedkeys[j+2], &tdb->lockedkeys[j+1], sizeof(u32) * (i-j));
1943 tdb->lockedkeys[j+1] = hash;
1945 /* Finally, lock in order */
1946 for (i = 0; i < number; i++)
1947 if (tdb_lock(tdb, i, F_WRLCK))
1950 /* If error, release locks we have... */
1952 for ( j = 0; j < i; j++)
1953 tdb_unlock(tdb, j, F_WRLCK);
1954 SAFE_FREE(tdb->lockedkeys);
1955 return TDB_ERRCODE(TDB_ERR_NOLOCK, -1);
1960 /* Unlock the keys previously locked by tdb_lockkeys() */
1961 void tdb_unlockkeys(TDB_CONTEXT *tdb)
1964 if (!tdb->lockedkeys)
1966 for (i = 0; i < tdb->lockedkeys[0]; i++)
1967 tdb_unlock(tdb, tdb->lockedkeys[i+1], F_WRLCK);
1968 SAFE_FREE(tdb->lockedkeys);
1971 /* lock/unlock one hash chain. This is meant to be used to reduce
1972 contention - it cannot guarantee how many records will be locked */
1973 int tdb_chainlock(TDB_CONTEXT *tdb, TDB_DATA key)
1975 return tdb_lock(tdb, BUCKET(tdb_hash(&key)), F_WRLCK);
1978 int tdb_chainunlock(TDB_CONTEXT *tdb, TDB_DATA key)
1980 return tdb_unlock(tdb, BUCKET(tdb_hash(&key)), F_WRLCK);
1983 int tdb_chainlock_read(TDB_CONTEXT *tdb, TDB_DATA key)
1985 return tdb_lock(tdb, BUCKET(tdb_hash(&key)), F_RDLCK);
1988 int tdb_chainunlock_read(TDB_CONTEXT *tdb, TDB_DATA key)
1990 return tdb_unlock(tdb, BUCKET(tdb_hash(&key)), F_RDLCK);
1994 /* register a loging function */
1995 void tdb_logging_function(TDB_CONTEXT *tdb, void (*fn)(TDB_CONTEXT *, int , const char *, ...))
2001 /* reopen a tdb - this is used after a fork to ensure that we have an independent
2002 seek pointer from our parent and to re-establish locks */
2003 int tdb_reopen(TDB_CONTEXT *tdb)
2007 if (tdb_munmap(tdb) != 0) {
2008 TDB_LOG((tdb, 0, "tdb_reopen: munmap failed (%s)\n", strerror(errno)));
2011 if (close(tdb->fd) != 0)
2012 TDB_LOG((tdb, 0, "tdb_reopen: WARNING closing tdb->fd failed!\n"));
2013 tdb->fd = open(tdb->name, tdb->open_flags & ~(O_CREAT|O_TRUNC), 0);
2014 if (tdb->fd == -1) {
2015 TDB_LOG((tdb, 0, "tdb_reopen: open failed (%s)\n", strerror(errno)));
2018 if (fstat(tdb->fd, &st) != 0) {
2019 TDB_LOG((tdb, 0, "tdb_reopen: fstat failed (%s)\n", strerror(errno)));
2022 if (st.st_ino != tdb->inode || st.st_dev != tdb->device) {
2023 TDB_LOG((tdb, 0, "tdb_reopen: file dev/inode has changed!\n"));
2027 if (tdb_brlock(tdb, ACTIVE_LOCK, F_RDLCK, F_SETLKW, 0) == -1) {
2028 TDB_LOG((tdb, 0, "tdb_reopen: failed to obtain active lock\n"));
2039 /* reopen all tdb's */
2040 int tdb_reopen_all(void)
2044 for (tdb=tdbs; tdb; tdb = tdb->next) {
2045 if (tdb_reopen(tdb) != 0) return -1;