1 /* $Id: cnfs.c 7412 2005-10-09 03:44:35Z eagle $
3 ** Cyclic News File System.
8 #include "portable/mmap.h"
9 #include "portable/time.h"
16 #include <netinet/in.h>
21 #include "inn/innconf.h"
22 #include "interface.h"
30 #include "cnfs-private.h"
32 /* Temporary until cnfs_mapcntl is handled like mapcntl. Make MS_ASYNC
33 disappear on platforms that don't have it. */
38 /* We can give a more descriptive error below about not having largefile support if the platform has EOVERFLOW; on other platforms some other
39 * errno will be used and so we won't know when to give the descriptive
46 /**** Stuff to be cleaned up when we're done with the article */
47 char *base; /* Base of mmap()ed art */
48 int len; /* Length of article (and thus
50 CYCBUFF *cycbuff; /* pointer to current CYCBUFF */
51 off_t offset; /* offset to current article */
52 bool rollover; /* true if the search is rollovered */
55 static char LocalLogName[] = "CNFS-sm";
56 static CYCBUFF *cycbufftab = (CYCBUFF *)NULL;
57 static METACYCBUFF *metacycbufftab = (METACYCBUFF *)NULL;
58 static CNFSEXPIRERULES *metaexprulestab = (CNFSEXPIRERULES *)NULL;
59 static long pagesize = 0;
60 static int metabuff_update = METACYCBUFF_UPDATE;
61 static int refresh_interval = REFRESH_INTERVAL;
63 static TOKEN CNFSMakeToken(char *cycbuffname, off_t offset,
64 uint32_t cycnum, STORAGECLASS class) {
69 ** XXX We'll assume that TOKENSIZE is 16 bytes and that we divvy it
70 ** up as: 8 bytes for cycbuffname, 4 bytes for offset, 4 bytes
71 ** for cycnum. See also: CNFSBreakToken() for hard-coded constants.
73 token.type = TOKEN_CNFS;
75 memcpy(token.token, cycbuffname, CNFSMAXCYCBUFFNAME);
76 int32 = htonl(offset / CNFS_BLOCKSIZE);
77 memcpy(&token.token[8], &int32, sizeof(int32));
78 int32 = htonl(cycnum);
79 memcpy(&token.token[12], &int32, sizeof(int32));
84 ** NOTE: We assume that cycbuffname is 9 bytes long.
87 static bool CNFSBreakToken(TOKEN token, char *cycbuffname,
88 off_t *offset, uint32_t *cycnum) {
91 if (cycbuffname == NULL || offset == NULL || cycnum == NULL) {
92 syslog(L_ERROR, "%s: BreakToken: invalid argument: %s",
93 LocalLogName, cycbuffname);
94 SMseterror(SMERR_INTERNAL, "BreakToken: invalid argument");
97 memcpy(cycbuffname, token.token, CNFSMAXCYCBUFFNAME);
98 *(cycbuffname + CNFSMAXCYCBUFFNAME) = '\0'; /* Just to be paranoid */
99 memcpy(&int32, &token.token[8], sizeof(int32));
100 *offset = (off_t)ntohl(int32) * (off_t)CNFS_BLOCKSIZE;
101 memcpy(&int32, &token.token[12], sizeof(int32));
102 *cycnum = ntohl(int32);
106 static char hextbl[] = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9',
107 'a', 'b', 'c', 'd', 'e', 'f'};
110 ** CNFSofft2hex -- Given an argument of type off_t, return
111 ** a static ASCII string representing its value in hexadecimal.
113 ** If "leadingzeros" is true, the number returned will have leading 0's.
116 static char * CNFSofft2hex(off_t offset, bool leadingzeros) {
120 if (sizeof(off_t) <= 4) {
121 snprintf(buf, sizeof(buf), (leadingzeros) ? "%016lx" : "%lx", offset);
125 for (i = 0; i < CNFSLASIZ; i++)
126 buf[i] = '0'; /* Pad with zeros to start */
127 for (i = CNFSLASIZ - 1; i >= 0; i--) {
128 buf[i] = hextbl[offset & 0xf];
132 if (! leadingzeros) {
133 for (p = buf; *p == '0'; p++)
138 return p - 1; /* We converted a "0" and then bypassed all
145 ** CNFShex2offt -- Given an ASCII string containing a hexadecimal representation
146 ** of a off_t, return a off_t.
149 static off_t CNFShex2offt(char *hex) {
150 if (sizeof(off_t) <= 4) {
151 unsigned long rpofft;
153 sscanf(hex, "%lx", &rpofft);
159 for (; *hex != '\0'; hex++) {
160 if (*hex >= '0' && *hex <= '9')
162 else if (*hex >= 'a' && *hex <= 'f')
164 else if (*hex >= 'A' && *hex <= 'F')
168 ** We used to have a syslog() message here, but the case
169 ** where we land here because of a ":" happens, er, often.
174 if (isalnum((int)*(hex + 1)))
181 static bool CNFSflushhead(CYCBUFF *cycbuff) {
184 if (!cycbuff->needflush)
187 syslog(L_ERROR, "%s: CNFSflushhead: attempted flush whilst read only",
191 memset(&rpx, 0, sizeof(CYCBUFFEXTERN));
192 if (cycbuff->magicver == 3) {
193 cycbuff->updated = time(NULL);
194 strncpy(rpx.magic, CNFS_MAGICV3, strlen(CNFS_MAGICV3));
195 strncpy(rpx.name, cycbuff->name, CNFSNASIZ);
196 strncpy(rpx.path, cycbuff->path, CNFSPASIZ);
197 /* Don't use sprintf() directly ... the terminating '\0' causes grief */
198 strncpy(rpx.lena, CNFSofft2hex(cycbuff->len, true), CNFSLASIZ);
199 strncpy(rpx.freea, CNFSofft2hex(cycbuff->free, true), CNFSLASIZ);
200 strncpy(rpx.cyclenuma, CNFSofft2hex(cycbuff->cyclenum, true), CNFSLASIZ);
201 strncpy(rpx.updateda, CNFSofft2hex(cycbuff->updated, true), CNFSLASIZ);
202 strncpy(rpx.metaname, cycbuff->metaname, CNFSNASIZ);
203 strncpy(rpx.orderinmeta, CNFSofft2hex(cycbuff->order, true), CNFSLASIZ);
204 if (cycbuff->currentbuff) {
205 strncpy(rpx.currentbuff, "TRUE", CNFSMASIZ);
207 strncpy(rpx.currentbuff, "FALSE", CNFSMASIZ);
209 memcpy(cycbuff->bitfield, &rpx, sizeof(CYCBUFFEXTERN));
210 msync(cycbuff->bitfield, cycbuff->minartoffset, MS_ASYNC);
211 cycbuff->needflush = false;
213 syslog(L_ERROR, "%s: CNFSflushhead: bogus magicver for %s: %d",
214 LocalLogName, cycbuff->name, cycbuff->magicver);
220 static void CNFSshutdowncycbuff(CYCBUFF *cycbuff) {
221 if (cycbuff == (CYCBUFF *)NULL)
223 if (cycbuff->needflush) {
224 syslog(L_NOTICE, "%s: CNFSshutdowncycbuff: flushing %s", LocalLogName, cycbuff->name);
225 CNFSflushhead(cycbuff);
227 if (cycbuff->bitfield != NULL) {
228 munmap(cycbuff->bitfield, cycbuff->minartoffset);
229 cycbuff->bitfield = NULL;
231 if (cycbuff->fd >= 0)
236 static void CNFScleancycbuff(void) {
237 CYCBUFF *cycbuff, *nextcycbuff;
239 for (cycbuff = cycbufftab; cycbuff != (CYCBUFF *)NULL;) {
240 CNFSshutdowncycbuff(cycbuff);
241 nextcycbuff = cycbuff->next;
243 cycbuff = nextcycbuff;
245 cycbufftab = (CYCBUFF *)NULL;
248 static void CNFScleanmetacycbuff(void) {
249 METACYCBUFF *metacycbuff, *nextmetacycbuff;
251 for (metacycbuff = metacycbufftab; metacycbuff != (METACYCBUFF *)NULL;) {
252 nextmetacycbuff = metacycbuff->next;
253 free(metacycbuff->members);
254 free(metacycbuff->name);
256 metacycbuff = nextmetacycbuff;
258 metacycbufftab = (METACYCBUFF *)NULL;
261 static void CNFScleanexpirerule(void) {
262 CNFSEXPIRERULES *metaexprule, *nextmetaexprule;
264 for (metaexprule = metaexprulestab; metaexprule != (CNFSEXPIRERULES *)NULL;) {
265 nextmetaexprule = metaexprule->next;
267 metaexprule = nextmetaexprule;
269 metaexprulestab = (CNFSEXPIRERULES *)NULL;
272 static CYCBUFF *CNFSgetcycbuffbyname(char *name) {
277 for (cycbuff = cycbufftab; cycbuff != (CYCBUFF *)NULL; cycbuff = cycbuff->next)
278 if (strcmp(name, cycbuff->name) == 0)
283 static METACYCBUFF *CNFSgetmetacycbuffbyname(char *name) {
284 METACYCBUFF *metacycbuff;
288 for (metacycbuff = metacycbufftab; metacycbuff != (METACYCBUFF *)NULL; metacycbuff = metacycbuff->next)
289 if (strcmp(name, metacycbuff->name) == 0)
294 static void CNFSflushallheads(void) {
297 for (cycbuff = cycbufftab; cycbuff != (CYCBUFF *)NULL; cycbuff = cycbuff->next) {
298 if (cycbuff->needflush)
299 syslog(L_NOTICE, "%s: CNFSflushallheads: flushing %s", LocalLogName, cycbuff->name);
300 CNFSflushhead(cycbuff);
305 ** CNFSReadFreeAndCycle() -- Read from disk the current values of CYCBUFF's
306 ** free pointer and cycle number. Return 1 on success, 0 otherwise.
309 static void CNFSReadFreeAndCycle(CYCBUFF *cycbuff) {
313 memcpy(&rpx, cycbuff->bitfield, sizeof(CYCBUFFEXTERN));
314 /* Sanity checks are not needed since CNFSinit_disks() has already done. */
315 strncpy(buf, rpx.freea, CNFSLASIZ);
316 buf[CNFSLASIZ] = '\0';
317 cycbuff->free = CNFShex2offt(buf);
318 strncpy(buf, rpx.updateda, CNFSLASIZ);
319 buf[CNFSLASIZ] = '\0';
320 cycbuff->updated = CNFShex2offt(buf);
321 strncpy(buf, rpx.cyclenuma, CNFSLASIZ);
322 buf[CNFSLASIZ] = '\0';
323 cycbuff->cyclenum = CNFShex2offt(buf);
327 static bool CNFSparse_part_line(char *l) {
330 off_t len, minartoffset;
332 CYCBUFF *cycbuff, *tmp;
334 /* Symbolic cnfs partition name */
335 if ((p = strchr(l, ':')) == NULL || p - l <= 0 || p - l > CNFSMAXCYCBUFFNAME - 1) {
336 syslog(L_ERROR, "%s: bad cycbuff name in line '%s'", LocalLogName, l);
340 if (CNFSgetcycbuffbyname(l) != NULL) {
342 syslog(L_ERROR, "%s: duplicate cycbuff name in line '%s'", LocalLogName, l);
345 cycbuff = xmalloc(sizeof(CYCBUFF));
346 memset(cycbuff->name, '\0', CNFSNASIZ);
347 strlcpy(cycbuff->name, l, CNFSNASIZ);
350 /* Path to cnfs partition */
351 if ((p = strchr(l, ':')) == NULL || p - l <= 0 || p - l > CNFSPASIZ - 1) {
352 syslog(L_ERROR, "%s: bad pathname in line '%s'", LocalLogName, l);
357 memset(cycbuff->path, '\0', CNFSPASIZ);
358 strlcpy(cycbuff->path, l, CNFSPASIZ);
359 if (stat(cycbuff->path, &sb) < 0) {
360 if (errno == EOVERFLOW) {
361 syslog(L_ERROR, "%s: file '%s' : %s, ignoring '%s' cycbuff",
362 LocalLogName, cycbuff->path,
363 "Overflow (probably >2GB without largefile support)",
366 syslog(L_ERROR, "%s: file '%s' : %m, ignoring '%s' cycbuff",
367 LocalLogName, cycbuff->path, cycbuff->name);
374 /* Length/size of symbolic partition */
375 len = strtoul(l, NULL, 10) * (off_t)1024; /* This value in KB in decimal */
376 if (S_ISREG(sb.st_mode) && len != sb.st_size) {
377 if (sizeof(CYCBUFFEXTERN) > (size_t) sb.st_size) {
378 syslog(L_NOTICE, "%s: length must be at least '%lu' for '%s' cycbuff(%lu bytes)",
379 LocalLogName, (unsigned long) sizeof(CYCBUFFEXTERN), cycbuff->name,
380 (unsigned long) sb.st_size);
387 cycbuff->next = (CYCBUFF *)NULL;
388 cycbuff->needflush = false;
389 cycbuff->bitfield = NULL;
391 ** The minimum article offset will be the size of the bitfield itself,
392 ** len / (blocksize * 8), plus however many additional blocks the CYCBUFF
393 ** external header occupies ... then round up to the next block.
396 cycbuff->len / (CNFS_BLOCKSIZE * 8) + CNFS_BEFOREBITF;
397 tonextblock = CNFS_HDR_PAGESIZE - (minartoffset & (CNFS_HDR_PAGESIZE - 1));
398 cycbuff->minartoffset = minartoffset + tonextblock;
400 if (cycbufftab == (CYCBUFF *)NULL)
401 cycbufftab = cycbuff;
403 for (tmp = cycbufftab; tmp->next != (CYCBUFF *)NULL; tmp = tmp->next);
410 static bool CNFSparse_metapart_line(char *l) {
411 char *p, *cycbuff, *q = l;
413 METACYCBUFF *metacycbuff, *tmp;
415 /* Symbolic metacycbuff name */
416 if ((p = strchr(l, ':')) == NULL || p - l <= 0) {
417 syslog(L_ERROR, "%s: bad partition name in line '%s'", LocalLogName, l);
421 if (CNFSgetmetacycbuffbyname(l) != NULL) {
423 syslog(L_ERROR, "%s: duplicate metabuff name in line '%s'", LocalLogName, l);
426 metacycbuff = xmalloc(sizeof(METACYCBUFF));
427 metacycbuff->members = (CYCBUFF **)NULL;
428 metacycbuff->count = 0;
429 metacycbuff->name = xstrdup(l);
430 metacycbuff->next = (METACYCBUFF *)NULL;
431 metacycbuff->metamode = INTERLEAVE;
434 if ((p = strchr(l, ':')) != NULL) {
436 syslog(L_ERROR, "%s: bad mode in line '%s'", LocalLogName, q);
439 if (strcmp(++p, "INTERLEAVE") == 0)
440 metacycbuff->metamode = INTERLEAVE;
441 else if (strcmp(p, "SEQUENTIAL") == 0)
442 metacycbuff->metamode = SEQUENTIAL;
444 syslog(L_ERROR, "%s: unknown mode in line '%s'", LocalLogName, q);
450 while ((p = strchr(l, ',')) != NULL && p - l > 0) {
454 if ((rp = CNFSgetcycbuffbyname(cycbuff)) == NULL) {
455 syslog(L_ERROR, "%s: bogus cycbuff '%s' (metacycbuff '%s')",
456 LocalLogName, cycbuff, metacycbuff->name);
457 free(metacycbuff->members);
458 free(metacycbuff->name);
462 if (metacycbuff->count == 0)
463 metacycbuff->members = xmalloc(sizeof(CYCBUFF *));
465 metacycbuff->members = xrealloc(metacycbuff->members, (metacycbuff->count + 1) * sizeof(CYCBUFF *));
466 metacycbuff->members[metacycbuff->count++] = rp;
468 /* Gotta deal with the last cycbuff on the list */
470 if ((rp = CNFSgetcycbuffbyname(cycbuff)) == NULL) {
471 syslog(L_ERROR, "%s: bogus cycbuff '%s' (metacycbuff '%s')",
472 LocalLogName, cycbuff, metacycbuff->name);
473 free(metacycbuff->members);
474 free(metacycbuff->name);
478 if (metacycbuff->count == 0)
479 metacycbuff->members = xmalloc(sizeof(CYCBUFF *));
481 metacycbuff->members = xrealloc(metacycbuff->members, (metacycbuff->count + 1) * sizeof(CYCBUFF *));
482 metacycbuff->members[metacycbuff->count++] = rp;
485 if (metacycbuff->count == 0) {
486 syslog(L_ERROR, "%s: no cycbuffs assigned to cycbuff '%s'",
487 LocalLogName, metacycbuff->name);
488 free(metacycbuff->name);
492 if (metacycbufftab == (METACYCBUFF *)NULL)
493 metacycbufftab = metacycbuff;
495 for (tmp = metacycbufftab; tmp->next != (METACYCBUFF *)NULL; tmp = tmp->next);
496 tmp->next = metacycbuff;
502 static bool CNFSparse_groups_line(void) {
504 STORAGE_SUB *sub = (STORAGE_SUB *)NULL;
505 CNFSEXPIRERULES *metaexprule, *tmp;
507 sub = SMGetConfig(TOKEN_CNFS, sub);
508 for (;sub != (STORAGE_SUB *)NULL; sub = SMGetConfig(TOKEN_CNFS, sub)) {
509 if (sub->options == (char *)NULL) {
510 syslog(L_ERROR, "%s: storage.conf options field is missing",
512 CNFScleanexpirerule();
515 if ((mrp = CNFSgetmetacycbuffbyname(sub->options)) == NULL) {
516 syslog(L_ERROR, "%s: storage.conf options field '%s' undefined",
517 LocalLogName, sub->options);
518 CNFScleanexpirerule();
521 metaexprule = xmalloc(sizeof(CNFSEXPIRERULES));
522 metaexprule->class = sub->class;
523 metaexprule->dest = mrp;
524 metaexprule->next = (CNFSEXPIRERULES *)NULL;
525 if (metaexprulestab == (CNFSEXPIRERULES *)NULL)
526 metaexprulestab = metaexprule;
528 for (tmp = metaexprulestab; tmp->next != (CNFSEXPIRERULES *)NULL; tmp = tmp->next);
529 tmp->next = metaexprule;
537 ** CNFSinit_disks -- Finish initializing cycbufftab
538 ** Called by "innd" only -- we open (and keep) a read/write
539 ** file descriptor for each CYCBUFF.
541 ** Calling this function repeatedly shouldn't cause any harm
542 ** speed-wise or bug-wise, as long as the caller is accessing the
543 ** CYCBUFFs _read-only_. If innd calls this function repeatedly,
544 ** bad things will happen.
547 static bool CNFSinit_disks(CYCBUFF *cycbuff) {
555 ** Discover the state of our cycbuffs. If any of them are in icky shape,
556 ** duck shamelessly & return false.
559 if (cycbuff != (CYCBUFF *)NULL)
563 cycbuff = cycbufftab;
565 for (; cycbuff != (CYCBUFF *)NULL; cycbuff = cycbuff->next) {
566 if (strcmp(cycbuff->path, "/dev/null") == 0) {
567 syslog(L_ERROR, "%s: ERROR opening '%s' is not available",
568 LocalLogName, cycbuff->path);
571 if (cycbuff->fd < 0) {
572 if ((fd = open(cycbuff->path, SMopenmode ? O_RDWR : O_RDONLY)) < 0) {
573 syslog(L_ERROR, "%s: ERROR opening '%s' O_RDONLY : %m",
574 LocalLogName, cycbuff->path);
577 close_on_exec(fd, true);
582 cycbuff->bitfield = mmap(NULL, cycbuff->minartoffset,
583 SMopenmode ? (PROT_READ | PROT_WRITE) : PROT_READ,
584 MAP_SHARED, cycbuff->fd, 0);
585 if (cycbuff->bitfield == MAP_FAILED || errno != 0) {
587 "%s: CNFSinitdisks: mmap for %s offset %d len %ld failed: %m",
588 LocalLogName, cycbuff->path, 0, (long) cycbuff->minartoffset);
589 cycbuff->bitfield = NULL;
594 ** Much of this checking from previous revisions is (probably) bogus
595 ** & buggy & particularly icky & unupdated. Use at your own risk. :-)
597 rpx = (CYCBUFFEXTERN *)cycbuff->bitfield;
598 if (strncmp(rpx->magic, CNFS_MAGICV3, strlen(CNFS_MAGICV3)) == 0) {
599 cycbuff->magicver = 3;
600 if (strncmp(rpx->name, cycbuff->name, CNFSNASIZ) != 0) {
601 syslog(L_ERROR, "%s: Mismatch 3: read %s for cycbuff %s", LocalLogName,
602 rpx->name, cycbuff->name);
605 if (strncmp(rpx->path, cycbuff->path, CNFSPASIZ) != 0) {
606 syslog(L_ERROR, "%s: Path mismatch: read %s for cycbuff %s",
607 LocalLogName, rpx->path, cycbuff->path);
609 strncpy(buf, rpx->lena, CNFSLASIZ);
610 buf[CNFSLASIZ] = '\0';
611 tmpo = CNFShex2offt(buf);
612 if (tmpo != cycbuff->len) {
613 syslog(L_ERROR, "%s: Mismatch: read 0x%s length for cycbuff %s",
614 LocalLogName, CNFSofft2hex(tmpo, false), cycbuff->path);
617 strncpy(buf, rpx->freea, CNFSLASIZ);
618 buf[CNFSLASIZ] = '\0';
619 cycbuff->free = CNFShex2offt(buf);
620 strncpy(buf, rpx->updateda, CNFSLASIZ);
621 buf[CNFSLASIZ] = '\0';
622 cycbuff->updated = CNFShex2offt(buf);
623 strncpy(buf, rpx->cyclenuma, CNFSLASIZ);
624 buf[CNFSLASIZ] = '\0';
625 cycbuff->cyclenum = CNFShex2offt(buf);
626 strncpy(cycbuff->metaname, rpx->metaname, CNFSLASIZ);
627 strncpy(buf, rpx->orderinmeta, CNFSLASIZ);
628 cycbuff->order = CNFShex2offt(buf);
629 if (strncmp(rpx->currentbuff, "TRUE", CNFSMASIZ) == 0) {
630 cycbuff->currentbuff = true;
632 cycbuff->currentbuff = false;
635 "%s: No magic cookie found for cycbuff %s, initializing",
636 LocalLogName, cycbuff->name);
637 cycbuff->magicver = 3;
638 cycbuff->free = cycbuff->minartoffset;
639 cycbuff->updated = 0;
640 cycbuff->cyclenum = 1;
641 cycbuff->currentbuff = true;
642 cycbuff->order = 0; /* to indicate this is newly added cycbuff */
643 cycbuff->needflush = true;
644 memset(cycbuff->metaname, '\0', CNFSLASIZ);
645 if (!CNFSflushhead(cycbuff))
654 static bool CNFS_setcurrent(METACYCBUFF *metacycbuff) {
656 int i, currentcycbuff = 0, order = -1;
657 bool foundcurrent = false;
658 for (i = 0 ; i < metacycbuff->count ; i++) {
659 cycbuff = metacycbuff->members[i];
660 if (strncmp(cycbuff->metaname, metacycbuff->name, CNFSNASIZ) != 0) {
661 /* this cycbuff is moved from other metacycbuff , or is new */
662 cycbuff->order = i + 1;
663 cycbuff->currentbuff = false;
664 strncpy(cycbuff->metaname, metacycbuff->name, CNFSLASIZ);
665 cycbuff->needflush = true;
668 if (foundcurrent == false && cycbuff->currentbuff == true) {
672 if (foundcurrent == false || order == -1 || order > cycbuff->order) {
673 /* this cycbuff is a candidate for current cycbuff */
675 order = cycbuff->order;
677 if (cycbuff->order != i + 1) {
678 /* cycbuff order seems to be changed */
679 cycbuff->order = i + 1;
680 cycbuff->needflush = true;
683 /* If no current cycbuff found (say, all our cycbuffs are new) default to 0 */
684 if (foundcurrent == false) {
687 for (i = 0 ; i < metacycbuff->count ; i++) {
688 cycbuff = metacycbuff->members[i];
689 if (currentcycbuff == i && cycbuff->currentbuff == false) {
690 cycbuff->currentbuff = true;
691 cycbuff->needflush = true;
693 if (currentcycbuff != i && cycbuff->currentbuff == true) {
694 cycbuff->currentbuff = false;
695 cycbuff->needflush = true;
697 if (cycbuff->needflush == true && !CNFSflushhead(cycbuff))
700 metacycbuff->memb_next = currentcycbuff;
705 ** CNFSread_config() -- Read the cnfs partition/file configuration file.
707 ** Oh, for the want of Perl! My parser probably shows that I don't use
708 ** C all that often anymore....
711 static bool CNFSread_config(void) {
712 char *path, *config, *from, *to, **ctab = (char **)NULL;
713 int ctab_free = 0; /* Index to next free slot in ctab */
715 bool metacycbufffound = false;
716 bool cycbuffupdatefound = false;
717 bool refreshintervalfound = false;
720 path = concatpath(innconf->pathetc, _PATH_CYCBUFFCONFIG);
721 config = ReadInFile(path, NULL);
722 if (config == NULL) {
723 syslog(L_ERROR, "%s: cannot read %s", LocalLogName, path);
729 for (from = to = config; *from; ) {
730 if (*from == '#') { /* Comment line? */
731 while (*from && *from != '\n')
732 from++; /* Skip past it */
734 continue; /* Back to top of loop */
736 if (*from == '\n') { /* End or just a blank line? */
738 continue; /* Back to top of loop */
741 ctab = xmalloc(sizeof(char *));
743 ctab = xrealloc(ctab, (ctab_free + 1) * sizeof(char *));
744 /* If we're here, we've got the beginning of a real entry */
745 ctab[ctab_free++] = to = from;
747 if (*from && *from == '\\' && *(from + 1) == '\n') {
748 from += 2; /* Skip past backslash+newline */
749 while (*from && isspace((int)*from))
753 if (*from && *from != '\n')
765 for (ctab_i = 0; ctab_i < ctab_free; ctab_i++) {
766 if (strncmp(ctab[ctab_i], "cycbuff:", 8) == 0) {
767 if (metacycbufffound) {
768 syslog(L_ERROR, "%s: all cycbuff entries shoud be before metacycbuff entries", LocalLogName);
773 if (!CNFSparse_part_line(ctab[ctab_i] + 8)) {
778 } else if (strncmp(ctab[ctab_i], "metacycbuff:", 12) == 0) {
779 metacycbufffound = true;
780 if (!CNFSparse_metapart_line(ctab[ctab_i] + 12)) {
785 } else if (strncmp(ctab[ctab_i], "cycbuffupdate:", 14) == 0) {
786 if (cycbuffupdatefound) {
787 syslog(L_ERROR, "%s: duplicate cycbuffupdate entries", LocalLogName);
792 cycbuffupdatefound = true;
793 update = atoi(ctab[ctab_i] + 14);
795 syslog(L_ERROR, "%s: invalid cycbuffupdate", LocalLogName);
801 metabuff_update = METACYCBUFF_UPDATE;
803 metabuff_update = update;
804 } else if (strncmp(ctab[ctab_i], "refreshinterval:", 16) == 0) {
805 if (refreshintervalfound) {
806 syslog(L_ERROR, "%s: duplicate refreshinterval entries", LocalLogName);
811 refreshintervalfound = true;
812 refresh = atoi(ctab[ctab_i] + 16);
814 syslog(L_ERROR, "%s: invalid refreshinterval", LocalLogName);
820 refresh_interval = REFRESH_INTERVAL;
822 refresh_interval = refresh;
824 syslog(L_ERROR, "%s: Bogus metacycbuff config line '%s' ignored",
825 LocalLogName, ctab[ctab_i]);
830 if (!CNFSparse_groups_line()) {
833 if (cycbufftab == (CYCBUFF *)NULL) {
834 syslog(L_ERROR, "%s: zero cycbuffs defined", LocalLogName);
837 if (metacycbufftab == (METACYCBUFF *)NULL) {
838 syslog(L_ERROR, "%s: zero metacycbuffs defined", LocalLogName);
844 /* Figure out what page an address is in and flush those pages */
846 cnfs_mapcntl(void *p, size_t length, int flags)
850 start = (char *)((size_t)p & ~(size_t)(pagesize - 1));
851 end = (char *)((size_t)((char *)p + length + pagesize) &
852 ~(size_t)(pagesize - 1));
853 if (flags == MS_INVALIDATE) {
854 msync(start, end - start, flags);
856 static char *sstart, *send;
858 /* Don't thrash the system with msync()s - keep the last value
859 * and check each time, only if the pages which we should
860 * flush change actually flush the previous ones. Calling
861 * cnfs_mapcntl(NULL, 0, MS_ASYNC) then flushes the final
863 if (start != sstart || end != send) {
864 if (sstart != NULL && send != NULL) {
865 msync(sstart, send - sstart, flags);
874 ** Bit arithmetic by brute force.
876 ** XXXYYYXXX WARNING: the code below is not endian-neutral!
879 typedef unsigned long ULONG;
881 static int CNFSUsedBlock(CYCBUFF *cycbuff, off_t offset,
882 bool set_operation, bool setbitvalue) {
885 int bitoffset; /* From the 'left' side of the long */
886 static int uninitialized = 1;
887 static int longsize = sizeof(long);
889 ULONG bitlong, on, off, mask;
890 static ULONG onarray[64], offarray[64];
897 for (i = (longsize * 8) - 1; i >= 0; i--) {
907 /* We allow bit-setting under minartoffset, but it better be false */
908 if ((offset < cycbuff->minartoffset && setbitvalue) ||
909 offset > cycbuff->len) {
910 char bufoff[64], bufmin[64], bufmax[64];
911 SMseterror(SMERR_INTERNAL, NULL);
912 strlcpy(bufoff, CNFSofft2hex(offset, false), sizeof(bufoff));
913 strlcpy(bufmin, CNFSofft2hex(cycbuff->minartoffset, false),
915 strlcpy(bufmax, CNFSofft2hex(cycbuff->len, false), sizeof(bufmax));
917 "%s: CNFSUsedBlock: invalid offset %s, min = %s, max = %s",
918 LocalLogName, bufoff, bufmin, bufmax);
921 if (offset % CNFS_BLOCKSIZE != 0) {
922 SMseterror(SMERR_INTERNAL, NULL);
924 "%s: CNFSsetusedbitbyrp: offset %s not on %d-byte block boundary",
925 LocalLogName, CNFSofft2hex(offset, false), CNFS_BLOCKSIZE);
928 blocknum = offset / CNFS_BLOCKSIZE;
929 longoffset = blocknum / (longsize * 8);
930 bitoffset = blocknum % (longsize * 8);
931 where = (ULONG *)cycbuff->bitfield + (CNFS_BEFOREBITF / longsize)
936 mask = onarray[bitoffset];
939 mask = offarray[bitoffset];
943 if (innconf->nfswriter) {
944 cnfs_mapcntl(where, sizeof *where, MS_ASYNC);
946 return 2; /* XXX Clean up return semantics */
948 /* It's a read operation */
949 mask = onarray[bitoffset];
952 * return bitlong & mask; doesn't work if sizeof(ulong) > sizeof(int)
954 if ( bitlong & mask ) return 1; else return 0;
958 static int CNFSArtMayBeHere(CYCBUFF *cycbuff, off_t offset, uint32_t cycnum) {
959 static time_t lastupdate = 0;
962 if (SMpreopen && !SMopenmode) {
963 if ((time(NULL) - lastupdate) > refresh_interval) { /* XXX Changed to refresh every 30sec - cmo*/
964 for (tmp = cycbufftab; tmp != (CYCBUFF *)NULL; tmp = tmp->next) {
965 CNFSReadFreeAndCycle(tmp);
967 lastupdate = time(NULL);
968 } else if (cycnum == cycbuff->cyclenum + 1) { /* rollover ? */
969 CNFSReadFreeAndCycle(cycbuff);
973 ** The current cycle number may have advanced since the last time we
974 ** checked it, so use a ">=" check instead of "==". Our intent is
975 ** avoid a false negative response, *not* a false positive response.
977 if (! (cycnum == cycbuff->cyclenum ||
978 (cycnum == cycbuff->cyclenum - 1 && offset > cycbuff->free) ||
979 (cycnum + 1 == 0 && cycbuff->cyclenum == 2 && offset > cycbuff->free))) {
980 /* We've been overwritten */
983 return CNFSUsedBlock(cycbuff, offset, false, false);
986 bool cnfs_init(SMATTRIBUTE *attr) {
987 METACYCBUFF *metacycbuff;
991 syslog(L_ERROR, "%s: attr is NULL", LocalLogName);
992 SMseterror(SMERR_INTERNAL, "attr is NULL");
995 attr->selfexpire = true;
996 attr->expensivestat = false;
997 if (innconf == NULL) {
998 if (!innconf_read(NULL)) {
999 syslog(L_ERROR, "%s: innconf_read failed", LocalLogName);
1000 SMseterror(SMERR_INTERNAL, "ReadInnConf() failed");
1004 if (pagesize == 0) {
1005 pagesize = getpagesize();
1006 if (pagesize == -1) {
1007 syslog(L_ERROR, "%s: getpagesize failed: %m", LocalLogName);
1008 SMseterror(SMERR_INTERNAL, "getpagesize failed");
1012 if ((pagesize > CNFS_HDR_PAGESIZE) || (CNFS_HDR_PAGESIZE % pagesize)) {
1013 syslog(L_ERROR, "%s: CNFS_HDR_PAGESIZE (%d) is not a multiple of pagesize (%ld)", LocalLogName, CNFS_HDR_PAGESIZE, pagesize);
1014 SMseterror(SMERR_INTERNAL, "CNFS_HDR_PAGESIZE not multiple of pagesize");
1018 if (STORAGE_TOKEN_LENGTH < 16) {
1019 syslog(L_ERROR, "%s: token length is less than 16 bytes", LocalLogName);
1020 SMseterror(SMERR_TOKENSHORT, NULL);
1024 if (!CNFSread_config()) {
1026 CNFScleanmetacycbuff();
1027 CNFScleanexpirerule();
1028 SMseterror(SMERR_INTERNAL, NULL);
1031 if (!CNFSinit_disks(NULL)) {
1033 CNFScleanmetacycbuff();
1034 CNFScleanexpirerule();
1035 SMseterror(SMERR_INTERNAL, NULL);
1038 for (metacycbuff = metacycbufftab; metacycbuff != (METACYCBUFF *)NULL; metacycbuff = metacycbuff->next) {
1039 metacycbuff->memb_next = 0;
1040 metacycbuff->write_count = 0; /* Let's not forget this */
1041 if (metacycbuff->metamode == SEQUENTIAL)
1042 /* mark current cycbuff */
1043 if (CNFS_setcurrent(metacycbuff) == false) {
1045 CNFScleanmetacycbuff();
1046 CNFScleanexpirerule();
1047 SMseterror(SMERR_INTERNAL, NULL);
1052 for (cycbuff = cycbufftab; cycbuff != (CYCBUFF *)NULL; cycbuff = cycbuff->next) {
1053 CNFSshutdowncycbuff(cycbuff);
1059 TOKEN cnfs_store(const ARTHANDLE article, const STORAGECLASS class) {
1061 CYCBUFF *cycbuff = NULL;
1062 METACYCBUFF *metacycbuff = NULL;
1064 static char buf[1024];
1065 static char alignbuf[CNFS_BLOCKSIZE];
1066 char *artcycbuffname;
1067 off_t artoffset, middle;
1068 uint32_t artcyclenum;
1070 static struct iovec *iov;
1073 CNFSEXPIRERULES *metaexprule;
1077 for (metaexprule = metaexprulestab; metaexprule != (CNFSEXPIRERULES *)NULL; metaexprule = metaexprule->next) {
1078 if (metaexprule->class == class)
1081 if (metaexprule == (CNFSEXPIRERULES *)NULL) {
1082 SMseterror(SMERR_INTERNAL, "no rules match");
1083 syslog(L_ERROR, "%s: no matches for group '%s'",
1085 token.type = TOKEN_EMPTY;
1088 metacycbuff = metaexprule->dest;
1090 cycbuff = metacycbuff->members[metacycbuff->memb_next];
1091 if (cycbuff == NULL) {
1092 SMseterror(SMERR_INTERNAL, "no cycbuff found");
1093 syslog(L_ERROR, "%s: no cycbuff found for %d", LocalLogName, metacycbuff->memb_next);
1094 token.type = TOKEN_EMPTY;
1096 } else if (!SMpreopen && !CNFSinit_disks(cycbuff)) {
1097 SMseterror(SMERR_INTERNAL, "cycbuff initialization fail");
1098 syslog(L_ERROR, "%s: cycbuff '%s' initialization fail", LocalLogName, cycbuff->name);
1099 token.type = TOKEN_EMPTY;
1103 /* cycbuff->free should have already been aligned by the last write, but
1104 realign it just to be sure. */
1105 tonextblock = CNFS_BLOCKSIZE - (cycbuff->free & (CNFS_BLOCKSIZE - 1));
1106 if (tonextblock != CNFS_BLOCKSIZE)
1107 cycbuff->free += tonextblock;
1109 /* Article too big? */
1110 if (cycbuff->len - cycbuff->free < CNFS_BLOCKSIZE + 1)
1113 left = cycbuff->len - cycbuff->free - CNFS_BLOCKSIZE - 1;
1114 if (article.len > left) {
1115 for (middle = cycbuff->free ;middle < cycbuff->len - CNFS_BLOCKSIZE - 1;
1116 middle += CNFS_BLOCKSIZE) {
1117 CNFSUsedBlock(cycbuff, middle, true, false);
1119 if (innconf->nfswriter) {
1120 cnfs_mapcntl(NULL, 0, MS_ASYNC);
1122 cycbuff->free = cycbuff->minartoffset;
1123 cycbuff->cyclenum++;
1124 if (cycbuff->cyclenum == 0)
1125 cycbuff->cyclenum += 2; /* cnfs_next() needs this */
1126 cycbuff->needflush = true;
1127 if (metacycbuff->metamode == INTERLEAVE) {
1128 CNFSflushhead(cycbuff); /* Flush, just for giggles */
1129 syslog(L_NOTICE, "%s: cycbuff %s rollover to cycle 0x%x... remain calm",
1130 LocalLogName, cycbuff->name, cycbuff->cyclenum);
1133 cycbuff->currentbuff = false;
1134 CNFSflushhead(cycbuff); /* Flush, just for giggles */
1135 if (!SMpreopen) CNFSshutdowncycbuff(cycbuff);
1136 metacycbuff->memb_next = (metacycbuff->memb_next + 1) % metacycbuff->count;
1137 cycbuff = metacycbuff->members[metacycbuff->memb_next];
1138 if (!SMpreopen && !CNFSinit_disks(cycbuff)) {
1139 SMseterror(SMERR_INTERNAL, "cycbuff initialization fail");
1140 syslog(L_ERROR, "%s: cycbuff '%s' initialization fail", LocalLogName, cycbuff->name);
1141 token.type = TOKEN_EMPTY;
1144 syslog(L_NOTICE, "%s: metacycbuff %s cycbuff is moved to %s remain calm",
1145 LocalLogName, metacycbuff->name, cycbuff->name);
1146 cycbuff->currentbuff = true;
1147 cycbuff->needflush = true;
1148 CNFSflushhead(cycbuff); /* Flush, just for giggles */
1152 /* Ah, at least we know all three important data */
1153 artcycbuffname = cycbuff->name;
1154 artoffset = cycbuff->free;
1155 artcyclenum = cycbuff->cyclenum;
1157 memset(&cah, 0, sizeof(cah));
1158 cah.size = htonl(article.len);
1159 if (article.arrived == (time_t)0)
1160 cah.arrived = htonl(time(NULL));
1162 cah.arrived = htonl(article.arrived);
1165 if (lseek(cycbuff->fd, artoffset, SEEK_SET) < 0) {
1166 SMseterror(SMERR_INTERNAL, "lseek failed");
1167 syslog(L_ERROR, "%s: lseek failed for '%s' offset 0x%s: %m",
1168 LocalLogName, cycbuff->name, CNFSofft2hex(artoffset, false));
1169 token.type = TOKEN_EMPTY;
1170 if (!SMpreopen) CNFSshutdowncycbuff(cycbuff);
1174 iov = xmalloc((article.iovcnt + 2) * sizeof(struct iovec));
1175 iovcnt = article.iovcnt + 2;
1176 } else if (iovcnt < article.iovcnt + 2) {
1177 iov = xrealloc(iov, (article.iovcnt + 2) * sizeof(struct iovec));
1178 iovcnt = article.iovcnt + 2;
1180 iov[0].iov_base = (char *) &cah;
1181 iov[0].iov_len = sizeof(cah);
1182 totlen = iov[0].iov_len;
1183 for (i = 1; i <= article.iovcnt; i++) {
1184 iov[i].iov_base = article.iov[i-1].iov_base;
1185 iov[i].iov_len = article.iov[i-1].iov_len;
1186 totlen += iov[i].iov_len;
1188 if ((totlen & (CNFS_BLOCKSIZE - 1)) != 0) {
1189 /* Want to xwritev an exact multiple of CNFS_BLOCKSIZE */
1190 iov[i].iov_base = alignbuf;
1191 iov[i].iov_len = CNFS_BLOCKSIZE - (totlen & (CNFS_BLOCKSIZE - 1));
1192 totlen += iov[i].iov_len;
1195 if (xwritev(cycbuff->fd, iov, i) < 0) {
1196 SMseterror(SMERR_INTERNAL, "cnfs_store() xwritev() failed");
1198 "%s: cnfs_store xwritev failed for '%s' offset 0x%s: %m",
1199 LocalLogName, artcycbuffname, CNFSofft2hex(artoffset, false));
1200 token.type = TOKEN_EMPTY;
1201 if (!SMpreopen) CNFSshutdowncycbuff(cycbuff);
1204 cycbuff->needflush = true;
1206 /* Now that the article is written, advance the free pointer & flush */
1207 cycbuff->free += totlen;
1210 ** If cycbuff->free > cycbuff->len, don't worry. The next cnfs_store()
1211 ** will detect the situation & wrap around correctly.
1213 if (metacycbuff->metamode == INTERLEAVE)
1214 metacycbuff->memb_next = (metacycbuff->memb_next + 1) % metacycbuff->count;
1215 if (++metacycbuff->write_count % metabuff_update == 0) {
1216 for (i = 0; i < metacycbuff->count; i++) {
1217 CNFSflushhead(metacycbuff->members[i]);
1220 CNFSUsedBlock(cycbuff, artoffset, true, true);
1221 for (middle = artoffset + CNFS_BLOCKSIZE; middle < cycbuff->free;
1222 middle += CNFS_BLOCKSIZE) {
1223 CNFSUsedBlock(cycbuff, middle, true, false);
1225 if (innconf->nfswriter) {
1226 cnfs_mapcntl(NULL, 0, MS_ASYNC);
1228 if (!SMpreopen) CNFSshutdowncycbuff(cycbuff);
1229 return CNFSMakeToken(artcycbuffname, artoffset, artcyclenum, class);
1232 ARTHANDLE *cnfs_retrieve(const TOKEN token, const RETRTYPE amount) {
1233 char cycbuffname[9];
1243 static TOKEN ret_token;
1244 static bool nomessage = false;
1247 if (token.type != TOKEN_CNFS) {
1248 SMseterror(SMERR_INTERNAL, NULL);
1251 if (! CNFSBreakToken(token, cycbuffname, &offset, &cycnum)) {
1252 /* SMseterror() should have already been called */
1255 if ((cycbuff = CNFSgetcycbuffbyname(cycbuffname)) == NULL) {
1256 SMseterror(SMERR_NOENT, NULL);
1258 syslog(L_ERROR, "%s: cnfs_retrieve: token %s: bogus cycbuff name: %s:0x%s:%d",
1259 LocalLogName, TokenToText(token), cycbuffname, CNFSofft2hex(offset, false), cycnum);
1264 if (!SMpreopen && !CNFSinit_disks(cycbuff)) {
1265 SMseterror(SMERR_INTERNAL, "cycbuff initialization fail");
1266 syslog(L_ERROR, "%s: cycbuff '%s' initialization fail", LocalLogName, cycbuff->name);
1269 if (! CNFSArtMayBeHere(cycbuff, offset, cycnum)) {
1270 SMseterror(SMERR_NOENT, NULL);
1271 if (!SMpreopen) CNFSshutdowncycbuff(cycbuff);
1275 art = xmalloc(sizeof(ARTHANDLE));
1276 art->type = TOKEN_CNFS;
1277 if (amount == RETR_STAT) {
1280 art->private = NULL;
1282 art->token = &ret_token;
1283 if (!SMpreopen) CNFSshutdowncycbuff(cycbuff);
1287 ** Because we don't know the length of the article (yet), we'll
1288 ** just mmap() a chunk of memory which is guaranteed to be larger
1289 ** than the largest article can be.
1290 ** XXX Because the max article size can be changed, we could get into hot
1291 ** XXX water here. So, to be safe, we double MAX_ART_SIZE and add enough
1292 ** XXX extra for the pagesize fudge factor and CNFSARTHEADER structure.
1294 if (pread(cycbuff->fd, &cah, sizeof(cah), offset) != sizeof(cah)) {
1295 SMseterror(SMERR_UNDEFINED, "read failed");
1296 syslog(L_ERROR, "%s: could not read token %s %s:0x%s:%d: %m",
1297 LocalLogName, TokenToText(token), cycbuffname, CNFSofft2hex(offset, false), cycnum);
1299 if (!SMpreopen) CNFSshutdowncycbuff(cycbuff);
1303 if(cah.size == htonl(0x1234) && ntohl(cah.arrived) < time(NULL)-10*365*24*3600) {
1304 oldCNFSARTHEADER cahh;
1305 *(CNFSARTHEADER *)&cahh = cah;
1306 if(pread(cycbuff->fd, ((char *)&cahh)+sizeof(CNFSARTHEADER), sizeof(oldCNFSARTHEADER)-sizeof(CNFSARTHEADER), offset+sizeof(cah)) != sizeof(oldCNFSARTHEADER)-sizeof(CNFSARTHEADER)) {
1307 SMseterror(SMERR_UNDEFINED, "read2 failed");
1308 syslog(L_ERROR, "%s: could not read2 token %s %s:0x%s:%ld: %m",
1309 LocalLogName, TokenToText(token), cycbuffname,
1310 CNFSofft2hex(offset, false), cycnum);
1312 if (!SMpreopen) CNFSshutdowncycbuff(cycbuff);
1315 cah.size = cahh.size;
1316 cah.arrived = htonl(time(NULL));
1318 plusoffset = sizeof(oldCNFSARTHEADER)-sizeof(CNFSARTHEADER);
1320 #endif /* OLD_CNFS */
1321 if (offset > cycbuff->len - CNFS_BLOCKSIZE - (off_t) ntohl(cah.size) - 1) {
1323 SMseterror(SMERR_UNDEFINED, "CNFSARTHEADER size overflow");
1324 syslog(L_ERROR, "%s: could not match article size token %s %s:0x%s:%d: %d",
1325 LocalLogName, TokenToText(token), cycbuffname, CNFSofft2hex(offset, false), cycnum, ntohl(cah.size));
1327 CNFSshutdowncycbuff(cycbuff);
1330 CNFSReadFreeAndCycle(cycbuff);
1331 if (offset > cycbuff->len - CNFS_BLOCKSIZE - (off_t) ntohl(cah.size) - 1) {
1332 SMseterror(SMERR_UNDEFINED, "CNFSARTHEADER size overflow");
1333 syslog(L_ERROR, "%s: could not match article size token %s %s:0x%s:%d: %d",
1334 LocalLogName, TokenToText(token), cycbuffname, CNFSofft2hex(offset, false), cycnum, ntohl(cah.size));
1339 /* checking the bitmap to ensure cah.size is not broken was dropped */
1340 if (innconf->cnfscheckfudgesize != 0 && innconf->maxartsize != 0 &&
1341 (ntohl(cah.size) > (size_t) innconf->maxartsize + innconf->cnfscheckfudgesize)) {
1343 strlcpy(buf1, CNFSofft2hex(cycbuff->free, false), sizeof(buf1));
1344 SMseterror(SMERR_UNDEFINED, "CNFSARTHEADER fudge size overflow");
1345 syslog(L_ERROR, "%s: fudge size overflows bitmaps %s %s:0x%s: %u",
1346 LocalLogName, TokenToText(token), cycbuffname, buf1, ntohl(cah.size));
1347 if (!SMpreopen) CNFSshutdowncycbuff(cycbuff);
1351 private = xmalloc(sizeof(PRIV_CNFS));
1352 art->private = (void *)private;
1353 art->arrived = ntohl(cah.arrived);
1354 offset += sizeof(cah) + plusoffset;
1355 if (innconf->articlemmap) {
1356 pagefudge = offset % pagesize;
1357 mmapoffset = offset - pagefudge;
1358 private->len = pagefudge + ntohl(cah.size);
1359 if ((private->base = mmap(NULL, private->len, PROT_READ,
1360 MAP_SHARED, cycbuff->fd, mmapoffset)) == MAP_FAILED) {
1361 SMseterror(SMERR_UNDEFINED, "mmap failed");
1362 syslog(L_ERROR, "%s: could not mmap token %s %s:0x%s:%d: %m",
1363 LocalLogName, TokenToText(token), cycbuffname, CNFSofft2hex(offset, false), cycnum);
1366 if (!SMpreopen) CNFSshutdowncycbuff(cycbuff);
1369 mmap_invalidate(private->base, private->len);
1370 if (amount == RETR_ALL)
1371 madvise(private->base, private->len, MADV_WILLNEED);
1373 madvise(private->base, private->len, MADV_SEQUENTIAL);
1375 private->base = xmalloc(ntohl(cah.size));
1377 if (pread(cycbuff->fd, private->base, ntohl(cah.size), offset) < 0) {
1378 SMseterror(SMERR_UNDEFINED, "read failed");
1379 syslog(L_ERROR, "%s: could not read token %s %s:0x%s:%d: %m",
1380 LocalLogName, TokenToText(token), cycbuffname, CNFSofft2hex(offset, false), cycnum);
1381 free(private->base);
1384 if (!SMpreopen) CNFSshutdowncycbuff(cycbuff);
1389 art->token = &ret_token;
1390 art->len = ntohl(cah.size);
1391 if (amount == RETR_ALL) {
1392 art->data = innconf->articlemmap ? private->base + pagefudge : private->base;
1393 if (!SMpreopen) CNFSshutdowncycbuff(cycbuff);
1396 if ((p = wire_findbody(innconf->articlemmap ? private->base + pagefudge : private->base, art->len)) == NULL) {
1397 SMseterror(SMERR_NOBODY, NULL);
1398 if (innconf->articlemmap)
1399 munmap(private->base, private->len);
1401 free(private->base);
1404 if (!SMpreopen) CNFSshutdowncycbuff(cycbuff);
1407 if (amount == RETR_HEAD) {
1408 if (innconf->articlemmap) {
1409 art->data = private->base + pagefudge;
1410 art->len = p - private->base - pagefudge;
1412 art->data = private->base;
1413 art->len = p - private->base;
1415 if (!SMpreopen) CNFSshutdowncycbuff(cycbuff);
1418 if (amount == RETR_BODY) {
1420 if (innconf->articlemmap)
1421 art->len = art->len - (p - private->base - pagefudge);
1423 art->len = art->len - (p - private->base);
1424 if (!SMpreopen) CNFSshutdowncycbuff(cycbuff);
1427 SMseterror(SMERR_UNDEFINED, "Invalid retrieve request");
1428 if (innconf->articlemmap)
1429 munmap(private->base, private->len);
1431 free(private->base);
1434 if (!SMpreopen) CNFSshutdowncycbuff(cycbuff);
1438 void cnfs_freearticle(ARTHANDLE *article) {
1444 if (article->private) {
1445 private = (PRIV_CNFS *)article->private;
1446 if (innconf->articlemmap)
1447 munmap(private->base, private->len);
1449 free(private->base);
1455 bool cnfs_cancel(TOKEN token) {
1456 char cycbuffname[9];
1461 if (token.type != TOKEN_CNFS) {
1462 SMseterror(SMERR_INTERNAL, NULL);
1465 if (! CNFSBreakToken(token, cycbuffname, &offset, &cycnum)) {
1466 SMseterror(SMERR_INTERNAL, NULL);
1467 /* SMseterror() should have already been called */
1470 if ((cycbuff = CNFSgetcycbuffbyname(cycbuffname)) == NULL) {
1471 SMseterror(SMERR_INTERNAL, "bogus cycbuff name");
1474 if (!SMpreopen && !CNFSinit_disks(cycbuff)) {
1475 SMseterror(SMERR_INTERNAL, "cycbuff initialization fail");
1476 syslog(L_ERROR, "%s: cycbuff '%s' initialization fail", LocalLogName, cycbuff->name);
1479 if (! (cycnum == cycbuff->cyclenum ||
1480 (cycnum == cycbuff->cyclenum - 1 && offset > cycbuff->free) ||
1481 (cycnum + 1 == 0 && cycbuff->cyclenum == 2 && offset > cycbuff->free))) {
1482 SMseterror(SMERR_NOENT, NULL);
1483 if (!SMpreopen) CNFSshutdowncycbuff(cycbuff);
1486 if (CNFSUsedBlock(cycbuff, offset, false, false) == 0) {
1487 SMseterror(SMERR_NOENT, NULL);
1488 if (!SMpreopen) CNFSshutdowncycbuff(cycbuff);
1491 CNFSUsedBlock(cycbuff, offset, true, false);
1492 if (innconf->nfswriter) {
1493 cnfs_mapcntl(NULL, 0, MS_ASYNC);
1495 if (!SMpreopen) CNFSshutdowncycbuff(cycbuff);
1499 ARTHANDLE *cnfs_next(const ARTHANDLE *article, const RETRTYPE amount) {
1502 PRIV_CNFS priv, *private;
1503 off_t middle = 0, limit;
1506 long pagefudge, blockfudge;
1513 if (article == (ARTHANDLE *)NULL) {
1514 if ((cycbuff = cycbufftab) == (CYCBUFF *)NULL)
1515 return (ARTHANDLE *)NULL;
1517 priv.rollover = false;
1519 priv = *(PRIV_CNFS *)article->private;
1520 free(article->private);
1521 free((void *)article);
1522 if (innconf->articlemmap)
1523 munmap(priv.base, priv.len);
1525 /* In the case we return art->data = NULL, we
1526 * must not free an already stale pointer.
1527 -mibsoft@mibsoftware.com
1534 cycbuff = priv.cycbuff;
1537 for (;cycbuff != (CYCBUFF *)NULL;
1538 cycbuff = cycbuff->next,
1541 if (!SMpreopen && !CNFSinit_disks(cycbuff)) {
1542 SMseterror(SMERR_INTERNAL, "cycbuff initialization fail");
1545 if (priv.rollover && priv.offset >= cycbuff->free) {
1547 if (!SMpreopen) CNFSshutdowncycbuff(cycbuff);
1550 if (priv.offset == 0) {
1551 if (cycbuff->cyclenum == 1) {
1552 priv.offset = cycbuff->minartoffset;
1553 priv.rollover = true;
1555 priv.offset = cycbuff->free;
1556 priv.rollover = false;
1559 if (!priv.rollover) {
1560 for (middle = priv.offset ;middle < cycbuff->len - CNFS_BLOCKSIZE - 1;
1561 middle += CNFS_BLOCKSIZE) {
1562 if (CNFSUsedBlock(cycbuff, middle, false, false) != 0)
1565 if (middle >= cycbuff->len - CNFS_BLOCKSIZE - 1) {
1566 priv.rollover = true;
1567 middle = cycbuff->minartoffset;
1571 for (middle = priv.offset ;middle < cycbuff->free;
1572 middle += CNFS_BLOCKSIZE) {
1573 if (CNFSUsedBlock(cycbuff, middle, false, false) != 0)
1576 if (middle >= cycbuff->free) {
1578 if (!SMpreopen) CNFSshutdowncycbuff(cycbuff);
1584 if (cycbuff == (CYCBUFF *)NULL)
1585 return (ARTHANDLE *)NULL;
1588 if (pread(cycbuff->fd, &cah, sizeof(cah), offset) != sizeof(cah)) {
1589 if (!SMpreopen) CNFSshutdowncycbuff(cycbuff);
1590 return (ARTHANDLE *)NULL;
1593 if(cah.size == htonl(0x1234) && ntohl(cah.arrived) < time(NULL)-10*365*24*3600) {
1594 oldCNFSARTHEADER cahh;
1595 *(CNFSARTHEADER *)&cahh = cah;
1596 if(pread(cycbuff->fd, ((char *)&cahh)+sizeof(CNFSARTHEADER), sizeof(oldCNFSARTHEADER)-sizeof(CNFSARTHEADER), offset+sizeof(cah)) != sizeof(oldCNFSARTHEADER)-sizeof(CNFSARTHEADER)) {
1597 if (!SMpreopen) CNFSshutdowncycbuff(cycbuff);
1598 return (ARTHANDLE *)NULL;
1600 cah.size = cahh.size;
1601 cah.arrived = htonl(time(NULL));
1603 plusoffset = sizeof(oldCNFSARTHEADER)-sizeof(CNFSARTHEADER);
1605 #endif /* OLD_CNFS */
1606 art = xmalloc(sizeof(ARTHANDLE));
1607 private = xmalloc(sizeof(PRIV_CNFS));
1608 art->private = (void *)private;
1609 art->type = TOKEN_CNFS;
1611 private->cycbuff = cycbuff;
1612 private->offset = middle;
1613 if (cycbuff->len - cycbuff->free < (off_t) ntohl(cah.size) + CNFS_BLOCKSIZE + 1) {
1614 private->offset += CNFS_BLOCKSIZE;
1618 if (!SMpreopen) CNFSshutdowncycbuff(cycbuff);
1621 /* check the bitmap to ensure cah.size is not broken */
1622 blockfudge = (sizeof(cah) + plusoffset + ntohl(cah.size)) % CNFS_BLOCKSIZE;
1623 limit = private->offset + sizeof(cah) + plusoffset + ntohl(cah.size) - blockfudge + CNFS_BLOCKSIZE;
1624 if (offset < cycbuff->free) {
1625 for (middle = offset + CNFS_BLOCKSIZE; (middle < cycbuff->free) && (middle < limit);
1626 middle += CNFS_BLOCKSIZE) {
1627 if (CNFSUsedBlock(cycbuff, middle, false, false) != 0)
1628 /* Bitmap set. This article assumes to be broken */
1631 if ((middle > cycbuff->free) || (middle != limit)) {
1632 private->offset = middle;
1636 if (!SMpreopen) CNFSshutdowncycbuff(cycbuff);
1640 for (middle = offset + CNFS_BLOCKSIZE; (middle < cycbuff->len) && (middle < limit);
1641 middle += CNFS_BLOCKSIZE) {
1642 if (CNFSUsedBlock(cycbuff, middle, false, false) != 0)
1643 /* Bitmap set. This article assumes to be broken */
1646 if ((middle >= cycbuff->len) || (middle != limit)) {
1647 private->offset = middle;
1651 if (!SMpreopen) CNFSshutdowncycbuff(cycbuff);
1655 if (innconf->cnfscheckfudgesize != 0 && innconf->maxartsize != 0 &&
1656 ((off_t) ntohl(cah.size) > innconf->maxartsize + innconf->cnfscheckfudgesize)) {
1661 if (!SMpreopen) CNFSshutdowncycbuff(cycbuff);
1665 private->offset += (off_t) ntohl(cah.size) + sizeof(cah) + plusoffset;
1666 tonextblock = CNFS_BLOCKSIZE - (private->offset & (CNFS_BLOCKSIZE - 1));
1667 private->offset += (off_t) tonextblock;
1668 art->arrived = ntohl(cah.arrived);
1669 token = CNFSMakeToken(cycbuff->name, offset, (offset > cycbuff->free) ? cycbuff->cyclenum - 1 : cycbuff->cyclenum, cah.class);
1670 art->token = &token;
1671 offset += sizeof(cah) + plusoffset;
1672 if (innconf->articlemmap) {
1673 pagefudge = offset % pagesize;
1674 mmapoffset = offset - pagefudge;
1675 private->len = pagefudge + ntohl(cah.size);
1676 if ((private->base = mmap(0, private->len, PROT_READ,
1677 MAP_SHARED, cycbuff->fd, mmapoffset)) == MAP_FAILED) {
1681 if (!SMpreopen) CNFSshutdowncycbuff(cycbuff);
1684 mmap_invalidate(private->base, private->len);
1685 madvise(private->base, private->len, MADV_SEQUENTIAL);
1687 private->base = xmalloc(ntohl(cah.size));
1689 if (pread(cycbuff->fd, private->base, ntohl(cah.size), offset) < 0) {
1693 if (!SMpreopen) CNFSshutdowncycbuff(cycbuff);
1694 free(private->base);
1699 art->len = ntohl(cah.size);
1700 if (amount == RETR_ALL) {
1701 art->data = innconf->articlemmap ? private->base + pagefudge : private->base;
1702 if (!SMpreopen) CNFSshutdowncycbuff(cycbuff);
1705 if ((p = wire_findbody(innconf->articlemmap ? private->base + pagefudge : private->base, art->len)) == NULL) {
1709 if (!SMpreopen) CNFSshutdowncycbuff(cycbuff);
1712 if (amount == RETR_HEAD) {
1713 if (innconf->articlemmap) {
1714 art->data = private->base + pagefudge;
1715 art->len = p - private->base - pagefudge;
1717 art->data = private->base;
1718 art->len = p - private->base;
1720 if (!SMpreopen) CNFSshutdowncycbuff(cycbuff);
1723 if (amount == RETR_BODY) {
1725 if (innconf->articlemmap)
1726 art->len = art->len - (p - private->base - pagefudge);
1728 art->len = art->len - (p - private->base);
1729 if (!SMpreopen) CNFSshutdowncycbuff(cycbuff);
1735 if (!SMpreopen) CNFSshutdowncycbuff(cycbuff);
1739 bool cnfs_ctl(PROBETYPE type, TOKEN *token UNUSED, void *value) {
1740 struct artngnum *ann;
1744 if ((ann = (struct artngnum *)value) == NULL)
1746 /* make SMprobe() call cnfs_retrieve() */
1754 bool cnfs_flushcacheddata(FLUSHTYPE type) {
1755 if (type == SM_ALL || type == SM_HEAD)
1756 CNFSflushallheads();
1761 cnfs_printfiles(FILE *file, TOKEN token, char **xref UNUSED,
1764 fprintf(file, "%s\n", TokenToText(token));
1767 void cnfs_shutdown(void) {
1769 CNFScleanmetacycbuff();
1770 CNFScleanexpirerule();