1 /* $Id: tradspool.c 7412 2005-10-09 03:44:35Z eagle $
3 ** Storage manager module for traditional spool format.
8 #include "portable/mmap.h"
18 /* Needed for htonl() and friends on AIX 4.1. */
19 #include <netinet/in.h>
21 #include "inn/innconf.h"
28 #include "tradspool.h"
31 char *artbase; /* start of the article data -- may be mmaped */
32 unsigned int artlen; /* art length. */
41 ** The 64-bit hashed representation of a ng name that gets stashed in each token.
46 char hash[HASHEDNGLEN];
50 ** We have two structures here for facilitating newsgroup name->number mapping
51 ** and number->name mapping. NGTable is a hash table based on hashing the
52 ** newsgroup name, and is used to give the name->number mapping. NGTree is
53 ** a binary tree, indexed by newsgroup number, used for the number->name
59 typedef struct _ngtent {
61 /* HASHEDNG hash; XXX */
62 unsigned long ngnumber;
64 struct _ngtreenode *node;
67 typedef struct _ngtreenode {
68 unsigned long ngnumber;
69 struct _ngtreenode *left, *right;
73 NGTENT *NGTable[NGT_SIZE];
74 unsigned long MaxNgNumber = 0;
77 bool NGTableUpdated; /* set to true if we've added any entries since reading
78 in the database file */
81 ** Convert all .s to /s in a newsgroup name. Modifies the passed string
85 DeDotify(char *ngname) {
89 if (*p == '.') *p = '/';
95 ** Hash a newsgroup name to an 8-byte. Basically, we convert all .s to
96 ** /s (so it doesn't matter if we're passed the spooldir name or newsgroup
97 ** name) and then call Hash to MD5 the mess, then take 4 bytes worth of
98 ** data from the front of the hash. This should be good enough for our
103 HashNGName(char *ng) {
105 HASHEDNG return_hash;
110 hash = Hash(p, strlen(p));
113 memcpy(return_hash.hash, hash.hash, HASHEDNGLEN);
119 /* compare two hashes */
121 CompareHash(HASHEDNG *h1, HASHEDNG *h2) {
123 for (i = 0 ; i < HASHEDNGLEN ; ++i) {
124 if (h1->hash[i] != h2->hash[i]) {
125 return h1->hash[i] - h2->hash[i];
132 /* Add a new newsgroup name to the NG table. */
134 AddNG(char *ng, unsigned long number) {
138 NGTENT *ngtp, **ngtpp;
139 NGTREENODE *newnode, *curnode, **nextnode;
142 DeDotify(p); /* canonicalize p to standard (/) form. */
143 hash = HashNGName(p);
145 h = (unsigned char)hash.hash[0];
146 h = h + (((unsigned char)hash.hash[1])<<8);
154 /* ng wasn't in table, add new entry. */
155 NGTableUpdated = true;
157 ngtp = xmalloc(sizeof(NGTENT));
158 ngtp->ngname = p; /* note: we store canonicalized name */
159 /* ngtp->hash = hash XXX */
162 /* assign a new NG number if needed (not given) */
164 number = ++MaxNgNumber;
166 ngtp->ngnumber = number;
168 /* link new table entry into the hash table chain. */
171 /* Now insert an appropriate record into the binary tree */
172 newnode = xmalloc(sizeof(NGTREENODE));
173 newnode->left = newnode->right = (NGTREENODE *) NULL;
174 newnode->ngnumber = number;
175 newnode->ngtp = ngtp;
176 ngtp->node = newnode;
178 if (NGTree == NULL) {
179 /* tree was empty, so put our one element in and return */
186 if (curnode->ngnumber < number) {
187 nextnode = &curnode->right;
188 } else if (curnode->ngnumber > number) {
189 nextnode = &curnode->left;
191 /* Error, same number is already in NGtree (shouldn't happen!) */
192 syslog(L_ERROR, "tradspool: AddNG: duplicate newsgroup number in NGtree: %ld(%s)", number, p);
199 } else if (strcmp(ngtp->ngname, p) == 0) {
200 /* entry in table already, so return */
204 } else if (CompareHash(&ngtp->hash, &hash) == 0) {
205 /* eep! we hit a hash collision. */
206 syslog(L_ERROR, "tradspool: AddNG: Hash collison %s/%s", ngtp->ngname, p);
211 /* not found yet, so advance to next entry in chain */
212 ngtpp = &(ngtp->next);
218 /* find a newsgroup table entry, given only the name. */
220 FindNGByName(char *ngname) {
227 DeDotify(p); /* canonicalize p to standard (/) form. */
228 hash = HashNGName(p);
230 h = (unsigned char)hash.hash[0];
231 h = h + (((unsigned char)hash.hash[1])<<8);
238 if (strcmp(p, ngtp->ngname) == 0) {
248 /* find a newsgroup/spooldir name, given only the newsgroup number */
250 FindNGByNum(unsigned long ngnumber) {
257 if (curnode->ngnumber == ngnumber) {
258 ngtp = curnode->ngtp;
261 if (curnode->ngnumber < ngnumber) {
262 curnode = curnode->right;
264 curnode = curnode->left;
267 /* not in tree, return NULL */
271 #define _PATH_TRADSPOOLNGDB "tradspool.map"
272 #define _PATH_NEWTSNGDB "tradspool.map.new"
275 /* dump DB to file. */
279 char *fname, *fnamenew;
284 if (!SMopenmode) return; /* don't write if we're not in read/write mode. */
285 if (!NGTableUpdated) return; /* no need to dump new DB */
287 fname = concatpath(innconf->pathspool, _PATH_TRADSPOOLNGDB);
288 fnamenew = concatpath(innconf->pathspool, _PATH_NEWTSNGDB);
290 if ((out = fopen(fnamenew, "w")) == NULL) {
291 syslog(L_ERROR, "tradspool: DumpDB: can't write %s: %m", fnamenew);
296 for (i = 0 ; i < NGT_SIZE ; ++i) {
298 for ( ; ngtp ; ngtp = ngtp->next) {
299 fprintf(out, "%s %lu\n", ngtp->ngname, ngtp->ngnumber);
302 if (fclose(out) < 0) {
303 syslog(L_ERROR, "tradspool: DumpDB: can't close %s: %m", fnamenew);
308 if (rename(fnamenew, fname) < 0) {
309 syslog(L_ERROR, "tradspool: can't rename %s", fnamenew);
316 NGTableUpdated = false; /* reset modification flag. */
321 ** init NGTable from saved database file and from active. Note that
322 ** entries in the database file get added first, and get their specifications
323 ** of newsgroup number from there.
333 unsigned long number;
335 fname = concatpath(innconf->pathspool, _PATH_TRADSPOOLNGDB);
336 if ((qp = QIOopen(fname)) == NULL) {
337 /* only warn if db not found. */
338 syslog(L_NOTICE, "tradspool: %s not found", fname);
340 while ((line = QIOread(qp)) != NULL) {
341 p = strchr(line, ' ');
343 syslog(L_FATAL, "tradspool: corrupt line in active %s", line);
351 if (MaxNgNumber < number) MaxNgNumber = number;
367 fname = concatpath(innconf->pathdb, _PATH_ACTIVE);
368 if ((qp = QIOopen(fname)) == NULL) {
369 syslog(L_FATAL, "tradspool: can't open %s", fname);
374 while ((line = QIOread(qp)) != NULL) {
375 p = strchr(line, ' ');
377 syslog(L_FATAL, "tradspool: corrupt line in active %s", line);
387 /* dump any newly added changes to database */
395 if (!ReadDBFile()) return false;
398 ** set NGTableUpdated to false; that way we know if the load of active or
399 ** any AddNGs later on did in fact add new entries to the db.
401 NGTableUpdated = false;
403 /* don't read active unless write mode. */
405 return ReadActiveFile();
409 ** Routine called to check every so often to see if we need to reload the
410 ** database and add in any new groups that have been added. This is primarily
411 ** for the benefit of innfeed in funnel mode, which otherwise would never
412 ** get word that any new newsgroups had been added.
415 #define RELOAD_TIME_CHECK 600
418 CheckNeedReloadDB(bool force)
420 static TIMEINFO lastcheck, oldlastcheck, now;
424 if (GetTimeInfo(&now) < 0) return; /* anyone ever seen gettimeofday fail? :-) */
425 if (!force && lastcheck.time + RELOAD_TIME_CHECK > now.time) return;
427 oldlastcheck = lastcheck;
430 fname = concatpath(innconf->pathspool, _PATH_TRADSPOOLNGDB);
431 if (stat(fname, &sb) < 0) {
436 if (sb.st_mtime > oldlastcheck.time) {
437 /* add any newly added ngs to our in-memory copy of the db. */
442 /* Init routine, called by SMinit */
445 tradspool_init(SMATTRIBUTE *attr) {
447 syslog(L_ERROR, "tradspool: attr is NULL");
448 SMseterror(SMERR_INTERNAL, "attr is NULL");
451 if (!innconf->storeonxref) {
452 syslog(L_ERROR, "tradspool: storeonxref needs to be true");
453 SMseterror(SMERR_INTERNAL, "storeonxref needs to be true");
456 attr->selfexpire = false;
457 attr->expensivestat = true;
458 return InitNGTable();
461 /* Make a token for an article given the primary newsgroup name and article # */
463 MakeToken(char *ng, unsigned long artnum, STORAGECLASS class) {
468 memset(&token, '\0', sizeof(token));
470 token.type = TOKEN_TRADSPOOL;
474 ** if not already in the NG Table, be sure to add this ng! This way we
475 ** catch things like newsgroups added since startup.
477 if ((ngtp = FindNGByName(ng)) == NULL) {
479 DumpDB(); /* flush to disk so other programs can see the change */
480 ngtp = FindNGByName(ng);
483 num = ngtp->ngnumber;
486 memcpy(token.token, &num, sizeof(num));
487 artnum = htonl(artnum);
488 memcpy(&token.token[sizeof(num)], &artnum, sizeof(artnum));
493 ** Convert a token back to a pathname.
496 TokenToPath(TOKEN token) {
498 unsigned long artnum;
502 CheckNeedReloadDB(false);
504 memcpy(&ngnum, &token.token[0], sizeof(ngnum));
505 memcpy(&artnum, &token.token[sizeof(ngnum)], sizeof(artnum));
506 artnum = ntohl(artnum);
507 ngnum = ntohl(ngnum);
509 ng = FindNGByNum(ngnum);
511 CheckNeedReloadDB(true);
512 ng = FindNGByNum(ngnum);
517 length = strlen(ng) + 20 + strlen(innconf->patharticles);
518 path = xmalloc(length);
519 snprintf(path, length, "%s/%s/%lu", innconf->patharticles, ng, artnum);
524 ** Crack an Xref line apart into separate strings, each of the form "ng:artnum".
525 ** Return in "num" the number of newsgroups found.
528 CrackXref(char *xref, unsigned int *lenp) {
532 unsigned int len, xrefsize;
536 xrefs = xmalloc(xrefsize * sizeof(char *));
538 /* no path element should exist, nor heading white spaces exist */
542 /* shouldn't ever hit null w/o hitting a \r\n first, but best to be paranoid */
543 if (*p == '\n' || *p == '\r' || *p == 0) {
544 /* hit EOL, return. */
548 /* skip to next space or EOL */
549 for (q=p; *q && *q != ' ' && *q != '\n' && *q != '\r' ; ++q) ;
551 xrefs[len] = xstrndup(p, q - p);
553 if (++len == xrefsize) {
554 /* grow xrefs if needed. */
556 xrefs = xrealloc(xrefs, xrefsize * sizeof(char *));
561 for ( ; *p == ' ' ; p++) ;
566 tradspool_store(const ARTHANDLE article, const STORAGECLASS class) {
570 unsigned int numxrefs;
571 char *ng, *p, *onebuffer;
572 unsigned long artnum;
573 char *path, *linkpath, *dirname;
576 char *nonwfarticle; /* copy of article converted to non-wire format */
578 size_t length, nonwflen;
580 xrefhdr = article.groups;
581 if ((xrefs = CrackXref(xrefhdr, &numxrefs)) == NULL || numxrefs == 0) {
582 token.type = TOKEN_EMPTY;
583 SMseterror(SMERR_UNDEFINED, "bogus Xref: header");
589 if ((p = strchr(xrefs[0], ':')) == NULL) {
590 token.type = TOKEN_EMPTY;
591 SMseterror(SMERR_UNDEFINED, "bogus Xref: header");
592 for (i = 0 ; i < numxrefs; ++i) free(xrefs[i]);
601 token = MakeToken(ng, artnum, class);
603 length = strlen(innconf->patharticles) + strlen(ng) + 32;
604 path = xmalloc(length);
605 snprintf(path, length, "%s/%s/%lu", innconf->patharticles, ng, artnum);
607 /* following chunk of code boldly stolen from timehash.c :-) */
608 if ((fd = open(path, O_CREAT|O_EXCL|O_WRONLY, ARTFILE_MODE)) < 0) {
609 p = strrchr(path, '/');
611 if (!MakeDirectory(path, true)) {
612 syslog(L_ERROR, "tradspool: could not make directory %s %m", path);
613 token.type = TOKEN_EMPTY;
615 SMseterror(SMERR_UNDEFINED, NULL);
616 for (i = 0 ; i < numxrefs; ++i) free(xrefs[i]);
621 if ((fd = open(path, O_CREAT|O_EXCL|O_WRONLY, ARTFILE_MODE)) < 0) {
622 SMseterror(SMERR_UNDEFINED, NULL);
623 syslog(L_ERROR, "tradspool: could not open %s %m", path);
624 token.type = TOKEN_EMPTY;
626 for (i = 0 ; i < numxrefs; ++i) free(xrefs[i]);
632 if (innconf->wireformat) {
633 if (xwritev(fd, article.iov, article.iovcnt) != (ssize_t) article.len) {
634 SMseterror(SMERR_UNDEFINED, NULL);
635 syslog(L_ERROR, "tradspool error writing %s %m", path);
637 token.type = TOKEN_EMPTY;
640 for (i = 0 ; i < numxrefs; ++i) free(xrefs[i]);
645 onebuffer = xmalloc(article.len);
646 for (used = i = 0 ; i < article.iovcnt ; i++) {
647 memcpy(&onebuffer[used], article.iov[i].iov_base, article.iov[i].iov_len);
648 used += article.iov[i].iov_len;
650 nonwfarticle = FromWireFmt(onebuffer, used, &nonwflen);
652 if (write(fd, nonwfarticle, nonwflen) != (ssize_t) nonwflen) {
654 SMseterror(SMERR_UNDEFINED, NULL);
655 syslog(L_ERROR, "tradspool error writing %s %m", path);
657 token.type = TOKEN_EMPTY;
660 for (i = 0 ; i < numxrefs; ++i) free(xrefs[i]);
669 ** blah, this is ugly. Have to make symlinks under other pathnames for
670 ** backwards compatiblility purposes.
674 for (i = 1; i < numxrefs ; ++i) {
675 if ((p = strchr(xrefs[i], ':')) == NULL) continue;
681 length = strlen(innconf->patharticles) + strlen(ng) + 32;
682 linkpath = xmalloc(length);
683 snprintf(linkpath, length, "%s/%s/%lu", innconf->patharticles,
685 if (link(path, linkpath) < 0) {
686 p = strrchr(linkpath, '/');
688 dirname = xstrdup(linkpath);
690 if (!MakeDirectory(dirname, true) || link(path, linkpath) < 0) {
691 #if !defined(HAVE_SYMLINK)
692 SMseterror(SMERR_UNDEFINED, NULL);
693 syslog(L_ERROR, "tradspool: could not link %s to %s %m", path, linkpath);
694 token.type = TOKEN_EMPTY;
698 for (i = 0 ; i < numxrefs; ++i) free(xrefs[i]);
702 if (symlink(path, linkpath) < 0) {
703 SMseterror(SMERR_UNDEFINED, NULL);
704 syslog(L_ERROR, "tradspool: could not symlink %s to %s %m", path, linkpath);
705 token.type = TOKEN_EMPTY;
709 for (i = 0 ; i < numxrefs; ++i) free(xrefs[i]);
713 #endif /* !defined(HAVE_SYMLINK) */
721 for (i = 0 ; i < numxrefs; ++i) free(xrefs[i]);
727 OpenArticle(const char *path, RETRTYPE amount) {
729 PRIV_TRADSPOOL *private;
736 if (amount == RETR_STAT) {
737 if (access(path, R_OK) < 0) {
738 SMseterror(SMERR_UNDEFINED, NULL);
741 art = xmalloc(sizeof(ARTHANDLE));
742 art->type = TOKEN_TRADSPOOL;
749 if ((fd = open(path, O_RDONLY)) < 0) {
750 SMseterror(SMERR_UNDEFINED, NULL);
754 art = xmalloc(sizeof(ARTHANDLE));
755 art->type = TOKEN_TRADSPOOL;
757 if (fstat(fd, &sb) < 0) {
758 SMseterror(SMERR_UNDEFINED, NULL);
759 syslog(L_ERROR, "tradspool: could not fstat article: %m");
765 art->arrived = sb.st_mtime;
767 private = xmalloc(sizeof(PRIV_TRADSPOOL));
768 art->private = (void *)private;
769 private->artlen = sb.st_size;
770 if (innconf->articlemmap) {
771 if ((private->artbase = mmap(NULL, sb.st_size, PROT_READ, MAP_SHARED, fd, 0)) == MAP_FAILED) {
772 SMseterror(SMERR_UNDEFINED, NULL);
773 syslog(L_ERROR, "tradspool: could not mmap article: %m");
779 if (amount == RETR_ALL)
780 madvise(private->artbase, sb.st_size, MADV_WILLNEED);
782 madvise(private->artbase, sb.st_size, MADV_SEQUENTIAL);
784 /* consider coexisting both wireformatted and nonwireformatted */
785 p = memchr(private->artbase, '\n', private->artlen);
786 if (p == NULL || p == private->artbase) {
787 SMseterror(SMERR_UNDEFINED, NULL);
788 syslog(L_ERROR, "tradspool: could not mmap article: %m");
789 munmap(private->artbase, private->artlen);
796 private->mmapped = true;
798 wfarticle = ToWireFmt(private->artbase, private->artlen, &wflen);
799 munmap(private->artbase, private->artlen);
800 private->artbase = wfarticle;
801 private->artlen = wflen;
802 private->mmapped = false;
805 private->mmapped = false;
806 private->artbase = xmalloc(private->artlen);
807 if (read(fd, private->artbase, private->artlen) < 0) {
808 SMseterror(SMERR_UNDEFINED, NULL);
809 syslog(L_ERROR, "tradspool: could not read article: %m");
810 free(private->artbase);
816 p = memchr(private->artbase, '\n', private->artlen);
817 if (p == NULL || p == private->artbase) {
818 SMseterror(SMERR_UNDEFINED, NULL);
819 syslog(L_ERROR, "tradspool: could not mmap article: %m");
826 /* need to make a wireformat copy of the article */
827 wfarticle = ToWireFmt(private->artbase, private->artlen, &wflen);
828 free(private->artbase);
829 private->artbase = wfarticle;
830 private->artlen = wflen;
835 private->ngtp = NULL;
836 private->curdir = NULL;
837 private->curdirname = NULL;
838 private->nextindex = -1;
840 if (amount == RETR_ALL) {
841 art->data = private->artbase;
842 art->len = private->artlen;
846 if (((p = wire_findbody(private->artbase, private->artlen)) == NULL)) {
847 if (private->mmapped)
848 munmap(private->artbase, private->artlen);
850 free(private->artbase);
851 SMseterror(SMERR_NOBODY, NULL);
857 if (amount == RETR_HEAD) {
858 art->data = private->artbase;
859 art->len = p - private->artbase;
863 if (amount == RETR_BODY) {
865 art->len = private->artlen - (p - private->artbase);
868 SMseterror(SMERR_UNDEFINED, "Invalid retrieve request");
869 if (private->mmapped)
870 munmap(private->artbase, private->artlen);
872 free(private->artbase);
880 tradspool_retrieve(const TOKEN token, const RETRTYPE amount) {
883 static TOKEN ret_token;
885 if (token.type != TOKEN_TRADSPOOL) {
886 SMseterror(SMERR_INTERNAL, NULL);
890 if ((path = TokenToPath(token)) == NULL) {
891 SMseterror(SMERR_NOENT, NULL);
894 if ((art = OpenArticle(path, amount)) != (ARTHANDLE *)NULL) {
896 art->token = &ret_token;
903 tradspool_freearticle(ARTHANDLE *article)
905 PRIV_TRADSPOOL *private;
910 if (article->private) {
911 private = (PRIV_TRADSPOOL *) article->private;
912 if (private->mmapped)
913 munmap(private->artbase, private->artlen);
915 free(private->artbase);
917 closedir(private->curdir);
918 free(private->curdirname);
925 tradspool_cancel(TOKEN token) {
929 unsigned int numxrefs;
931 char *path, *linkpath;
934 unsigned long artnum;
937 if ((path = TokenToPath(token)) == NULL) {
938 SMseterror(SMERR_UNDEFINED, NULL);
943 ** Ooooh, this is gross. To find the symlinks pointing to this article,
944 ** we open the article and grab its Xref line (since the token isn't long
945 ** enough to store this info on its own). This is *not* going to do
946 ** good things for performance of fastrm... -- rmtodd
948 if ((article = OpenArticle(path, RETR_HEAD)) == NULL) {
950 SMseterror(SMERR_UNDEFINED, NULL);
954 xrefhdr = wire_findheader(article->data, article->len, "Xref");
955 if (xrefhdr == NULL) {
956 /* for backwards compatibility; there is no Xref unless crossposted
958 if (unlink(path) < 0) result = false;
960 tradspool_freearticle(article);
964 if ((xrefs = CrackXref(xrefhdr, &numxrefs)) == NULL || numxrefs == 0) {
968 tradspool_freearticle(article);
969 SMseterror(SMERR_UNDEFINED, NULL);
973 tradspool_freearticle(article);
974 for (i = 1 ; i < numxrefs ; ++i) {
975 if ((p = strchr(xrefs[i], ':')) == NULL) continue;
981 length = strlen(innconf->patharticles) + strlen(ng) + 32;
982 linkpath = xmalloc(length);
983 snprintf(linkpath, length, "%s/%s/%lu", innconf->patharticles, ng,
985 /* repeated unlinkings of a crossposted article may fail on account
986 of the file no longer existing without it truly being an error */
987 if (unlink(linkpath) < 0)
988 if (errno != ENOENT || i == 1)
992 if (unlink(path) < 0)
993 if (errno != ENOENT || numxrefs == 1)
996 for (i = 0 ; i < numxrefs ; ++i) free(xrefs[i]);
1003 ** Find entries for possible articles in dir. "dir" (directory name "dirname").
1004 ** The dirname is needed so we can do stats in the directory to disambiguate
1005 ** files from symlinks and directories.
1008 static struct dirent *
1009 FindDir(DIR *dir, char *dirname) {
1016 unsigned char namelen;
1018 while ((de = readdir(dir)) != NULL) {
1019 namelen = strlen(de->d_name);
1020 for (i = 0, flag = true ; i < namelen ; ++i) {
1021 if (!CTYPE(isdigit, de->d_name[i])) {
1026 if (!flag) continue; /* if not all digits, skip this entry. */
1028 length = strlen(dirname) + namelen + 2;
1029 path = xmalloc(length);
1030 strlcpy(path, dirname, length);
1031 strlcat(path, "/", length);
1032 strlcat(path, de->d_name, length);
1034 if (lstat(path, &sb) < 0) {
1039 if (!S_ISREG(sb.st_mode)) continue;
1045 ARTHANDLE *tradspool_next(const ARTHANDLE *article, const RETRTYPE amount) {
1046 PRIV_TRADSPOOL priv;
1047 PRIV_TRADSPOOL *newpriv;
1048 char *path, *linkpath;
1051 unsigned long artnum;
1055 char *xrefhdr, *ng, *p, *expires, *x;
1056 unsigned int numxrefs;
1060 if (article == NULL) {
1063 priv.curdirname = NULL;
1064 priv.nextindex = -1;
1066 priv = *(PRIV_TRADSPOOL *) article->private;
1067 free(article->private);
1068 free((void*)article);
1069 if (priv.artbase != NULL) {
1071 munmap(priv.artbase, priv.artlen);
1077 while (!priv.curdir || ((de = FindDir(priv.curdir, priv.curdirname)) == NULL)) {
1079 closedir(priv.curdir);
1081 free(priv.curdirname);
1082 priv.curdirname = NULL;
1086 ** advance ngtp to the next entry, if it exists, otherwise start
1087 ** searching down another ngtable hashchain.
1089 while (priv.ngtp == NULL || (priv.ngtp = priv.ngtp->next) == NULL) {
1091 ** note that at the start of a search nextindex is -1, so the inc.
1092 ** makes nextindex 0, as it should be.
1095 if (priv.nextindex >= NGT_SIZE) {
1096 /* ran off the end of the table, so return. */
1099 priv.ngtp = NGTable[priv.nextindex];
1100 if (priv.ngtp != NULL)
1104 priv.curdirname = concatpath(innconf->patharticles, priv.ngtp->ngname);
1105 priv.curdir = opendir(priv.curdirname);
1108 path = concatpath(priv.curdirname, de->d_name);
1109 i = strlen(priv.curdirname);
1110 /* get the article number while we're here, we'll need it later. */
1111 artnum = atol(&path[i+1]);
1113 art = OpenArticle(path, amount);
1114 if (art == (ARTHANDLE *)NULL) {
1115 art = xmalloc(sizeof(ARTHANDLE));
1116 art->type = TOKEN_TRADSPOOL;
1119 art->private = xmalloc(sizeof(PRIV_TRADSPOOL));
1123 newpriv = (PRIV_TRADSPOOL *) art->private;
1124 newpriv->artbase = NULL;
1126 /* Skip linked (not symlinked) crossposted articles.
1128 This algorithm is rather questionable; it only works if the first
1129 group/number combination listed in the Xref header is the
1130 canonical path. This will always be true for spools created by
1131 this implementation, but for traditional INN 1.x servers,
1132 articles are expired indepedently from each group and may expire
1133 out of the first listed newsgroup before other groups. This
1134 algorithm will orphan such articles, not adding them to history.
1136 The bit of skipping articles by setting the length of the article
1137 to zero is also rather suspect, and I'm not sure what
1138 implications that might have for the callers of SMnext.
1140 Basically, this whole area really needs to be rethought. */
1141 xrefhdr = wire_findheader(art->data, art->len, "Xref");
1142 if (xrefhdr != NULL) {
1143 if ((xrefs = CrackXref(xrefhdr, &numxrefs)) == NULL || numxrefs == 0) {
1146 /* assumes first one is the original */
1147 if ((p = strchr(xrefs[1], ':')) != NULL) {
1153 length = strlen(innconf->patharticles) + strlen(ng) + 32;
1154 linkpath = xmalloc(length);
1155 snprintf(linkpath, length, "%s/%s/%lu",
1156 innconf->patharticles, ng, artnum);
1157 if (strcmp(path, linkpath) != 0) {
1158 /* this is linked article, skip it */
1164 for (i = 0 ; i < numxrefs ; ++i) free(xrefs[i]);
1166 if (innconf->storeonxref) {
1167 /* skip path element */
1168 if ((xrefhdr = strchr(xrefhdr, ' ')) == NULL) {
1172 for (xrefhdr++; *xrefhdr == ' '; xrefhdr++);
1173 art->groups = xrefhdr;
1174 for (p = xrefhdr ; (*p != '\n') && (*p != '\r') ; p++);
1175 art->groupslen = p - xrefhdr;
1179 if (xrefhdr == NULL || !innconf->storeonxref) {
1180 ng = wire_findheader(art->data, art->len, "Newsgroups");
1186 for (p = ng ; (*p != '\n') && (*p != '\r') ; p++);
1187 art->groupslen = p - ng;
1190 expires = wire_findheader(art->data, art->len, "Expires");
1191 if (expires == NULL) {
1194 /* optionally parse expire header */
1195 for (p = expires + 1; (*p != '\n') && (*(p - 1) != '\r'); p++);
1196 x = xmalloc(p - expires);
1197 memcpy(x, expires, p - expires - 1);
1198 x[p - expires - 1] = '\0';
1200 art->expires = parsedate(x, NULL);
1201 if (art->expires == -1)
1204 art->expires -= time(0);
1207 /* for backwards compatibility; assumes no Xref unless crossposted
1208 for 1.4 and 1.5: just fall through */
1210 newpriv = (PRIV_TRADSPOOL *) art->private;
1211 newpriv->nextindex = priv.nextindex;
1212 newpriv->curdir = priv.curdir;
1213 newpriv->curdirname = priv.curdirname;
1214 newpriv->ngtp = priv.ngtp;
1216 if ((sub = SMgetsub(*art)) == NULL || sub->type != TOKEN_TRADSPOOL) {
1217 /* maybe storage.conf is modified, after receiving article */
1218 token = MakeToken(priv.ngtp->ngname, artnum, 0);
1220 /* Only log an error if art->len is non-zero, since otherwise we get
1221 all the ones skipped via the hard-link skipping algorithm
1224 syslog(L_ERROR, "tradspool: can't determine class: %s: %s",
1225 TokenToText(token), SMerrorstr);
1227 token = MakeToken(priv.ngtp->ngname, artnum, sub->class);
1229 art->token = &token;
1238 NGTENT *ngtp, *nextngtp;
1240 for (i = 0 ; i < NGT_SIZE ; i++) {
1242 for ( ; ngtp != NULL ; ngtp = nextngtp) {
1243 nextngtp = ngtp->next;
1254 bool tradspool_ctl(PROBETYPE type, TOKEN *token, void *value) {
1255 struct artngnum *ann;
1256 unsigned long ngnum;
1257 unsigned long artnum;
1262 if ((ann = (struct artngnum *)value) == NULL)
1264 CheckNeedReloadDB(false);
1265 memcpy(&ngnum, &token->token[0], sizeof(ngnum));
1266 memcpy(&artnum, &token->token[sizeof(ngnum)], sizeof(artnum));
1267 artnum = ntohl(artnum);
1268 ngnum = ntohl(ngnum);
1269 ng = FindNGByNum(ngnum);
1271 CheckNeedReloadDB(true);
1272 ng = FindNGByNum(ngnum);
1276 ann->groupname = xstrdup(ng);
1277 for (p = ann->groupname; *p != 0; p++)
1280 ann->artnum = (ARTNUM)artnum;
1288 tradspool_flushcacheddata(FLUSHTYPE type UNUSED)
1294 tradspool_printfiles(FILE *file, TOKEN token UNUSED, char **xref, int ngroups)
1299 for (i = 0; i < ngroups; i++) {
1300 path = xstrdup(xref[i]);
1301 for (p = path; *p != '\0'; p++)
1302 if (*p == '.' || *p == ':')
1304 fprintf(file, "%s\n", path);
1310 tradspool_shutdown(void) {