1 /* $Id: article.c 7538 2006-08-26 05:44:06Z eagle $
3 ** Article-related routines.
14 #include "inn/innconf.h"
15 #include "inn/messages.h"
27 ** Data structures for use in ARTICLE/HEAD/BODY/STAT common code.
29 typedef enum _SENDTYPE {
36 typedef struct _SENDDATA {
42 static char ARTnotingroup[] = NNTP_NOTINGROUP;
43 static char ARTnoartingroup[] = NNTP_NOARTINGRP;
44 static char ARTnocurrart[] = NNTP_NOCURRART;
45 static ARTHANDLE *ARThandle = NULL;
46 static SENDDATA SENDbody = {
47 STbody, NNTP_BODY_FOLLOWS_VAL, "body"
49 static SENDDATA SENDarticle = {
50 STarticle, NNTP_ARTICLE_FOLLOWS_VAL, "article"
52 static SENDDATA SENDstat = {
53 STstat, NNTP_NOTHING_FOLLOWS_VAL, "status"
55 static SENDDATA SENDhead = {
56 SThead, NNTP_HEAD_FOLLOWS_VAL, "head"
60 static struct iovec iov[IOV_MAX > 1024 ? 1024 : IOV_MAX];
61 static int queued_iov = 0;
63 static void PushIOvHelper(struct iovec* vec, int* countp) {
65 TMRstart(TMR_NNTPWRITE);
69 result = SSL_writev(tls_conn, vec, *countp);
70 switch (SSL_get_error(tls_conn, result)) {
72 case SSL_ERROR_SYSCALL:
74 case SSL_ERROR_WANT_WRITE:
78 SSL_shutdown(tls_conn);
82 case SSL_ERROR_ZERO_RETURN:
86 result = xwritev(STDOUT_FILENO, vec, *countp);
89 result = xwritev(STDOUT_FILENO, vec, *countp);
91 TMRstop(TMR_NNTPWRITE);
93 /* we can't recover, since we can't resynchronise with our
95 ExitWithStats(1, true);
101 PushIOvRateLimited(void) {
102 double start, end, elapsed, target;
103 struct iovec newiov[IOV_MAX > 1024 ? 1024 : IOV_MAX];
109 struct timeval waittime;
112 bytesfound = newiov_len = 0;
114 for (i = 0; (i < queued_iov) && (bytesfound < MaxBytesPerSecond); i++) {
115 if ((signed)iov[i].iov_len + bytesfound > MaxBytesPerSecond) {
116 chunkbittenoff = MaxBytesPerSecond - bytesfound;
117 newiov[newiov_len].iov_base = iov[i].iov_base;
118 newiov[newiov_len++].iov_len = chunkbittenoff;
119 iov[i].iov_base = (char *)iov[i].iov_base + chunkbittenoff;
120 iov[i].iov_len -= chunkbittenoff;
121 bytesfound += chunkbittenoff;
123 newiov[newiov_len++] = iov[i];
125 bytesfound += iov[i].iov_len;
128 assert(sentiov <= queued_iov);
129 start = TMRnow_double();
130 PushIOvHelper(newiov, &newiov_len);
131 end = TMRnow_double();
132 target = (double) bytesfound / MaxBytesPerSecond;
133 elapsed = end - start;
134 if (elapsed < 1 && elapsed < target) {
136 waittime.tv_usec = (target - elapsed) * 1e6;
137 start = TMRnow_double();
138 if (select(0, NULL, NULL, NULL, &waittime) != 0)
139 syswarn("%s: select in PushIOvRateLimit failed", ClientHost);
140 end = TMRnow_double();
141 IDLEtime += end - start;
143 memmove(iov, &iov[sentiov], (queued_iov - sentiov) * sizeof(struct iovec));
144 queued_iov -= sentiov;
150 TMRstart(TMR_NNTPWRITE);
152 TMRstop(TMR_NNTPWRITE);
153 if (MaxBytesPerSecond != 0)
154 PushIOvRateLimited();
156 PushIOvHelper(iov, &queued_iov);
160 SendIOv(const char *p, int len) {
164 q = (char *)iov[queued_iov - 1].iov_base + iov[queued_iov - 1].iov_len;
166 iov[queued_iov - 1].iov_len += len;
170 iov[queued_iov].iov_base = (char*)p;
171 iov[queued_iov++].iov_len = len;
172 if (queued_iov == IOV_MAX)
176 static char *_IO_buffer_ = NULL;
177 static int highwater = 0;
181 TMRstart(TMR_NNTPWRITE);
187 r = SSL_write(tls_conn, _IO_buffer_, highwater);
188 switch (SSL_get_error(tls_conn, r)) {
190 case SSL_ERROR_SYSCALL:
192 case SSL_ERROR_WANT_WRITE:
196 SSL_shutdown(tls_conn);
200 case SSL_ERROR_ZERO_RETURN:
203 if (r != highwater) {
204 TMRstop(TMR_NNTPWRITE);
209 if (xwrite(STDOUT_FILENO, _IO_buffer_, highwater) != highwater) {
210 TMRstop(TMR_NNTPWRITE);
216 if (xwrite(STDOUT_FILENO, _IO_buffer_, highwater) != highwater) {
217 TMRstop(TMR_NNTPWRITE);
222 TMRstop(TMR_NNTPWRITE);
227 SendIOb(const char *p, int len) {
230 if (_IO_buffer_ == NULL)
231 _IO_buffer_ = xmalloc(BIG_BUFFER);
234 tocopy = (len > (BIG_BUFFER - highwater)) ? (BIG_BUFFER - highwater) : len;
235 memcpy(&_IO_buffer_[highwater], p, tocopy);
239 if (highwater == BIG_BUFFER)
246 ** If we have an article open, close it.
251 SMfreearticle(ARThandle);
256 bool ARTinstorebytoken(TOKEN token)
259 struct timeval stv, etv;
261 if (PERMaccessconf->nnrpdoverstats) {
262 gettimeofday(&stv, NULL);
264 art = SMretrieve(token, RETR_STAT); /* XXX This isn't really overstats, is it? */
265 if (PERMaccessconf->nnrpdoverstats) {
266 gettimeofday(&etv, NULL);
267 OVERartcheck+=(etv.tv_sec - stv.tv_sec) * 1000;
268 OVERartcheck+=(etv.tv_usec - stv.tv_usec) / 1000;
278 ** If the article name is valid, open it and stuff in the ID.
280 static bool ARTopen(ARTNUM artnum)
282 static ARTNUM save_artnum;
285 /* Re-use article if it's the same one. */
286 if (save_artnum == artnum) {
292 if (!OVgetartinfo(GRPcur, artnum, &token))
295 TMRstart(TMR_READART);
296 ARThandle = SMretrieve(token, RETR_ALL);
297 TMRstop(TMR_READART);
298 if (ARThandle == NULL) {
302 save_artnum = artnum;
308 ** Open the article for a given Message-ID.
311 ARTopenbyid(char *msg_id, ARTNUM *ap, bool final)
316 token = cache_get(HashMessageID(msg_id), final);
317 if (token.type == TOKEN_EMPTY) {
318 if (History == NULL) {
321 /* Do lazy opens of the history file - lots of clients
322 * will never ask for anything by message id, so put off
323 * doing the work until we have to */
324 History = HISopen(HISTORY, innconf->hismethod, HIS_RDONLY);
326 syslog(L_NOTICE, "cant initialize history");
327 Reply("%d NNTP server unavailable. Try later.\r\n",
329 ExitWithStats(1, true);
332 HISctl(History, HISCTLS_STATINTERVAL, &statinterval);
334 if (!HISlookup(History, msg_id, NULL, NULL, NULL, &token))
337 if (token.type == TOKEN_EMPTY)
339 TMRstart(TMR_READART);
340 ARThandle = SMretrieve(token, RETR_ALL);
341 TMRstop(TMR_READART);
342 if (ARThandle == NULL) {
350 ** Send a (part of) a file to stdout, doing newline and dot conversion.
352 static void ARTsendmmap(SENDTYPE what)
354 const char *p, *q, *r;
355 const char *s, *path, *xref, *endofpath;
364 /* Get the headers and detect if wire format. */
365 if (what == STarticle) {
367 p = ARThandle->data + ARThandle->len;
369 for (q = p = ARThandle->data; p < (ARThandle->data + ARThandle->len); p++) {
373 if (lastchar == '\n') {
374 if (what == SThead) {
380 p = ARThandle->data + ARThandle->len;
389 /* q points to the start of the article buffer, p to the end of it */
390 if (VirtualPathlen > 0 && (what != STbody)) {
391 path = wire_findheader(ARThandle->data, ARThandle->len, "Path");
399 xref = wire_findheader(ARThandle->data, ARThandle->len, "Xref");
408 endofpath = wire_endheader(path, ARThandle->data + ARThandle->len - 1);
409 if (endofpath == NULL) {
416 if ((r = memchr(xref, ' ', p - xref)) == NULL || r == p) {
423 /* r points to the first space in the Xref header */
424 for (s = path, lastchar = '\0';
425 s + VirtualPathlen + 1 < endofpath;
427 if ((lastchar != '\0' && lastchar != '!') || *s != *VirtualPath ||
428 strncmp(s, VirtualPath, VirtualPathlen - 1) != 0)
430 if (*(s + VirtualPathlen - 1) != '\0' &&
431 *(s + VirtualPathlen - 1) != '!')
435 if (s + VirtualPathlen + 1 < endofpath) {
437 SendIOv(q, path - q);
438 SendIOv(s, xref - s);
439 SendIOv(VirtualPath, VirtualPathlen - 1);
442 SendIOv(q, xref - q);
443 SendIOv(VirtualPath, VirtualPathlen - 1);
444 SendIOv(r, path - r);
449 SendIOv(q, path - q);
450 SendIOv(VirtualPath, VirtualPathlen);
451 SendIOv(path, xref - path);
452 SendIOv(VirtualPath, VirtualPathlen - 1);
455 SendIOv(q, xref - q);
456 SendIOv(VirtualPath, VirtualPathlen - 1);
457 SendIOv(r, path - r);
458 SendIOv(VirtualPath, VirtualPathlen);
459 SendIOv(path, p - path);
465 if (what == SThead) {
468 } else if (memcmp((ARThandle->data + ARThandle->len - 5), "\r\n.\r\n", 5)) {
469 if (memcmp((ARThandle->data + ARThandle->len - 2), "\r\n", 2)) {
470 SendIOv("\r\n.\r\n", 5);
483 ** Return the header from the specified file, or NULL if not found.
485 char *GetHeader(const char *header)
487 const char *p, *q, *r, *s, *t;
489 /* Bogus value here to make sure that it isn't initialized to \n */
492 const char *cmplimit;
493 static char *retval = NULL;
494 static int retlen = 0;
496 bool pathheader = false;
497 bool xrefheader = false;
499 limit = ARThandle->data + ARThandle->len;
500 cmplimit = ARThandle->data + ARThandle->len - strlen(header) - 1;
501 for (p = ARThandle->data; p < cmplimit; p++) {
504 if ((lastchar == '\n') && (*p == '\n')) {
507 if ((lastchar == '\n') || (p == ARThandle->data)) {
508 headerlen = strlen(header);
509 if (strncasecmp(p, header, headerlen) == 0 && p[headerlen] == ':') {
510 for (; (p < limit) && !isspace((int)*p) ; p++);
511 for (; (p < limit) && isspace((int)*p) ; p++);
512 for (q = p; q < limit; q++)
513 if ((*q == '\r') || (*q == '\n')) {
514 /* Check for continuation header lines */
517 if ((*q == '\r' && *t == '\n')) {
522 if ((*t == '\t' || *t == ' ')) {
523 for (; (t < limit) && isspace((int)*t) ; t++);
534 if (strncasecmp("Path", header, headerlen) == 0)
536 else if (strncasecmp("Xref", header, headerlen) == 0)
538 if (retval == NULL) {
539 retlen = q - p + VirtualPathlen + 1;
540 retval = xmalloc(retlen);
542 if ((q - p + VirtualPathlen + 1) > retlen) {
543 retlen = q - p + VirtualPathlen + 1;
544 retval = xrealloc(retval, retlen);
547 if (pathheader && (VirtualPathlen > 0)) {
548 const char *endofpath;
549 const char *endofarticle;
551 endofarticle = ARThandle->data + ARThandle->len - 1;
552 endofpath = wire_endheader(p, endofarticle);
553 if (endofpath == NULL)
555 for (s = p, prevchar = '\0';
556 s + VirtualPathlen + 1 < endofpath;
558 if ((prevchar != '\0' && prevchar != '!') ||
559 *s != *VirtualPath ||
560 strncmp(s, VirtualPath, VirtualPathlen - 1) != 0)
562 if (*(s + VirtualPathlen - 1) != '\0' &&
563 *(s + VirtualPathlen - 1) != '!')
567 if (s + VirtualPathlen + 1 < endofpath) {
568 memcpy(retval, s, q - s);
569 *(retval + (int)(q - s)) = '\0';
571 memcpy(retval, VirtualPath, VirtualPathlen);
572 memcpy(retval + VirtualPathlen, p, q - p);
573 *(retval + (int)(q - p) + VirtualPathlen) = '\0';
575 } else if (xrefheader && (VirtualPathlen > 0)) {
576 if ((r = memchr(p, ' ', q - p)) == NULL)
578 for (; (r < q) && isspace((int)*r) ; r++);
581 memcpy(retval, VirtualPath, VirtualPathlen - 1);
582 memcpy(retval + VirtualPathlen - 1, r - 1, q - r + 1);
583 *(retval + (int)(q - r) + VirtualPathlen) = '\0';
585 memcpy(retval, p, q - p);
586 *(retval + (int)(q - p)) = '\0';
588 for (w = retval; *w; w++)
589 if (*w == '\n' || *w == '\r')
600 ** Fetch part or all of an article and send it to the client.
602 void CMDfetch(int ac, char *av[])
612 /* Find what to send; get permissions. */
628 /* Poster might do a "head" command to verify the article. */
629 ok = PERMcanread || PERMcanpost;
634 Reply("%s\r\n", NOACCESS);
638 /* Requesting by Message-ID? */
639 if (ac == 2 && av[1][0] == '<') {
640 if (!ARTopenbyid(av[1], &art, final)) {
641 Reply("%d No such article\r\n", NNTP_DONTHAVEIT_VAL);
646 Reply("%s\r\n", NOACCESS);
650 Reply("%d %lu %s %s\r\n", what->ReplyCode, (unsigned long) art,
652 if (what->Type != STstat) {
653 ARTsendmmap(what->Type);
659 /* Trying to read. */
661 Reply("%s\r\n", ARTnotingroup);
665 /* Default is to get current article, or specified article. */
667 if (ARTnumber < ARTlow || ARTnumber > ARThigh) {
668 Reply("%s\r\n", ARTnocurrart);
671 snprintf(buff, sizeof(buff), "%d", ARTnumber);
675 if (strspn(av[1], "0123456789") != strlen(av[1])) {
676 Reply("%s\r\n", ARTnoartingroup);
679 strlcpy(buff, av[1], sizeof(buff));
680 tart=(ARTNUM)atol(buff);
683 /* Open the article and send the reply. */
684 if (!ARTopen(atol(buff))) {
685 Reply("%s\r\n", ARTnoartingroup);
690 if ((msgid = GetHeader("Message-ID")) == NULL) {
691 Reply("%s\r\n", ARTnoartingroup);
694 Reply("%d %s %.512s %s\r\n", what->ReplyCode, buff, msgid, what->Item);
695 if (what->Type != STstat)
696 ARTsendmmap(what->Type);
702 ** Go to the next or last (really previous) article in the group.
704 void CMDnextlast(int ac UNUSED, char *av[])
707 int save, delta, errcode;
712 Reply("%s\r\n", NOACCESS);
716 Reply("%s\r\n", ARTnotingroup);
719 if (ARTnumber < ARTlow || ARTnumber > ARThigh) {
720 Reply("%s\r\n", ARTnocurrart);
724 next = (av[0][0] == 'n' || av[0][0] == 'N');
727 errcode = NNTP_NONEXT_VAL;
732 errcode = NNTP_NOPREV_VAL;
733 message = "previous";
740 if (ARTnumber < ARTlow || ARTnumber > ARThigh) {
741 Reply("%d No %s to retrieve.\r\n", errcode, message);
745 if (!ARTopen(ARTnumber))
747 msgid = GetHeader("Message-ID");
748 } while (msgid == NULL);
751 Reply("%d %d %s Article retrieved; request text separately.\r\n",
752 NNTP_NOTHING_FOLLOWS_VAL, ARTnumber, msgid);
756 static bool CMDgetrange(int ac, char *av[], ARTRANGE *rp, bool *DidReply)
762 Reply("%s\r\n", ARTnotingroup);
768 /* No argument, do only current article. */
769 if (ARTnumber < ARTlow || ARTnumber > ARThigh) {
770 Reply("%s\r\n", ARTnocurrart);
774 rp->High = rp->Low = ARTnumber;
778 /* Got just a single number? */
779 if ((p = strchr(av[1], '-')) == NULL) {
780 rp->Low = rp->High = atol(av[1]);
786 rp->Low = atol(av[1]);
787 if (*p == '\0' || (rp->High = atol(p)) < rp->Low)
788 /* "XHDR 234-0 header" gives everything to the end. */
790 else if (rp->High > ARThigh)
792 if (rp->Low < ARTlow)
802 ** Apply virtual hosting to an Xref field.
811 space = strchr(p, ' ');
813 warn("malformed Xref `%s'", field);
816 offset = space + 1 - p;
817 space = strchr(p + offset, ' ');
819 warn("malformed Xref `%s'", field);
822 field = concat(PERMaccessconf->domain, space, NULL);
829 ** XOVER another extension. Dump parts of the overview database.
831 void CMDxover(int ac, char *av[])
835 struct timeval stv, etv;
842 struct cvector *vector = NULL;
845 Printf("%s\r\n", NOACCESS);
849 /* Trying to read. */
851 Reply("%s\r\n", ARTnotingroup);
856 if (!CMDgetrange(ac, av, &range, &DidReply)) {
858 Reply("%d data follows\r\n", NNTP_OVERVIEW_FOLLOWS_VAL);
865 gettimeofday(&stv, NULL);
866 if ((handle = (void *)OVopensearch(GRPcur, range.Low, range.High)) == NULL) {
868 Reply("%d %s fields follow\r\n.\r\n", NNTP_OVERVIEW_FOLLOWS_VAL, av[1]);
870 Reply("%d %d fields follow\r\n.\r\n", NNTP_OVERVIEW_FOLLOWS_VAL, ARTnumber);
873 if (PERMaccessconf->nnrpdoverstats) {
874 gettimeofday(&etv, NULL);
875 OVERtime+=(etv.tv_sec - stv.tv_sec) * 1000;
876 OVERtime+=(etv.tv_usec - stv.tv_usec) / 1000;
880 Reply("%d %s fields follow\r\n", NNTP_OVERVIEW_FOLLOWS_VAL, av[1]);
882 Reply("%d %d fields follow\r\n", NNTP_OVERVIEW_FOLLOWS_VAL, ARTnumber);
884 if (PERMaccessconf->nnrpdoverstats)
885 gettimeofday(&stv, NULL);
887 /* If OVSTATICSEARCH is true, then the data returned by OVsearch is only
888 valid until the next call to OVsearch. In this case, we must use
889 SendIOb because it copies the data. */
890 OVctl(OVSTATICSEARCH, &useIOb);
892 while (OVsearch(handle, &artnum, &data, &len, &token, NULL)) {
893 if (PERMaccessconf->nnrpdoverstats) {
894 gettimeofday(&etv, NULL);
895 OVERtime+=(etv.tv_sec - stv.tv_sec) * 1000;
896 OVERtime+=(etv.tv_usec - stv.tv_usec) / 1000;
898 if (len == 0 || (PERMaccessconf->nnrpdcheckart && !ARTinstorebytoken(token))) {
899 if (PERMaccessconf->nnrpdoverstats) {
901 gettimeofday(&stv, NULL);
905 if (PERMaccessconf->nnrpdoverstats) {
909 vector = overview_split(data, len, NULL, vector);
910 r = overview_getheader(vector, OVERVIEW_MESSAGE_ID, OVextra);
911 cache_add(HashMessageID(r), token);
913 if (VirtualPathlen > 0 && overhdr_xref != -1) {
914 if ((overhdr_xref + 1) >= vector->count)
916 p = vector->strings[overhdr_xref] + sizeof("Xref: ") - 1;
917 while ((p < data + len) && *p == ' ')
919 q = memchr(p, ' ', data + len - p);
923 SendIOb(data, p - data);
924 SendIOb(VirtualPath, VirtualPathlen - 1);
925 SendIOb(q, len - (q - data));
927 SendIOv(data, p - data);
928 SendIOv(VirtualPath, VirtualPathlen - 1);
929 SendIOv(q, len - (q - data));
937 if (PERMaccessconf->nnrpdoverstats)
938 gettimeofday(&stv, NULL);
942 cvector_free(vector);
944 if (PERMaccessconf->nnrpdoverstats) {
945 gettimeofday(&etv, NULL);
946 OVERtime+=(etv.tv_sec - stv.tv_sec) * 1000;
947 OVERtime+=(etv.tv_usec - stv.tv_usec) / 1000;
956 if (PERMaccessconf->nnrpdoverstats)
957 gettimeofday(&stv, NULL);
958 OVclosesearch(handle);
959 if (PERMaccessconf->nnrpdoverstats) {
960 gettimeofday(&etv, NULL);
961 OVERtime+=(etv.tv_sec - stv.tv_sec) * 1000;
962 OVERtime+=(etv.tv_usec - stv.tv_usec) / 1000;
968 ** XHDR and XPAT extensions. Note that HDR as specified in the new NNTP
969 ** draft works differently than XHDR has historically, so don't just use this
970 ** function to implement it without reviewing the differences.
973 void CMDpat(int ac, char *av[])
985 char buff[SPOOLNAMEBUFF];
990 struct cvector *vector = NULL;
993 Printf("%s\r\n", NOACCESS);
998 IsLines = (strcasecmp(header, "lines") == 0);
1000 if (ac > 3) /* XPAT */
1001 pattern = Glom(&av[3]);
1006 /* Message-ID specified? */
1007 if (ac > 2 && av[2][0] == '<') {
1009 if (!ARTopenbyid(p, &artnum, false)) {
1010 Printf("%d No such article.\r\n", NNTP_DONTHAVEIT_VAL);
1013 Printf("%d %s matches follow (ID)\r\n", NNTP_HEAD_FOLLOWS_VAL,
1015 if ((text = GetHeader(header)) != NULL
1016 && (!pattern || uwildmat_simple(text, pattern)))
1017 Printf("%s %s\r\n", p, text);
1024 if (GRPcount == 0) {
1025 Reply("%s\r\n", ARTnotingroup);
1029 /* Range specified. */
1030 if (!CMDgetrange(ac - 1, av + 1, &range, &DidReply)) {
1032 Reply("%d %s no matches follow (range)\r\n",
1033 NNTP_HEAD_FOLLOWS_VAL, header ? header : "\"\"");
1040 Overview = overview_index(header, OVextra);
1042 /* Not in overview, we have to fish headers out from the articles */
1043 if (Overview < 0 ) {
1044 Reply("%d %s matches follow (art)\r\n", NNTP_HEAD_FOLLOWS_VAL,
1046 for (i = range.Low; i <= range.High && range.High > 0; i++) {
1049 p = GetHeader(header);
1050 if (p && (!pattern || uwildmat_simple(p, pattern))) {
1051 snprintf(buff, sizeof(buff), "%u ", i);
1052 SendIOb(buff, strlen(buff));
1053 SendIOb(p, strlen(p));
1058 SendIOb(".\r\n", 3);
1063 /* Okay then, we can grab values from overview. */
1064 handle = (void *)OVopensearch(GRPcur, range.Low, range.High);
1065 if (handle == NULL) {
1066 Reply("%d %s no matches follow (NOV)\r\n.\r\n",
1067 NNTP_HEAD_FOLLOWS_VAL, header);
1071 Printf("%d %s matches follow (NOV)\r\n", NNTP_HEAD_FOLLOWS_VAL,
1073 while (OVsearch(handle, &artnum, &data, &len, &token, NULL)) {
1074 if (len == 0 || (PERMaccessconf->nnrpdcheckart
1075 && !ARTinstorebytoken(token)))
1077 vector = overview_split(data, len, NULL, vector);
1078 p = overview_getheader(vector, Overview, OVextra);
1080 if (PERMaccessconf->virtualhost &&
1081 Overview == overhdr_xref) {
1086 if (!pattern || uwildmat_simple(p, pattern)) {
1087 snprintf(buff, sizeof(buff), "%lu ", artnum);
1088 SendIOb(buff, strlen(buff));
1089 SendIOb(p, strlen(p));
1095 SendIOb(".\r\n", 3);
1097 OVclosesearch(handle);
1101 cvector_free(vector);