chiark / gitweb /
Fix problem with allocating large buffers and log leftovers
authorZbigniew Jędrzejewski-Szmek <zbyszek@in.waw.pl>
Wed, 2 Jul 2014 03:07:45 +0000 (23:07 -0400)
committerZbigniew Jędrzejewski-Szmek <zbyszek@in.waw.pl>
Wed, 16 Jul 2014 02:34:40 +0000 (22:34 -0400)
src/journal-remote/journal-remote-parse.c
src/journal-remote/journal-remote-parse.h
src/journal-remote/journal-remote.c

index a08ca2f..14af55c 100644 (file)
@@ -22,7 +22,7 @@
 #include "journal-remote-parse.h"
 #include "journald-native.h"
 
-#define LINE_CHUNK 1024u
+#define LINE_CHUNK 8*1024u
 
 void source_free(RemoteSource *source) {
         if (!source)
@@ -71,13 +71,17 @@ static int get_line(RemoteSource *source, char **line, size_t *size) {
 
                 if (source->size - source->filled < LINE_CHUNK &&
                     !GREEDY_REALLOC(source->buf, source->size,
-                                    MAX(source->filled + LINE_CHUNK, DATA_SIZE_MAX)))
+                                    MIN(source->filled + LINE_CHUNK, DATA_SIZE_MAX)))
                                 return log_oom();
 
-                assert(source->size - source->filled >= LINE_CHUNK);
+                assert(source->size - source->filled >= LINE_CHUNK ||
+                       source->size == DATA_SIZE_MAX);
+
+                // FIXME: the buffer probably needs to be bigger than DATA_SIZE_MAX
+                // to accomodate such big fields.
 
                 n = read(source->fd, source->buf + source->filled,
-                         MAX(source->size, DATA_SIZE_MAX) - source->filled);
+                         source->size - source->filled);
                 if (n < 0) {
                         if (errno != EAGAIN && errno != EWOULDBLOCK)
                                 log_error("read(%d, ..., %zd): %m", source->fd,
index b02b5a6..5b7f236 100644 (file)
@@ -50,10 +50,10 @@ typedef struct RemoteSource {
         sd_event_source *event;
 } RemoteSource;
 
-static inline int source_non_empty(RemoteSource *source) {
+static inline size_t source_non_empty(RemoteSource *source) {
         assert(source);
 
-        return source->filled > 0;
+        return source->filled;
 }
 
 void source_free(RemoteSource *source);
index 6925fe2..827916d 100644 (file)
@@ -491,6 +491,7 @@ static int process_http_upload(
         Writer *w;
         int r;
         bool finished = false;
+        size_t remaining;
 
         assert(source);
 
@@ -542,10 +543,12 @@ static int process_http_upload(
 
         /* The upload is finished */
 
-        if (source_non_empty(source)) {
-                log_warning("EOF reached with incomplete data");
-                return mhd_respond(connection, MHD_HTTP_EXPECTATION_FAILED,
-                                   "Trailing data not processed.");
+        remaining = source_non_empty(source);
+        if (remaining > 0) {
+                log_warning("Premature EOFbyte. %zu bytes lost.", remaining);
+                return mhd_respondf(connection, MHD_HTTP_EXPECTATION_FAILED,
+                                    "Premature EOF. %zu bytes of trailing data not processed.",
+                                    remaining);
         }
 
         return mhd_respond(connection, MHD_HTTP_ACCEPTED, "OK.\n");
@@ -1003,6 +1006,7 @@ static int server_destroy(RemoteServer *s, uint64_t *event_count) {
         *event_count = 0;
 
         while ((w = hashmap_steal_first(s->writers))) {
+                log_info("seqnum %"PRIu64, w->seqnum);
                 *event_count += w->seqnum;
 
                 r = writer_close(w);
@@ -1064,10 +1068,14 @@ static int dispatch_raw_source_event(sd_event_source *event,
 
         r = process_source(source, w, arg_compress, arg_seal);
         if (source->state == STATE_EOF) {
+                size_t remaining;
+
                 log_info("EOF reached with source fd:%d (%s)",
                          source->fd, source->name);
-                if (source_non_empty(source))
-                        log_warning("EOF reached with incomplete data");
+
+                remaining = source_non_empty(source);
+                if (remaining > 0)
+                        log_warning("Premature EOF. %zu bytes lost.", remaining);
                 remove_source(s, source->fd);
                 log_info("%zd active source remaining", s->active);
                 return 0;