Fix problem with allocating large buffers and log leftovers

diff --git a/src/journal-remote/journal-remote-parse.c b/src/journal-remote/journal-remote-parse.c
index a08ca2fdcbe29d5f94d87c9c52953dbdb2a75dca..14af55ca552465ad4e276e8a52871781fdb11ed1 100644 (file)
--- a/src/journal-remote/journal-remote-parse.c
+++ b/src/journal-remote/journal-remote-parse.c
@@ -22,7 +22,7 @@
 #include "journal-remote-parse.h"
 #include "journald-native.h"
 
-#define LINE_CHUNK 1024u
+#define LINE_CHUNK 8*1024u
 
 void source_free(RemoteSource *source) {
         if (!source)
@@ -71,13 +71,17 @@ static int get_line(RemoteSource *source, char **line, size_t *size) {
 
                 if (source->size - source->filled < LINE_CHUNK &&
                     !GREEDY_REALLOC(source->buf, source->size,
-                                    MAX(source->filled + LINE_CHUNK, DATA_SIZE_MAX)))
+                                    MIN(source->filled + LINE_CHUNK, DATA_SIZE_MAX)))
                                 return log_oom();
 
-                assert(source->size - source->filled >= LINE_CHUNK);
+                assert(source->size - source->filled >= LINE_CHUNK ||
+                       source->size == DATA_SIZE_MAX);
+
+                // FIXME: the buffer probably needs to be bigger than DATA_SIZE_MAX
+                // to accomodate such big fields.
 
                 n = read(source->fd, source->buf + source->filled,
-                         MAX(source->size, DATA_SIZE_MAX) - source->filled);
+                         source->size - source->filled);
                 if (n < 0) {
                         if (errno != EAGAIN && errno != EWOULDBLOCK)
                                 log_error("read(%d, ..., %zd): %m", source->fd,

diff --git a/src/journal-remote/journal-remote-parse.h b/src/journal-remote/journal-remote-parse.h
index b02b5a6372a067360c3b45219edbef163e87a1e8..5b7f23650cbf9d3863360d058df2fd8a90d04248 100644 (file)
--- a/src/journal-remote/journal-remote-parse.h
+++ b/src/journal-remote/journal-remote-parse.h
@@ -50,10 +50,10 @@ typedef struct RemoteSource {
         sd_event_source *event;
 } RemoteSource;
 
-static inline int source_non_empty(RemoteSource *source) {
+static inline size_t source_non_empty(RemoteSource *source) {
         assert(source);
 
-        return source->filled > 0;
+        return source->filled;
 }
 
 void source_free(RemoteSource *source);

diff --git a/src/journal-remote/journal-remote.c b/src/journal-remote/journal-remote.c
index 6925fe2f8781d3ef1b3f3c2e4a4ee2fdb086e3b0..827916d1a0af6ffde0aee52c25259bdff379cafc 100644 (file)
--- a/src/journal-remote/journal-remote.c
+++ b/src/journal-remote/journal-remote.c
@@ -491,6 +491,7 @@ static int process_http_upload(
         Writer *w;
         int r;
         bool finished = false;
+        size_t remaining;
 
         assert(source);
 
@@ -542,10 +543,12 @@ static int process_http_upload(
 
         /* The upload is finished */
 
-        if (source_non_empty(source)) {
-                log_warning("EOF reached with incomplete data");
-                return mhd_respond(connection, MHD_HTTP_EXPECTATION_FAILED,
-                                   "Trailing data not processed.");
+        remaining = source_non_empty(source);
+        if (remaining > 0) {
+                log_warning("Premature EOFbyte. %zu bytes lost.", remaining);
+                return mhd_respondf(connection, MHD_HTTP_EXPECTATION_FAILED,
+                                    "Premature EOF. %zu bytes of trailing data not processed.",
+                                    remaining);
         }
 
         return mhd_respond(connection, MHD_HTTP_ACCEPTED, "OK.\n");
@@ -1003,6 +1006,7 @@ static int server_destroy(RemoteServer *s, uint64_t *event_count) {
         *event_count = 0;
 
         while ((w = hashmap_steal_first(s->writers))) {
+                log_info("seqnum %"PRIu64, w->seqnum);
                 *event_count += w->seqnum;
 
                 r = writer_close(w);
@@ -1064,10 +1068,14 @@ static int dispatch_raw_source_event(sd_event_source *event,
 
         r = process_source(source, w, arg_compress, arg_seal);
         if (source->state == STATE_EOF) {
+                size_t remaining;
+
                 log_info("EOF reached with source fd:%d (%s)",
                          source->fd, source->name);
-                if (source_non_empty(source))
-                        log_warning("EOF reached with incomplete data");
+
+                remaining = source_non_empty(source);
+                if (remaining > 0)
+                        log_warning("Premature EOF. %zu bytes lost.", remaining);
                 remove_source(s, source->fd);
                 log_info("%zd active source remaining", s->active);
                 return 0;