journal: provide an API that allows client to figure out whether they need to recheck...

Network file systems generally do not offer inotify() that would work
across the network. We hence cannot rely on inotify() exclusiely in
those case. Provide an API to determine these cases, and suggest doing
manual regular rechecks.

Note that this is not complete yet, as we need to rescan journal dirs on
network file systems explicitly to find new/removed files

diff --git a/Makefile.am b/Makefile.am
index 0bcde9c58440a382a8eb2260638da5ae18b4cb04..634c67d13f971f41e5799e861d781d51431f1e12 100644 (file)
--- a/Makefile.am
+++ b/Makefile.am
@@ -576,6 +576,7 @@ MANPAGES_ALIAS = \
        man/SD_JOURNAL_FOREACH_DATA.3 \
        man/sd_journal_get_monotonic_usec.3 \
        man/sd_journal_get_cutoff_monotonic_usec.3 \
        man/SD_JOURNAL_FOREACH_DATA.3 \
        man/sd_journal_get_monotonic_usec.3 \
        man/sd_journal_get_cutoff_monotonic_usec.3 \

+       man/sd_journal_reliable_fd.3 \

        man/sd_journal_process.3 \
        man/sd_journal_wait.3 \
        man/SD_JOURNAL_NOP.3 \
        man/sd_journal_process.3 \
        man/sd_journal_wait.3 \
        man/SD_JOURNAL_NOP.3 \


@@ -649,6 +650,7 @@ man/sd_journal_restart_data.3: man/sd_journal_get_data.3
 man/SD_JOURNAL_FOREACH_DATA.3: man/sd_journal_get_data.3
 man/sd_journal_get_monotonic_usec.3: man/sd_journal_get_realtime_usec.3
 man/sd_journal_get_cutoff_monotonic_usec.3: man/sd_journal_get_cutoff_realtime_usec.3
 man/SD_JOURNAL_FOREACH_DATA.3: man/sd_journal_get_data.3
 man/sd_journal_get_monotonic_usec.3: man/sd_journal_get_realtime_usec.3
 man/sd_journal_get_cutoff_monotonic_usec.3: man/sd_journal_get_cutoff_realtime_usec.3

+man/sd_journal_reliable_fd.3: man/sd_journal_get_fd.3

 man/sd_journal_process.3: man/sd_journal_get_fd.3
 man/sd_journal_wait.3: man/sd_journal_get_fd.3
 man/SD_JOURNAL_NOP.3: man/sd_journal_get_fd.3
 man/sd_journal_process.3: man/sd_journal_get_fd.3
 man/sd_journal_wait.3: man/sd_journal_get_fd.3
 man/SD_JOURNAL_NOP.3: man/sd_journal_get_fd.3

diff --git a/man/sd_journal_get_fd.xml b/man/sd_journal_get_fd.xml
index e3b5eaa13d13a11f37121dec1aee4a2fccb25abf..189d21352bd81a99d94af59496c65b2d0a4d1d57 100644 (file)
--- a/man/sd_journal_get_fd.xml
+++ b/man/sd_journal_get_fd.xml
@@ -44,6 +44,7 @@
 
         <refnamediv>
                 <refname>sd_journal_get_fd</refname>
 
         <refnamediv>
                 <refname>sd_journal_get_fd</refname>

+                <refname>sd_journal_reliable_fd</refname>

                 <refname>sd_journal_process</refname>
                 <refname>sd_journal_wait</refname>
                 <refname>SD_JOURNAL_NOP</refname>
                 <refname>sd_journal_process</refname>
                 <refname>sd_journal_wait</refname>
                 <refname>SD_JOURNAL_NOP</refname>


@@ -62,6 +63,11 @@
                                 <paramdef>sd_journal* <parameter>j</parameter></paramdef>
                         </funcprototype>
 
                                 <paramdef>sd_journal* <parameter>j</parameter></paramdef>
                         </funcprototype>
 

+                        <funcprototype>
+                                <funcdef>int <function>sd_journal_reliable_fd</function></funcdef>
+                                <paramdef>sd_journal* <parameter>j</parameter></paramdef>
+                        </funcprototype>
+

                         <funcprototype>
                                 <funcdef>int <function>sd_journal_process</function></funcdef>
                                 <paramdef>sd_journal* <parameter>j</parameter></paramdef>
                         <funcprototype>
                                 <funcdef>int <function>sd_journal_process</function></funcdef>
                                 <paramdef>sd_journal* <parameter>j</parameter></paramdef>


@@ -82,12 +88,36 @@
                 <para><function>sd_journal_get_fd()</function> returns
                 a file descriptor that may be asynchronously polled in
                 an external event loop and is signaled readable as
                 <para><function>sd_journal_get_fd()</function> returns
                 a file descriptor that may be asynchronously polled in
                 an external event loop and is signaled readable as

-                soon as the journal changes, for example because new
-                entries were added. The file descriptor is suitable
-                for usage in
+                soon as the journal changes, because new entries or
+                files were added, rotation took place, or files have
+                been deleted, and similar. The file descriptor is
+                suitable for usage in

                 <citerefentry><refentrytitle>poll</refentrytitle><manvolnum>2</manvolnum></citerefentry>
                 <citerefentry><refentrytitle>poll</refentrytitle><manvolnum>2</manvolnum></citerefentry>

-                where it will yield POLLIN on all changes. The call
-                takes one argument: the journal context object.</para>
+                where it will yield POLLIN on changes. The call takes
+                one argument: the journal context object. Note that
+                not all file systems are capable of generating the
+                necessary events for wakeups from this file descriptor
+                to be enirely reliable. In particular network files
+                systems do not generate suitable file change events in
+                all cases. In such a case an application should not
+                rely alone on wake-ups from this file descriptor but
+                wake up and recheck the journal in regular time
+                intervals, for example every 2s. To detect
+                cases where this is necessary, use
+                <function>sd_journal_reliable_fd()</function>,
+                below.</para>
+
+                <para><function>sd_journal_reliable_fd()</function>
+                may be used to check whether the wakeup events from
+                the file descriptor returned by
+                <function>sd_journal_get_fd</function> are sufficient
+                to track changes to the journal. If this call returns
+                0, it is necessary to regularly recheck for journal
+                changes (suggestion: every 2s). If this call returns a
+                positive integer this is not necessary, and wakeups
+                from the file descriptor returned by
+                <function>sd_journal_get_fd()</function> are
+                sufficient as only source for wake-ups.</para>

 
                 <para>After each POLLIN wake-up
                 <function>sd_journal_process()</function> needs to be
 
                 <para>After each POLLIN wake-up
                 <function>sd_journal_process()</function> needs to be


@@ -97,14 +127,19 @@
                 that spurious wake-ups are possible).</para>
 
                 <para>A synchronous alternative for using
                 that spurious wake-ups are possible).</para>
 
                 <para>A synchronous alternative for using

-                <function>sd_journal_get_fd()</function> and
+                <function>sd_journal_get_fd()</function>,
+                <function>sd_journal_reliable_fd()</function> and

                 <function>sd_journal_process()</function> is
                 <function>sd_journal_wait()</function>. It will
                 <function>sd_journal_process()</function> is
                 <function>sd_journal_wait()</function>. It will

-                synchronously wait until the journal gets changed up
-                to a certain time-out as specified in its second
-                argument. Pass <literal>(uint64_t) -1</literal> to
-                wait indefinitely. Internally this call simply
-                combines <function>sd_journal_get_fd()</function>,
+                synchronously wait until the journal gets changed,
+                possibly using a 2s time-out if this is necessary (see
+                above). In either way the maximum time this call
+                sleeps may be controlled with the
+                <parameter>timeout_usec</parameter> parameter. Pass
+                <literal>(uint64_t) -1</literal> to wait
+                indefinitely. Internally this call simply combines
+                <function>sd_journal_get_fd()</function>,
+                <function>sd_journal_reliable_fd()</function>,

                 <function>poll()</function> and
                 <function>sd_journal_process()</function> into
                 one.</para>
                 <function>poll()</function> and
                 <function>sd_journal_process()</function> into
                 one.</para>


@@ -117,6 +152,15 @@
                 <para><function>sd_journal_get_fd()</function> returns a valid file descriptor on success or a negative errno-style error
                 code.</para>
 
                 <para><function>sd_journal_get_fd()</function> returns a valid file descriptor on success or a negative errno-style error
                 code.</para>
 

+                <para><function>sd_journal_reliable_fd()</function>
+                returns a positive integer if the file descriptor
+                returned by <function>sd_journal_get_fd()</function>
+                is sufficient as sole wake-up source for journal
+                change events. Returns 0 if it is not sufficient and
+                the journal needs to be checked manually in regular
+                time intervals for changes. Returns a negative
+                errno-style error code on failure.</para>
+

                 <para><function>sd_journal_process()</function> and
                 <function>sd_journal_wait()</function> return one of
                 <literal>SD_JOURNAL_NOP</literal>,
                 <para><function>sd_journal_process()</function> and
                 <function>sd_journal_wait()</function> return one of
                 <literal>SD_JOURNAL_NOP</literal>,


@@ -140,6 +184,7 @@
                 <title>Notes</title>
 
                 <para>The <function>sd_journal_get_fd()</function>,
                 <title>Notes</title>
 
                 <para>The <function>sd_journal_get_fd()</function>,

+                <function>sd_journal_reliable_fd()</function>,

                 <function>sd_journal_process()</function> and
                 <function>sd_journal_wait()</function> interfaces are
                 available as shared library, which can be compiled and
                 <function>sd_journal_process()</function> and
                 <function>sd_journal_wait()</function> interfaces are
                 available as shared library, which can be compiled and


@@ -149,6 +194,69 @@
                 file.</para>
         </refsect1>
 
                 file.</para>
         </refsect1>
 

+        <refsect1>
+                <title>Examples</title>
+
+                <para>Iterating through the journal, in a live view tracking all changes:</para>
+
+                <programlisting>#include &lt;stdio.h&gt;
+#include &lt;string.h&gt;
+#include &lt;systemd/sd-journal.h&gt;
+
+int main(int argc, char *argv[]) {
+        int r;
+        sd_journal *j;
+        r = sd_journal_open(&amp;j, SD_JOURNAL_LOCAL_ONLY);
+        if (r &lt; 0) {
+                fprintf(stderr, "Failed to open journal: %s\n", strerror(-r));
+                return 1;
+        }
+        for (;;)  {
+                const char *d;
+                size_t l;
+                r = sd_journal_next(j);
+                if (r &lt; 0) {
+                        fprintf(stderr, "Failed to iterate to next entry: %s\n", strerror(-r));
+                        break;
+                }
+                if (r == 0) {
+                        /* Reached the end, let's wait for changes, and try again */
+                        r = sd_journal_wait(j, (uint64_t) -1);
+                        if (r &lt; 0) {
+                                fprintf(stderr, "Failed to wait for changes: %s\n", strerror(-r));
+                                break;
+                        }
+                        continue;
+                }
+                r = sd_journal_get_data(j, "MESSAGE", &amp;d, &amp;l);
+                if (r &lt; 0) {
+                        fprintf(stderr, "Failed to read message field: %s\n", strerror(-r));
+                        continue;
+                }
+                printf("%.*s\n", (int) l, d);
+        }
+        sd_journal_close(j);
+        return 0;
+}</programlisting>
+
+                <para>Waiting with <function>poll()</function> (this
+                example lacks all error checking for the sake of
+                simplicity):</para>
+
+                <programlisting>#include &lt;sys/poll.h&gt;
+#include &lt;systemd/sd-journal.h&gt;
+
+int wait_for_changes(sd_journal *j) {
+        struct pollfd pollfd;
+        pollfd.fd = sd_journal_get_fd();
+        pollfd.events = POLLIN;
+        poll(&amp;pollfd, 1, sd_journal_reliable_fd() &gt; 0 ? -1 : 2000);
+        return sd_journal_process(j);
+}
+                </programlisting>
+        </refsect1>
+
+

         <refsect1>
                 <title>See Also</title>
 
         <refsect1>
                 <title>See Also</title>
 


@@ -156,7 +264,8 @@
                         <citerefentry><refentrytitle>systemd</refentrytitle><manvolnum>1</manvolnum></citerefentry>,
                         <citerefentry><refentrytitle>sd-journal</refentrytitle><manvolnum>3</manvolnum></citerefentry>,
                         <citerefentry><refentrytitle>sd_journal_open</refentrytitle><manvolnum>3</manvolnum></citerefentry>,
                         <citerefentry><refentrytitle>systemd</refentrytitle><manvolnum>1</manvolnum></citerefentry>,
                         <citerefentry><refentrytitle>sd-journal</refentrytitle><manvolnum>3</manvolnum></citerefentry>,
                         <citerefentry><refentrytitle>sd_journal_open</refentrytitle><manvolnum>3</manvolnum></citerefentry>,

-                        <citerefentry><refentrytitle>sd_journal_next</refentrytitle><manvolnum>3</manvolnum></citerefentry>
+                        <citerefentry><refentrytitle>sd_journal_next</refentrytitle><manvolnum>3</manvolnum></citerefentry>,
+                        <citerefentry><refentrytitle>poll</refentrytitle><manvolnum>2</manvolnum></citerefentry>

                 </para>
         </refsect1>
 
                 </para>
         </refsect1>
 

diff --git a/src/journal/journal-internal.h b/src/journal/journal-internal.h
index f68ca996a83b793bc61a6ef337a2399f6f5a0b43..75a4129e5b4aac106be2ae5f2e7a6e962c42ce8a 100644 (file)
--- a/src/journal/journal-internal.h
+++ b/src/journal/journal-internal.h
@@ -119,6 +119,8 @@ struct sd_journal {
         char *unique_field;
         JournalFile *unique_file;
         uint64_t unique_offset;
         char *unique_field;
         JournalFile *unique_file;
         uint64_t unique_offset;

+
+        bool on_network;

 };
 
 char *journal_make_match_string(sd_journal *j);
 };
 
 char *journal_make_match_string(sd_journal *j);

diff --git a/src/journal/libsystemd-journal.sym b/src/journal/libsystemd-journal.sym
index 96293b8a29e17dfeec50d925c985351ef9177baa..ad78fcc74d9dd3526d85210ece42a994eb6a1c7e 100644 (file)
--- a/src/journal/libsystemd-journal.sym
+++ b/src/journal/libsystemd-journal.sym
@@ -80,3 +80,8 @@ global:
         sd_journal_enumerate_unique;
         sd_journal_restart_unique;
 } LIBSYSTEMD_JOURNAL_190;
         sd_journal_enumerate_unique;
         sd_journal_restart_unique;
 } LIBSYSTEMD_JOURNAL_190;

+
+LIBSYSTEMD_JOURNAL_196 {
+global:
+        sd_journal_fd_reliable;
+} LIBSYSTEMD_JOURNAL_195;

diff --git a/src/journal/sd-journal.c b/src/journal/sd-journal.c
index d5d2d78f1c30470389799028c3f7d4f8f62b79b8..a346691e21e0be33b151d3052b82297bdecad779 100644 (file)
--- a/src/journal/sd-journal.c
+++ b/src/journal/sd-journal.c
@@ -25,6 +25,8 @@
 #include <unistd.h>
 #include <sys/inotify.h>
 #include <sys/poll.h>
 #include <unistd.h>
 #include <sys/inotify.h>
 #include <sys/poll.h>

+#include <sys/vfs.h>
+#include <linux/magic.h>

 
 #include "sd-journal.h"
 #include "journal-def.h"
 
 #include "sd-journal.h"
 #include "journal-def.h"


@@ -35,9 +37,12 @@
 #include "lookup3.h"
 #include "compress.h"
 #include "journal-internal.h"
 #include "lookup3.h"
 #include "compress.h"
 #include "journal-internal.h"

+#include "missing.h"

 
 #define JOURNAL_FILES_MAX 1024
 
 
 #define JOURNAL_FILES_MAX 1024
 

+#define JOURNAL_FILES_RECHECK_USEC (2 * USEC_PER_SEC)
+

 static void detach_location(sd_journal *j) {
         Iterator i;
         JournalFile *f;
 static void detach_location(sd_journal *j) {
         Iterator i;
         JournalFile *f;


@@ -1184,6 +1189,25 @@ _public_ int sd_journal_seek_tail(sd_journal *j) {
         return 0;
 }
 
         return 0;
 }
 

+static void check_network(sd_journal *j, int fd) {
+        struct statfs sfs;
+
+        assert(j);
+
+        if (j->on_network)
+                return;
+
+        if (fstatfs(fd, &sfs) < 0)
+                return;
+
+        j->on_network =
+                sfs.f_type == CIFS_MAGIC_NUMBER ||
+                sfs.f_type == CODA_SUPER_MAGIC ||
+                sfs.f_type == NCP_SUPER_MAGIC ||
+                sfs.f_type == NFS_SUPER_MAGIC ||
+                sfs.f_type == SMB_SUPER_MAGIC;
+}
+

 static int add_file(sd_journal *j, const char *prefix, const char *filename) {
         char *path;
         int r;
 static int add_file(sd_journal *j, const char *prefix, const char *filename) {
         char *path;
         int r;


@@ -1233,6 +1257,8 @@ static int add_file(sd_journal *j, const char *prefix, const char *filename) {
                 return r;
         }
 
                 return r;
         }
 

+        check_network(j, f->fd);
+

         j->current_invalidate_counter ++;
 
         log_debug("File %s got added.", f->path);
         j->current_invalidate_counter ++;
 
         log_debug("File %s got added.", f->path);


@@ -1366,6 +1392,8 @@ static int add_directory(sd_journal *j, const char *prefix, const char *dirname)
                 }
         }
 
                 }
         }
 

+        check_network(j, dirfd(d));
+

         closedir(d);
 
         return 0;
         closedir(d);
 
         return 0;


@@ -1453,6 +1481,8 @@ static int add_root_directory(sd_journal *j, const char *p) {
                 }
         }
 
                 }
         }
 

+        check_network(j, dirfd(d));
+

         closedir(d);
 
         return 0;
         closedir(d);
 
         return 0;


@@ -2079,6 +2109,14 @@ _public_ int sd_journal_wait(sd_journal *j, uint64_t timeout_usec) {
                 return determine_change(j);
         }
 
                 return determine_change(j);
         }
 

+        if (j->on_network) {
+                /* If we are on the network we need to regularly check
+                 * for changes manually */
+
+                if (timeout_usec == (uint64_t) -1 || timeout_usec > JOURNAL_FILES_RECHECK_USEC)
+                        timeout_usec = JOURNAL_FILES_RECHECK_USEC;
+        }
+

         do {
                 r = fd_wait_for_event(j->inotify_fd, POLLIN, timeout_usec);
         } while (r == -EINTR);
         do {
                 r = fd_wait_for_event(j->inotify_fd, POLLIN, timeout_usec);
         } while (r == -EINTR);


@@ -2344,3 +2382,10 @@ _public_ void sd_journal_restart_unique(sd_journal *j) {
         j->unique_file = NULL;
         j->unique_offset = 0;
 }
         j->unique_file = NULL;
         j->unique_offset = 0;
 }

+
+_public_ int sd_journal_reliable_fd(sd_journal *j) {
+        if (!j)
+                return -EINVAL;
+
+        return !j->on_network;
+}

diff --git a/src/shared/missing.h b/src/shared/missing.h
index f58a609c7d7edd1ae365343d1188b1951bf65721..3777cf69b21015310e516a8c7b30bbc6c5ba2f70 100644 (file)
--- a/src/shared/missing.h
+++ b/src/shared/missing.h
@@ -249,3 +249,7 @@ static inline int name_to_handle_at(int fd, const char *name, struct file_handle
 #    error neither secure_getenv nor __secure_getenv are available
 #  endif
 #endif
 #    error neither secure_getenv nor __secure_getenv are available
 #  endif
 #endif

+
+#ifndef CIFS_MAGIC_NUMBER
+#define CIFS_MAGIC_NUMBER 0xFF534D42
+#endif

diff --git a/src/systemd/sd-journal.h b/src/systemd/sd-journal.h
index 55281b1346be0488df7318aef041b5c338544288..724117303512a0f271121767bd8a48de083f6e90 100644 (file)
--- a/src/systemd/sd-journal.h
+++ b/src/systemd/sd-journal.h
@@ -124,6 +124,7 @@ int sd_journal_enumerate_unique(sd_journal *j, const void **data, size_t *l);
 void sd_journal_restart_unique(sd_journal *j);
 
 int sd_journal_get_fd(sd_journal *j);
 void sd_journal_restart_unique(sd_journal *j);
 
 int sd_journal_get_fd(sd_journal *j);

+int sd_journal_reliable_fd(sd_journal *j);

 int sd_journal_process(sd_journal *j);
 int sd_journal_wait(sd_journal *j, uint64_t timeout_usec);
 
 int sd_journal_process(sd_journal *j);
 int sd_journal_wait(sd_journal *j, uint64_t timeout_usec);