Fix extending user-list feeds.

In lists of followers, favers, boosters etc, the ids you have to pass
in the 'max_id' or 'since_id' URL query parameters are not the same as
the account ids you get back. That's not surprising, now I think about
it, since the chronological order in which users did things has
nothing to do with any intrinsic ordering on the users themselves.

For status feeds, they seemed to match, so I got away without having
to pay attention to the Link: headers in the HTTP responses. But now I
realise I do have to worry about that after all. Happily, there's a
handy Rust crate that saves me having to do the work of parsing that
complicated header type!

diff --git a/Cargo.toml b/Cargo.toml
index 84977f07664ed7389636adc692dd18061c8d5391..2bbdf2e3b65f4ae1dbb444e01acab57fb8dc56cb 100644 (file)
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -11,6 +11,7 @@ crossterm = "0.27.0"
 html2text = { version = "0.10.1", features = ["css"] }
 itertools = "0.12.0"
 lazy_static = "1.4.0"
+parse_link_header = { version = "0.3.3", features = ["url"] }
 ratatui = "0.25.0"
 regex = "1.10.2"
 reqwest = { version = "0.11.23", features = ["blocking"] }

diff --git a/src/client.rs b/src/client.rs
index 46ce4a568be5e7542bd0773fa77ae58354fb23be..5c4d5b0bcf91c63c7a63b8d94b5bf9c06e69edc0 100644 (file)
--- a/src/client.rs
+++ b/src/client.rs
@@ -51,6 +51,9 @@ pub struct StreamUpdate {
 pub struct Feed {
     pub ids: VecDeque<String>, // ids, whether of statuses, accounts or what
     pub origin: isize,
+
+    extend_past: Option<HashMap<String, String>>,
+    extend_future: Option<HashMap<String, String>>,
 }
 
 pub struct Client {
@@ -180,6 +183,9 @@ impl ReqParam for String {
 impl ReqParam for &String {
     fn param_value(self) -> String { self.clone() }
 }
+impl ReqParam for i32 {
+    fn param_value(self) -> String { self.to_string() }
+}
 impl ReqParam for bool {
     fn param_value(self) -> String {
         match self {
@@ -640,17 +646,29 @@ impl Client {
         };
 
         let req = match ext {
-            FeedExtend::Initial => req,
+            FeedExtend::Initial => req.param("limit", 32),
             FeedExtend::Past => if let Some(ref feed) = self.feeds.get(&id) {
-                match feed.ids.front() {
-                    None => req,
-                    Some(id) => req.param("max_id", id),
+                match feed.extend_past {
+                    None => return Ok(false),
+                    Some(ref params) => {
+                        let mut req = req;
+                        for (key, value) in params {
+                            req = req.param(key, value);
+                        }
+                        req
+                    }
                 }
             } else { req },
             FeedExtend::Future => if let Some(ref feed) = self.feeds.get(&id) {
-                match feed.ids.back() {
-                    None => req,
-                    Some(id) => req.param("min_id", id),
+                match feed.extend_future {
+                    None => return Ok(false),
+                    Some(ref params) => {
+                        let mut req = req;
+                        for (key, value) in params {
+                            req = req.param(key, value);
+                        }
+                        req
+                    }
                 }
             } else { req },
         };
@@ -661,6 +679,13 @@ impl Client {
             return Err(ClientError::UrlError(
                 url.clone(), rspstatus.to_string()));
         }
+
+        // Keep the Link: headers after we consume the response, for
+        // use later once we've constructed a Feed
+        let link_headers: Vec<_> = rsp.headers()
+            .get_all(reqwest::header::LINK)
+            .iter().cloned().collect();
+
         let body = rsp.text()?;
 
         // Decode the JSON response as a different kind of type
@@ -736,6 +761,8 @@ impl Client {
                 self.feeds.insert(id.clone(), Feed {
                     ids,
                     origin: 0,
+                    extend_past: None,
+                    extend_future: None,
                 });
             }
             FeedExtend::Future => {
@@ -753,6 +780,31 @@ impl Client {
             }
         }
 
+        let feed = self.feeds.get_mut(id).unwrap();
+        for linkhdr in link_headers {
+            let linkhdr_str = match linkhdr.to_str() {
+                Ok(s) => Ok(s),
+                Err(e) => Err(ClientError::UrlError(
+                    url.clone(), e.to_string())),
+            }?;
+            let links = match parse_link_header::parse(linkhdr_str) {
+                Ok(links) => Ok(links),
+                Err(e) => Err(ClientError::UrlError(
+                    url.clone(), e.to_string())),
+            }?;
+            for (rel, link) in links {
+                match rel.as_deref() {
+                    // Oh, you think time flows _that_ way?
+                    // Confusingly, the Mastodon protocol considers
+                    // "next" to be heading into the past and "prev"
+                    // the future.
+                    Some("next") => feed.extend_past = Some(link.queries),
+                    Some("prev") => feed.extend_future = Some(link.queries),
+                    _ => (),
+                }
+            }
+        }
+
         Ok(any_new)
     }