add pagination to support limit > 100

src/main.rs

@@ -292,11 +292,9 @@ async fn main() -> Result<(), GertError> {
         posts.push(post);
     } else {
         for subreddit in &subreddits {
-            let listing = Subreddit::new(subreddit).get_feed(feed, limit, period).await?;
+            let subposts = Subreddit::new(subreddit).get_posts(feed, limit, period).await?;
             posts.extend(
-                listing
-                    .data
-                    .children
+                subposts
                     .into_iter()
                     .filter(|post| {
                         post.data.url.is_some() && !post.data.is_self && post.data.score > upvotes

src/subreddit.rs

@@ -1,7 +1,8 @@
 use crate::errors::GertError;
-use crate::structures::Listing;
+use crate::structures::{Listing, Post};
 use reqwest::Client;
 use std::fmt::Write;
+use log::debug;
 
 pub struct Subreddit {
     /// Name of subreddit.
@@ -18,42 +19,71 @@ impl Subreddit {
         Subreddit { name: name.to_owned(), url: subreddit_url, client: Client::new() }
     }
 
-    pub async fn get_feed(
+    async fn get_feed(
         &self,
         ty: &str,
         limit: u32,
-        options: Option<&str>,
+        period: Option<&str>,
+        after: Option<&str>
     ) -> Result<Listing, GertError> {
         let url = &mut format!("{}/{}.json?limit={}", self.url, ty, limit);
 
-        if let Some(period) = options {
-            let _ = write!(url, "&t={}", period);
+        if let Some(p) = period {
+            let _ = write!(url, "&t={}", p);
+        }
+
+        if let Some(a) = after {
+            let _ = write!(url, "&after={}", a);
         }
 
         Ok(self.client.get(&url.to_owned()).send().await?.json::<Listing>().await?)
     }
 
+    pub async fn get_posts(&self, feed:&str, limit: u32, period: Option<&str>) -> Result<Vec<Post>, GertError> {
+        if limit <= 100 {
+            return Ok(self.get_feed(feed, limit, period, None).await?.data
+            .children
+            .into_iter().collect())
+        }
+        let mut page = 1;
+        let mut posts: Vec<Post> = Vec::new();
+        let mut after = None;
+        let mut remaining = limit;
+        while remaining > 0 {
+            debug!("Fetching page {} of {} from r/{} [{}]", page, limit / 100, self.name, feed);
+            let limit = if remaining > 100 { 100 } else { remaining };
+            let listing = self.get_feed(feed, limit, period, after).await?;
+
+            posts.extend(listing.data.children.into_iter().collect::<Vec<Post>>());
+            let last_post = posts.last().unwrap();
+            after = Some(&last_post.data.name);
+            remaining -= limit;
+            page+=1;
+        }
+        Ok(posts)
+    }
+
     #[allow(dead_code)]
     /// Get hot posts.
     pub async fn hot(&self, limit: u32, options: Option<&str>) -> Result<Listing, GertError> {
-        self.get_feed("hot", limit, options).await
+        self.get_feed("hot", limit, options, None).await
     }
 
     #[allow(dead_code)]
     /// Get rising posts.
-    pub async fn rising(&self, limit: u32, options: Option<&str>) -> Result<Listing, GertError> {
-        self.get_feed("rising", limit, options).await
+    pub async fn rising(&self, limit: u32, period: Option<&str>) -> Result<Listing, GertError> {
+        self.get_feed("rising", limit, period, None).await
     }
 
     #[allow(dead_code)]
     /// Get top posts.
-    pub async fn top(&self, limit: u32, options: Option<&str>) -> Result<Listing, GertError> {
-        self.get_feed("top", limit, options).await
+    pub async fn top(&self, limit: u32, period: Option<&str>) -> Result<Listing, GertError> {
+        self.get_feed("top", limit, period, None).await
     }
 
     #[allow(dead_code)]
     /// Get latest posts.
-    pub async fn latest(&self, limit: u32, options: Option<&str>) -> Result<Listing, GertError> {
-        self.get_feed("new", limit, options).await
+    pub async fn latest(&self, limit: u32, period: Option<&str>) -> Result<Listing, GertError> {
+        self.get_feed("new", limit, period, None).await
     }
 }