Allow PaginatedList to use metadata for pagination (#614)

* Update `PaginatedList`

Based on #605, `PaginatedList` could not process requests which return
pagination info in the response body. This update checks for `Link` headers
before checking the `meta` property in the response body.

* Update based on review

* Update pag list test with no header no next meta to run properly. Update changelog.

---------

Co-authored-by: Matthew Emond <me@ucf.edu>

Author: bennettscience

CHANGELOG.md

@@ -2,6 +2,10 @@
 
 ## [Unreleased]
 
+### General
+
+- Added support for pagination with metadata when headers are missing (Thanks, [@bennettscience](https://github.com/bennettscience))
+
 ## [3.1.0] - 2023-04-21
 
 ### New Endpoint Coverage

canvasapi/paginated_list.py

@@ -57,8 +57,22 @@ def _get_next_page(self):
         )
         data = response.json()
         self._next_url = None
+        # Check the response headers first. This is the normal Canvas convention
+        # for pagination, but there are endpoints which return a `meta` property
+        # for pagination instead.
+        # See https://github.com/ucfopen/canvasapi/discussions/605
+        if response.links:
+            next_link = response.links.get("next")
+        elif isinstance(data, dict) and "meta" in data:
+            # requests parses headers into dicts, this returns the same
+            # structure so the regex will still work.
+            try:
+                next_link = {"url": data["meta"]["pagination"]["next"], "rel": "next"}
+            except KeyError:
+                next_link = None
+        else:
+            next_link = None
 
-        next_link = response.links.get("next")
         regex = r"{}(.*)".format(re.escape(self._requester.base_url))
 
         self._next_url = (
@@ -73,8 +87,9 @@ def _get_next_page(self):
             try:
                 data = data[self._root]
             except KeyError:
-                # TODO: Fix this message to make more sense to an end user.
-                raise ValueError("Invalid root value specified.")
+                raise ValueError(
+                    "The key <{}> does not exist in the response.".format(self._root)
+                )
 
         for element in data:
             if element is not None:

tests/fixtures/paginated_list.json

@@ -114,5 +114,66 @@
 			}
 		],
 		"status_code": 200
-	}
-}
+	},
+	"no_header_4_2_pages_p1": {
+		"method": "ANY",
+		"endpoint": "no_header_four_objects_two_pages",
+		"data": {
+			"assessments": [
+				{
+					"id": "1",
+					"name": "object 1"
+				},
+				{
+					"id": "2",
+					"name": "object 2"
+				}
+			],
+			"meta": {
+				"pagination": {
+					"next": "https://example.com/api/v1/no_header_four_objects_two_pages?page=2"
+				}
+			}
+		},
+		"status_code": 200
+	},
+	"no_header_4_2_pages_p2": {
+		"method": "ANY",
+		"endpoint": "no_header_four_objects_two_pages?page=2",
+		"data": {
+			"assessments": [
+				{
+					"id": "3",
+					"name": "object 3"
+				},
+				{
+					"id": "4",
+					"name": "object 4"
+				}
+			]
+		},
+		"status_code": 200
+	},
+	"no_header_no_next_key": {
+		"method": "ANY",
+		"endpoint": "no_header_no_next_key",
+		"data": {
+			"assessments": [
+				{
+					"id": "1",
+					"name": "object 1"
+				},
+				{
+					"id": "2",
+					"name": "object 2"
+				}
+			],
+			"meta": {
+				"pagination": {
+					"prev": "https://example.com/api/v1/previous"
+				}
+			}
+		}
+	},
+	"status_code": 200
+}

tests/test_paginated_list.py

@@ -156,6 +156,9 @@ def test_root_element_incorrect(self, m):
 
         with self.assertRaises(ValueError):
             pag_list[0]
+            self.assertEqual(
+                pag_list[0], "The key <wrong> does not exist in the response."
+            )
 
     def test_root_element(self, m):
         register_uris({"account": ["get_enrollment_terms"]}, m)
@@ -202,3 +205,31 @@ def test_negative_index_for_slice_end(self, m):
 
         with self.assertRaises(IndexError):
             pag_list[:-1]
+
+    def test_paginated_list_no_header(self, m):
+        register_uris(
+            {"paginated_list": ["no_header_4_2_pages_p1", "no_header_4_2_pages_p2"]}, m
+        )
+
+        pag_list = PaginatedList(
+            User,
+            self.requester,
+            "GET",
+            "no_header_four_objects_two_pages",
+            _root="assessments",
+        )
+
+        self.assertIsInstance(pag_list, PaginatedList)
+        self.assertEqual(len(list(pag_list)), 4)
+        self.assertIsInstance(pag_list[0], User)
+
+    def test_paginated_list_no_header_no_next(self, m):
+        register_uris({"paginated_list": ["no_header_no_next_key"]}, m)
+
+        pag_list = PaginatedList(
+            User, self.requester, "GET", "no_header_no_next_key", _root="assessments"
+        )
+
+        self.assertIsInstance(pag_list, PaginatedList)
+        self.assertEqual(len(list(pag_list)), 2)
+        self.assertIsInstance(pag_list[0], User)