Merge pull request #39 from palantir/pk/error_handling

handle failed result post with error post

Author: PaulJKathmann

changelog/@unreleased/pr-39.v2.yml

@@ -0,0 +1,6 @@
+type: fix
+fix:
+  description: 'After this fix: If the job posting fails 5 times (e.g. result too
+    large) then the client tries 5 more times to post just a simple error message.'
+  links:
+  - https://github.com/palantir/python-compute-module/pull/39

compute_modules/client/internal_query_client.py

@@ -33,6 +33,7 @@
 from .encoder import CustomJSONEncoder
 
 POST_RESULT_MAX_ATTEMPTS = 5
+POST_ERROR_MAX_ATTEMPTS = 3
 POST_SCHEMAS_MAX_ATTEMPTS = 5
 
 
@@ -162,6 +163,28 @@ def get_job_or_none(self) -> Any:
             self.logger.error(traceback.format_exc())
             return None
 
+    def report_job_result_failed(self, post_result_url: str, error: str) -> None:
+        for _ in range(POST_ERROR_MAX_ATTEMPTS):
+            try:
+                with self.session.request(
+                    method="POST",
+                    url=post_result_url,
+                    headers=self.post_result_headers,
+                    data=json.dumps({"error": error}).encode("utf-8"),
+                    verify=self.certPath,
+                ) as response:
+                    if response.status_code == 204:
+                        self.logger.debug("Successfully reported that job result posting has failed")
+                        return
+                    else:
+                        self.logger.error(
+                            f"Failed to post result: {response.status_code} {response.reason} {response.text}"
+                        )
+            except Exception as e:
+                self.logger.error(f"Failed to report that post result has failed: {str(e)}")
+
+        raise RuntimeError(f"Unable to report that post result has failed after {POST_ERROR_MAX_ATTEMPTS} attempts")
+
     def report_job_result(self, job_id: str, body: Any) -> None:
         post_result_path = f"{self.post_result_path}/{job_id}"
         post_result_url = self.build_url(post_result_path)
@@ -179,17 +202,20 @@ def report_job_result(self, job_id: str, body: Any) -> None:
                         self.logger.debug("Successfully reported job result")
                         return
                     else:
-                        self.logger.error(
-                            f"Failed to post result: {response.status_code} {response.reason} {response.text}"
-                        )
+                        error = f"Failed to post result: {response.status_code} {response.reason} {response.text}"
+                        self.logger.error(error)
             except TypeError as e:
-                self.logger.error(f"Failed to serialize result to json: {str(e)}")
-                self.report_job_result(job_id, json.dumps(self.get_failed_query(e)).encode("utf-8"))
+                error = f"Failed to serialize result to json: {self.get_failed_query(e)}"
+                self.logger.error(error)
+                self.report_job_result_failed(post_result_url, error)
                 return
             except Exception as e:
-                self.logger.error(f"POST of job result failed, attempting to re-establish connection: {str(e)}")
-                self.logger.error(traceback.format_exc())
-        raise RuntimeError(f"Unable to post job result after {POST_RESULT_MAX_ATTEMPTS} attempts")
+                error = f"POST of job result failed, attempting to re-establish connection: {str(e)} \n {traceback.format_exc()}"
+                self.logger.error(error)
+
+        error = f"Unable to post job result after {POST_RESULT_MAX_ATTEMPTS} attempts; \n Now attempting to return the error as the result: {error}"
+        self.logger.error(error)
+        self.report_job_result_failed(post_result_url, error)
 
     def handle_job(self, job: Dict[str, Any]) -> None:
         self.logger.info("handling job")