updated batch run with compatible to unstract-client V0.2.0

README.md

@@ -60,6 +60,7 @@ This will display detailed usage information.
 - `--skip_unprocessed`: Skip unprocessed files when retrying failed files.
 - `--log_level`: Log level (default: `INFO`).
 - `--print_report`: Print a detailed report of all processed files at the end.
+- `--include_metadata`: Include metadata on tokens consumed and the context passed to LLMs for prompt studio exported tools in the result for each file.
 
 ## Usage Examples
 
@@ -129,3 +130,15 @@ The following statuses are tracked for each file during processing:
 - **ERROR**: File processing encountered an error.
 - **COMPLETED**: File was processed successfully and will not be processed again unless forced by rerun options.
 
+For more about statuses : [API Docs](https://docs.unstract.com/unstract/unstract_platform/api_deployment/unstract_api_deployment_execution_api/#possible-execution-status)
+
+
+## Questions and Feedback
+
+On Slack, [join great conversations](https://join-slack.unstract.com/) around LLMs, their ecosystem and leveraging them to automate the previously unautomatable!
+
+[Unstract client](https://github.com/Zipstack/unstract-python-client): Learn more about Unstract API clint.
+
+[Unstract Cloud](https://unstract.com/): Signup and Try!.
+
+[Unstract developer documentation](https://docs.unstract.com/): Learn more about Unstract and its API.

main.py

@@ -34,6 +34,7 @@ class Arguments:
     skip_unprocessed: bool = False
     log_level: str = "INFO"
     print_report: bool = False
+    include_metadata: bool = False
 
 
 # Initialize SQLite DB
@@ -244,12 +245,13 @@ def process_file(
             api_key=args.api_key,
             api_timeout=args.api_timeout,
             logging_level=args.log_level,
+            include_metadata=args.include_metadata,
         )
 
         status_endpoint, execution_status, response = get_status_endpoint(
             file_path=file_path, client=client, args=args
         )
-        # Polling until status is COMPLETE or ERROR
+        # Polling until status is COMPLETED or ERROR
         while execution_status not in ["COMPLETED", "ERROR"]:
             time.sleep(args.poll_interval)
             response = client.check_execution_status(status_endpoint)
@@ -409,6 +411,13 @@ def main():
         help="Print a detailed report of all file processed.",
     )
 
+    parser.add_argument(
+        "--include_metadata",
+        dest="include_metadata",
+        action="store_true",
+        help="Include metadata on tokens consumed and the context passed to LLMs for prompt studio exported tools in the result for each file.",
+    )
+
     args = Arguments(**vars(parser.parse_args()))
 
     ch = logging.StreamHandler(sys.stdout)

requirements.txt

@@ -1,3 +1,3 @@
-unstract-client~=0.1.0
+unstract-client~=1.0.0
 tqdm~=4.66.5
 tabulate~=0.9.0