How to debug and inspect raw API calls langchain makes

[bluskript]

NOTE: I only figured out a way for ChatOpenAI. Other LLMs probably have a similar structure, but read langchain's code to find what attribute needs to be overridden.

if you want to be able to see exactly what raw API requests langchain is making, use the following code below.

llm = ChatOpenAI(model="gpt-3.5-turbo-0613")

# class that wraps another class and logs all function calls being executed 
class Wrapper:
    def __init__(self, wrapped_class):
        self.wrapped_class = wrapped_class

    def __getattr__(self, attr):
        original_func = getattr(self.wrapped_class, attr)

        def wrapper(*args, **kwargs):
            print(f"Calling function: {attr}")
            print(f"Arguments: {args}, {kwargs}")
            result = original_func(*args, **kwargs)
            print(f"Response: {result}")
            return result

        return wrapper

# overwrite the private `client` attribute inside of the LLM that contains the API client with our wrapped class
llm.client = Wrapper(llm.client)

Ideally this should be a basic Langchain functionality, being able to see raw API responses are SO important sometimes. For example, I faced a KeyError: 'choices' error which was impossible to debug because langchain doesn't provide a way to view raw API responses.

[brandaobrandisborges]

Thanks for the contribution; This is really helpful;

[brendenpetersen]

@bluskript THANK YOU! You are a hero.

I have been at this for many hours. I've tried debug mode, callback functions, etc. Even LangChain traces do not provide all of this information. Traces include part of the raw API call in "invocation_parameters", including "tools" (and within that, "description" of the "parameters"), which is one of the main things I was trying to find. But the traces do not show the "messages" argument anywhere. It shows "prompts", but this is some LangChain-formatted construct.

P.S. This works on AzureChatOpenAI too.

[lucasalvarezlacasa]

Thank you for this contribution!

[sasharosca]

For ChatAnthropic, you need to do
llm._client.messages = Wrapper(llm._client.messages)

[NavpreetDevpuri]

Hi @bluskript, I tried the same method with with_structured_output(), but it did not work. Could you please help me with that?

from pydantic import BaseModel, Field
# Class that wraps another class and logs all function calls being executed

from langchain_openai import ChatOpenAI  

class Wrapper:
    def __init__(self, wrapped_class):
        self.wrapped_class = wrapped_class

    def __getattr__(self, attr):
        original_attr = getattr(self.wrapped_class, attr)

        if callable(original_attr):
            def wrapper(*args, **kwargs):
                print(f"Calling function: {attr}")
                print(f"Arguments: {args}, {kwargs}")
                result = original_attr(*args, **kwargs)
                print(f"Response: {result}")
                return result
            return wrapper
        else:
            return original_attr
        
model = ChatOpenAI(model="gpt-4o", temperature=0)


# Overwrite the private `client` attribute inside of the LLM that contains the API client with our wrapped class
model.client = Wrapper(model.client)
model.async_client = Wrapper(model.async_client)
model.root_client = Wrapper(model.root_client)
model.root_async_client = Wrapper(model.root_async_client)


class ResponseFormatter(BaseModel):
    """Always use this tool to structure your response to the user."""
    answer: str = Field(description="The answer to the user's question")
    followup_question: str = Field(description="A followup question the user could ask")
    temp: str = Field(description="It should always have '101' value")

# # Invoke the model with CustomHandler
# structured_output = model.invoke(
#     "What is the powerhouse of the cell?"
# )

# Bind the schema to the model
model_with_structure = model.with_structured_output(ResponseFormatter)
# model_with_structure.client = Wrapper(model_with_structure.client)

# Invoke the model with CustomHandler
structured_output = model_with_structure.invoke(
    "What is the powerhouse of the cell?"
)

# Get back the pydantic object
print(structured_output)

[NavpreetDevpuri]

I found a universal way using a proxy.

from flask import Flask, request, Response
import requests
import json
from pprint import pformat

app = Flask(__name__)

@app.route('/<path:path>', methods=['GET', 'POST', 'PUT', 'DELETE', 'OPTIONS', 'HEAD', 'PATCH'])
def proxy(path):
    # Log the params, body, or queries with pretty formatting
    app.logger.info("Params:\n" + pformat(request.args.to_dict()))
    body_data = request.get_json(silent=True)
    if body_data is not None:
        app.logger.info("Body:\n" + json.dumps(body_data, indent=4))
    else:
        app.logger.info("Body:\n" + request.get_data(as_text=True))

    url = f'https://api.openai.com/v1/{path}'
    headers = {key: value for key, value in request.headers if key.lower() != 'host'}
    response = requests.request(
        method=request.method,
        url=url,
        headers=headers,
        data=request.get_data(),
        params=request.args,
        cookies=request.cookies,
        allow_redirects=False,
        stream=True
    )

    excluded_headers = ['content-encoding', 'content-length', 'transfer-encoding', 'connection']
    response_headers = [(name, value) for (name, value) in response.raw.headers.items() if name.lower() not in excluded_headers]
    return Response(response.content, response.status_code, response_headers)

if __name__ == "__main__":
    app.run(debug=True, port=5000)

Then, we just need to use the base URL.

    model = ChatOpenAI(temperature=0, model_name="gpt-4o", base_url="http://127.0.0.1:5000")