BThddlZddlZddlZddlmZmZmZmZmZddl m Z ddl Z ddl Z ddl mZddlmZddlmZmZddlmZmZmZmZmZmZmZddlmZdd lmZdd l m!Z!m"Z"dd l#m$Z$m%Z%dd l&m'Z'm(Z(dd l)m'Z*ddl)m(Z+ddl,m-Z-ddl.m/Z/m0Z0ddl1m2Z2ddl3m4Z4ddl5m6Z6deddefdZ7GddeZ8Gdde'Z9e"e9de:de:fdZ;e"de:fdZGd#d$e4Z?y)%N)AnyListLiteralOptionalcast) MagicMock)warn_deprecated)BaseCallbackHandler) BaseChatModelGenericFakeChatModel) AIMessageAIMessageChunk BaseMessageBaseMessageChunk HumanMessage SystemMessage ToolMessage)StrOutputParser)ChatPromptTemplate)BaseTooltool)convert_to_json_schematool_example_to_messages) BaseModelFieldrr)BenchmarkFixture) Annotated TypedDict)Cassette)ChatModelTests)PYDANTIC_MAJOR_VERSION schema_typepydantic typeddict json_schemareturncGddtdtdtffd }Gddt}dtdtfd}|d k(r|fS|d k(r||fS|d k(rj |fSt d ) z :private: cJeZdZUdZedZeed<edZeed<y)_get_joke_class..JokeJoke to tell user.question to set up a joke descriptionsetupanswer to resolve the joke punchlineN) __name__ __module__ __qualname____doc__rr1str__annotations__r3/var/www/catia.catastroantioquia-mas.com/valormas/lib/python3.12/site-packages/langchain_tests/integration_tests/chat_models.pyJoker,0s% 'BCsC+GH 3Hr;r=resultr)ct|SN) isinstance)r>r=s r< validate_jokez&_get_joke_class..validate_joke6s&$''r;c>eZdZUdZeeddfed<eeddfed<y)!_get_joke_class..JokeDictr-.r.r1r2r3N)r4r5r6r7rr8r9r:r;r<JokeDictrD9s+ c#>>??S#'CCDDr;rEcBtd|jDS)Nc3$K|]}|dv yw))r1r3Nr:).0keys r< z>_get_joke_class..validate_joke_dict..@sJS300Js)allkeys)r>s r<validate_joke_dictz+_get_joke_class..validate_joke_dict?sJFKKMJJJr;r&r'r(zInvalid schema type)rrboolr model_json_schema ValueError)r$rBrErMr=s @r<_get_joke_classrQ)sIyI (c(d(E9E K3K4Kj ]""  #+++  %%%');;;.//r;c jeZdZUeeeed<d fd Zdddededeee efdeddf d Z xZ S) _TestCallbackHandleroptionsr)Nc0t|g|_yr@)super__init__rT)self __class__s r<rWz_TestCallbackHandler.__init__Qs  r;)rT serializedmessageskwargsc :|jj|yr@)rTappend)rXrZr[rTr\s r<on_chat_model_startz(_TestCallbackHandler.on_chat_model_startUs G$r;r)N) r4r5r6listrdictr9rWrr8r_ __classcell__)rYs@r<rSrSNs^ (4. !!-1 %%% $sCx.) %  % %r;rSc.eZdZUedddZeed<y)_MagicFunctionSchema.ii)gtltinputN)r4r5r6rrhintr9r:r;r<rere`ssu.E3.r;re) args_schemarhc |dzS)z%Applies a magic function to an input.r:)rhs r<magic_functionrmds 19r;cy)zCalculates a magic function.r:r:r;r<magic_function_no_argsrpjs r;messagect|tsJt|jdk(sJ|jd}|ddk(sJ|dddik(sJ|dJ|d d k(sJy) Nrnamermargsrhidtype tool_callrAr len tool_callsrqrys r<_validate_tool_call_messager~ps gy )) ) w!! "a '' '""1%I V  0 00 0 V ! ,, , T? && & V  ++ +r;ct|tsJt|jdk(sJ|jd}|ddk(sJ|dik(sJ|dJ|ddk(sJy) Nrsrrtrprurwrxryrzr}s r<#_validate_tool_call_message_no_argsrzs gy )) ) w!! "a '' '""1%I V  8 88 8 V  "" " T? && & V  ++ +r;c eZdZdZedefdZdeddfdZdeddfdZ deddfdZ deddfd Z deddfd Z deddfd Z deddfd Zdeddfd ZdeddfdZdeddfdZdeddfdZdeddfdZdeddfdZdeddfdZdeddfdZdeddfdZej2j5dgddededdfdZej2j5dgddededdfdZej2j=edk7ddeddfdZ deddfdZ!deddfdZ"ded e#ddfd!Z$ded e#ddfd"Z%ded e#ddfd#Z&deddfd$Z'deddfd%Z(deddfd&Z)deddfd'Z*deddfd(Z+ded e#ddfd)Z,deddfd*Z-deddfd+Z.ej2j^ej2j`ded,e1d-e2ddfd.Z3d/d0d1e4de5fd2Z6d/d0d1e4de5fd3Z7d/d0d1e4de5fd4Z8d/d0d1e4de5fd5Z9d/d0d1e4de5fd6Z:y)7ChatModelIntegrationTestsaDBase class for chat model integration tests. Test subclasses must implement the ``chat_model_class`` and ``chat_model_params`` properties to specify what model to test and its initialization parameters. Example: .. code-block:: python from typing import Type from langchain_tests.integration_tests import ChatModelIntegrationTests from my_package.chat_models import MyChatModel class TestMyChatModelIntegration(ChatModelIntegrationTests): @property def chat_model_class(self) -> Type[MyChatModel]: # Return the chat model class to test here return MyChatModel @property def chat_model_params(self) -> dict: # Return initialization parameters for the model. return {"model": "model-001", "temperature": 0} .. note:: API references for individual test methods include troubleshooting tips. Test subclasses must implement the following two properties: chat_model_class The chat model class to test, e.g., ``ChatParrotLink``. Example: .. code-block:: python @property def chat_model_class(self) -> Type[ChatParrotLink]: return ChatParrotLink chat_model_params Initialization parameters for the chat model. Example: .. code-block:: python @property def chat_model_params(self) -> dict: return {"model": "bird-brain-001", "temperature": 0} In addition, test subclasses can control what features are tested (such as tool calling or multi-modality) by selectively overriding the following properties. Expand to see details: .. dropdown:: has_tool_calling Boolean property indicating whether the chat model supports tool calling. By default, this is determined by whether the chat model's `bind_tools` method is overridden. It typically does not need to be overridden on the test class. Example override: .. code-block:: python @property def has_tool_calling(self) -> bool: return True .. dropdown:: tool_choice_value Value to use for tool choice when used in tests. .. warning:: Deprecated since version 0.3.15: This property will be removed in version 0.3.20. If a model supports ``tool_choice``, it should accept ``tool_choice="any"`` and ``tool_choice=``. If a model does not support forcing tool calling, override the ``has_tool_choice`` property to return ``False``. Example: .. code-block:: python @property def tool_choice_value(self) -> Optional[str]: return "any" .. dropdown:: has_tool_choice Boolean property indicating whether the chat model supports forcing tool calling via a ``tool_choice`` parameter. By default, this is determined by whether the parameter is included in the signature for the corresponding ``bind_tools`` method. If ``True``, the minimum requirement for this feature is that ``tool_choice="any"`` will force a tool call, and ``tool_choice=`` will force a call to a specific tool. Example override: .. code-block:: python @property def has_tool_choice(self) -> bool: return False .. dropdown:: has_structured_output Boolean property indicating whether the chat model supports structured output. By default, this is determined by whether the chat model's ``with_structured_output`` method is overridden. If the base implementation is intended to be used, this method should be overridden. See: https://python.langchain.com/docs/concepts/structured_outputs/ Example: .. code-block:: python @property def has_structured_output(self) -> bool: return True .. dropdown:: structured_output_kwargs Dict property that can be used to specify additional kwargs for ``with_structured_output``. Useful for testing different models. Example: .. code-block:: python @property def structured_output_kwargs(self) -> dict: return {"method": "function_calling"} .. dropdown:: supports_json_mode Boolean property indicating whether the chat model supports JSON mode in ``with_structured_output``. See: https://python.langchain.com/docs/concepts/structured_outputs/#json-mode Example: .. code-block:: python @property def supports_json_mode(self) -> bool: return True .. dropdown:: supports_image_inputs Boolean property indicating whether the chat model supports image inputs. Defaults to ``False``. If set to ``True``, the chat model will be tested using content blocks of the form .. code-block:: python { "type": "image", "source_type": "base64", "data": "", "mime_type": "image/jpeg", # or appropriate mime-type } In addition to OpenAI-style content blocks: .. code-block:: python { "type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{image_data}"}, } See https://python.langchain.com/docs/concepts/multimodality/ Example: .. code-block:: python @property def supports_image_inputs(self) -> bool: return True .. dropdown:: supports_image_urls Boolean property indicating whether the chat model supports image inputs from URLs. Defaults to ``False``. If set to ``True``, the chat model will be tested using content blocks of the form .. code-block:: python { "type": "image", "source_type": "url", "url": "https://...", } See https://python.langchain.com/docs/concepts/multimodality/ Example: .. code-block:: python @property def supports_image_urls(self) -> bool: return True .. dropdown:: supports_pdf_inputs Boolean property indicating whether the chat model supports PDF inputs. Defaults to ``False``. If set to ``True``, the chat model will be tested using content blocks of the form .. code-block:: python { "type": "file", "source_type": "base64", "data": "", "mime_type": "application/pdf", } See https://python.langchain.com/docs/concepts/multimodality/ Example: .. code-block:: python @property def supports_pdf_inputs(self) -> bool: return True .. dropdown:: supports_audio_inputs Boolean property indicating whether the chat model supports audio inputs. Defaults to ``False``. If set to ``True``, the chat model will be tested using content blocks of the form .. code-block:: python { "type": "audio", "source_type": "base64", "data": "", "mime_type": "audio/wav", # or appropriate mime-type } See https://python.langchain.com/docs/concepts/multimodality/ Example: .. code-block:: python @property def supports_audio_inputs(self) -> bool: return True .. dropdown:: supports_video_inputs Boolean property indicating whether the chat model supports image inputs. Defaults to ``False``. No current tests are written for this feature. .. dropdown:: returns_usage_metadata Boolean property indicating whether the chat model returns usage metadata on invoke and streaming responses. ``usage_metadata`` is an optional dict attribute on AIMessages that track input and output tokens: https://python.langchain.com/api_reference/core/messages/langchain_core.messages.ai.UsageMetadata.html Example: .. code-block:: python @property def returns_usage_metadata(self) -> bool: return False Models supporting ``usage_metadata`` should also return the name of the underlying model in the ``response_metadata`` of the AIMessage. .. dropdown:: supports_anthropic_inputs Boolean property indicating whether the chat model supports Anthropic-style inputs. These inputs might feature "tool use" and "tool result" content blocks, e.g., .. code-block:: python [ {"type": "text", "text": "Hmm let me think about that"}, { "type": "tool_use", "input": {"fav_color": "green"}, "id": "foo", "name": "color_picker", }, ] If set to ``True``, the chat model will be tested using content blocks of this form. Example: .. code-block:: python @property def supports_anthropic_inputs(self) -> bool: return False .. dropdown:: supports_image_tool_message Boolean property indicating whether the chat model supports ToolMessages that include image content, e.g., .. code-block:: python ToolMessage( content=[ { "type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{image_data}"}, }, ], tool_call_id="1", name="random_image", ) (OpenAI Chat Completions format), as well as .. code-block:: python ToolMessage( content=[ { "type": "image", "source_type": "base64", "data": image_data, "mime_type": "image/jpeg", }, ], tool_call_id="1", name="random_image", ) (standard format). If set to ``True``, the chat model will be tested with message sequences that include ToolMessages of this form. Example: .. code-block:: python @property def supports_image_tool_message(self) -> bool: return False .. dropdown:: supported_usage_metadata_details Property controlling what usage metadata details are emitted in both invoke and stream. ``usage_metadata`` is an optional dict attribute on AIMessages that track input and output tokens: https://python.langchain.com/api_reference/core/messages/langchain_core.messages.ai.UsageMetadata.html It includes optional keys ``input_token_details`` and ``output_token_details`` that can track usage details associated with special types of tokens, such as cached, audio, or reasoning. Only needs to be overridden if these details are supplied. .. dropdown:: enable_vcr_tests Property controlling whether to enable select tests that rely on `VCR `_ caching of HTTP calls, such as benchmarking tests. To enable these tests, follow these steps: 1. Override the ``enable_vcr_tests`` property to return ``True``: .. code-block:: python @property def enable_vcr_tests(self) -> bool: return True 2. Configure VCR to exclude sensitive headers and other information from cassettes. .. important:: VCR will by default record authentication headers and other sensitive information in cassettes. Read below for how to configure what information is recorded in cassettes. To add configuration to VCR, add a ``conftest.py`` file to the ``tests/`` directory and implement the ``vcr_config`` fixture there. ``langchain-tests`` excludes the headers ``"authorization"``, ``"x-api-key"``, and ``"api-key"`` from VCR cassettes. To pick up this configuration, you will need to add ``conftest.py`` as shown below. You can also exclude additional headers, override the default exclusions, or apply other customizations to the VCR configuration. See example below: .. code-block:: python :caption: tests/conftest.py import pytest from langchain_tests.conftest import _base_vcr_config as _base_vcr_config _EXTRA_HEADERS = [ # Specify additional headers to redact ("user-agent", "PLACEHOLDER"), ] def remove_response_headers(response: dict) -> dict: # If desired, remove or modify headers in the response. response["headers"] = {} return response @pytest.fixture(scope="session") def vcr_config(_base_vcr_config: dict) -> dict: # noqa: F811 """Extend the default configuration from langchain_tests.""" config = _base_vcr_config.copy() config.setdefault("filter_headers", []).extend(_EXTRA_HEADERS) config["before_record_response"] = remove_response_headers return config .. dropdown:: Compressing cassettes ``langchain-tests`` includes a custom VCR serializer that compresses cassettes using gzip. To use it, register the ``"yaml.gz"`` serializer to your VCR fixture and enable this serializer in the config. See example below: .. code-block:: python :caption: tests/conftest.py import pytest from langchain_tests.conftest import CustomPersister, CustomSerializer from langchain_tests.conftest import _base_vcr_config as _base_vcr_config from vcr import VCR _EXTRA_HEADERS = [ # Specify additional headers to redact ("user-agent", "PLACEHOLDER"), ] def remove_response_headers(response: dict) -> dict: # If desired, remove or modify headers in the response. response["headers"] = {} return response @pytest.fixture(scope="session") def vcr_config(_base_vcr_config: dict) -> dict: # noqa: F811 """Extend the default configuration from langchain_tests.""" config = _base_vcr_config.copy() config.setdefault("filter_headers", []).extend(_EXTRA_HEADERS) config["before_record_response"] = remove_response_headers # New: enable serializer and set file extension config["serializer"] = "yaml.gz" config["path_transformer"] = VCR.ensure_suffix(".yaml.gz") return config def pytest_recording_configure(config: dict, vcr: VCR) -> None: vcr.register_persister(CustomPersister()) vcr.register_serializer("yaml.gz", CustomSerializer()) You can inspect the contents of the compressed cassettes (e.g., to ensure no sensitive information is recorded) using .. code-block:: bash gunzip -k /path/to/tests/cassettes/TestClass_test.yaml.gz or by using the serializer: .. code-block:: python from langchain_tests.conftest import CustomPersister, CustomSerializer cassette_path = "/path/to/tests/cassettes/TestClass_test.yaml.gz" requests, responses = CustomPersister().load_cassette(path, CustomSerializer()) 3. Run tests to generate VCR cassettes. Example: .. code-block:: bash uv run python -m pytest tests/integration_tests/test_chat_models.py::TestMyModel::test_stream_time This will generate a VCR cassette for the test in ``tests/integration_tests/cassettes/``. .. important:: You should inspect the generated cassette to ensure that it does not contain sensitive information. If it does, you can modify the ``vcr_config`` fixture to exclude headers or modify the response before it is recorded. You can then commit the cassette to your repository. Subsequent test runs will use the cassette instead of making HTTP calls. r)ciSz :private:r:)rXs r<standard_chat_model_paramsz4ChatModelIntegrationTests.standard_chat_model_paramss  r;modelNc|jd}|Jt|tsJt|jtsJt |j dkDsJy)a\Test to verify that `model.invoke(simple_message)` works. This should pass for all integrations. .. dropdown:: Troubleshooting If this test fails, you should make sure your _generate method does not raise any exceptions, and that it returns a valid :class:`~langchain_core.outputs.chat_result.ChatResult` like so: .. code-block:: python return ChatResult( generations=[ChatGeneration( message=AIMessage(content="Output text") )] ) HelloNr)invokerAr textr8r{contentrXrr>s r< test_invokez%ChatModelIntegrationTests.test_invokes[&g&!!!&),,,&++----6>>"Q&&&r;cK|jdd{}|Jt|tsJt|jtsJt |j dkDsJy7Uw)aTest to verify that `await model.ainvoke(simple_message)` works. This should pass for all integrations. Passing this test does not indicate a "natively async" implementation, but rather that the model can be used in an async context. .. dropdown:: Troubleshooting First, debug :meth:`~langchain_tests.integration_tests.chat_models.ChatModelIntegrationTests.test_invoke`. because `ainvoke` has a default implementation that calls `invoke` in an async context. If that test passes but not this one, you should make sure your _agenerate method does not raise any exceptions, and that it returns a valid :class:`~langchain_core.outputs.chat_result.ChatResult` like so: .. code-block:: python return ChatResult( generations=[ChatGeneration( message=AIMessage(content="Output text") )] ) rNr)ainvokerAr rr8r{rrs r< test_ainvokez&ChatModelIntegrationTests.test_ainvokesg4}}W--!!!&),,,&++----6>>"Q&&& .sA/A-AA/cd}|jdD]0}|Jt|tsJ|t|jz }2|dkDsJy)aTest to verify that `model.stream(simple_message)` works. This should pass for all integrations. Passing this test does not indicate a "streaming" implementation, but rather that the model can be used in a streaming context. .. dropdown:: Troubleshooting First, debug :meth:`~langchain_tests.integration_tests.chat_models.ChatModelIntegrationTests.test_invoke`. because `stream` has a default implementation that calls `invoke` and yields the result as a single chunk. If that test passes but not this one, you should make sure your _stream method does not raise any exceptions, and that it yields valid :class:`~langchain_core.outputs.chat_generation.ChatGenerationChunk` objects like so: .. code-block:: python yield ChatGenerationChunk( message=AIMessageChunk(content="chunk text") ) rrN)streamrArr{rrXr num_tokenstokens r< test_streamz%ChatModelIntegrationTests.test_streams^2 \\'* -E$ $$e^4 44 #emm, ,J -A~~r;cKd}|jd23d{}|Jt|tsJ|t|jz }8736|dkDsJyw)aTest to verify that `await model.astream(simple_message)` works. This should pass for all integrations. Passing this test does not indicate a "natively async" or "streaming" implementation, but rather that the model can be used in an async streaming context. .. dropdown:: Troubleshooting First, debug :meth:`~langchain_tests.integration_tests.chat_models.ChatModelIntegrationTests.test_stream`. and :meth:`~langchain_tests.integration_tests.chat_models.ChatModelIntegrationTests.test_ainvoke`. because `astream` has a default implementation that calls `_stream` in an async context if it is implemented, or `ainvoke` and yields the result as a single chunk if not. If those tests pass but not this one, you should make sure your _astream method does not raise any exceptions, and that it yields valid :class:`~langchain_core.outputs.chat_generation.ChatGenerationChunk` objects like so: .. code-block:: python yield ChatGenerationChunk( message=AIMessageChunk(content="chunk text") ) rrN)astreamrArr{rrs r< test_astreamz&ChatModelIntegrationTests.test_astreamsk8  ==1 - -%$ $$e^4 44 #emm, ,J -1A~~s%AAAA0AA Ac$|jddg}|Jt|tsJt|dk(sJ|D]R}|Jt|tsJt|j t sJt|jdkDrRJy)aTest to verify that `model.batch([messages])` works. This should pass for all integrations. Tests the model's ability to process multiple prompts in a single batch. .. dropdown:: Troubleshooting First, debug :meth:`~langchain_tests.integration_tests.chat_models.ChatModelIntegrationTests.test_invoke` because `batch` has a default implementation that calls `invoke` for each message in the batch. If that test passes but not this one, you should make sure your `batch` method does not raise any exceptions, and that it returns a list of valid :class:`~langchain_core.messages.AIMessage` objects. rHeyNrlr)batchrArar{r rr8rrXr batch_resultsr>s r< test_batchz$ChatModelIntegrationTests.test_batchs" We$45 (((-...=!Q&&&# +F% %%fi0 00fkkmS1 11v~~&* **  +r;c@K|jddgd{}|Jt|tsJt|dk(sJ|D]R}|Jt|tsJt|j t sJt|jdkDrRJy7w)a^Test to verify that `await model.abatch([messages])` works. This should pass for all integrations. Tests the model's ability to process multiple prompts in a single batch asynchronously. .. dropdown:: Troubleshooting First, debug :meth:`~langchain_tests.integration_tests.chat_models.ChatModelIntegrationTests.test_batch` and :meth:`~langchain_tests.integration_tests.chat_models.ChatModelIntegrationTests.test_ainvoke` because `abatch` has a default implementation that calls `ainvoke` for each message in the batch. If those tests pass but not this one, you should make sure your `abatch` method does not raise any exceptions, and that it returns a list of valid :class:`~langchain_core.messages.AIMessage` objects. rrNrlr)abatchrArar{r rr8rrs r< test_abatchz%ChatModelIntegrationTests.test_abatch6s&$llGU+;<< (((-...=!Q&&&# +F% %%fi0 00fkkmS1 11v~~&* **  + =sBBA=BBctdtdtdg}|j|}|Jt|tsJt|j t sJt |jdkDsJy)aTest to verify that the model can handle multi-turn conversations. This should pass for all integrations. Tests the model's ability to process a sequence of alternating human and AI messages as context for generating the next response. .. dropdown:: Troubleshooting First, debug :meth:`~langchain_tests.integration_tests.chat_models.ChatModelIntegrationTests.test_invoke` because this test also uses `model.invoke()`. If that test passes but not this one, you should verify that: 1. Your model correctly processes the message history 2. The model maintains appropriate context from previous messages 3. The response is a valid :class:`~langchain_core.messages.AIMessage` hello how are youNr)rr rrArr8r{rrXrr[r>s r<test_conversationz+ChatModelIntegrationTests.test_conversationSs|&  ! g   '  h'!!!&),,,&++----6>>"Q&&&r;c Vtdtdtdtdtdtdtdg}|j|}|Jt |tsJt |j t sJt|jdkDsJy)a Test to verify that the model can handle double-message conversations. This should pass for all integrations. Tests the model's ability to process a sequence of double-system, double-human, and double-ai messages as context for generating the next response. .. dropdown:: Troubleshooting First, debug :meth:`~langchain_tests.integration_tests.chat_models.ChatModelIntegrationTests.test_invoke` because this test also uses `model.invoke()`. Second, debug :meth:`~langchain_tests.integration_tests.chat_models.ChatModelIntegrationTests.test_conversation` because this test is the "basic case" without double messages. If that test passes those but not this one, you should verify that: 1. Your model API can handle double messages, or the integration should merge messages before sending them to the API. 2. The response is a valid :class:`~langchain_core.messages.AIMessage` rrNr) rrr rrArr8r{rrs r<!test_double_messages_conversationz;ChatModelIntegrationTests.test_double_messages_conversationps0 ' " ' "  !  ! g  g   ' h'!!!&),,,&++----6>>"Q&&&r;c*|jstjd|jd}|Jt |t sJ|j Jt |j dtsJt |j dtsJt |j dtsJ|jjd}t |tsJ|sJd|jd vr|j}|j J|j d Jt |j d d tsJ|j dtd |j d jDk\sJd |jd vr|j}|j J|j dJt |j dd tsJt|j dtd|j djDk\sJd|jd vr|j!}|j J|j dJt |j ddtsJ|j dtd|j djDk\sJd|jd vr|j#}|j J|j d Jt |j d dtsJ|j dtd|j d jDk\sJd|jd vr|j%}|j J|j d Jt |j d dtsJ|j dtd|j d jDk\sJyy)a( Test to verify that the model returns correct usage metadata. This test is optional and should be skipped if the model does not return usage metadata (see Configuration below). .. versionchanged:: 0.3.17 Additionally check for the presence of `model_name` in the response metadata, which is needed for usage tracking in callback handlers. .. dropdown:: Configuration By default, this test is run. To disable this feature, set `returns_usage_metadata` to False in your test class: .. code-block:: python class TestMyChatModelIntegration(ChatModelIntegrationTests): @property def returns_usage_metadata(self) -> bool: return False This test can also check the format of specific kinds of usage metadata based on the `supported_usage_metadata_details` property. This property should be configured as follows with the types of tokens that the model supports tracking: .. code-block:: python class TestMyChatModelIntegration(ChatModelIntegrationTests): @property def supported_usage_metadata_details(self) -> dict: return { "invoke": [ "audio_input", "audio_output", "reasoning_output", "cache_read_input", "cache_creation_input", ], "stream": [ "audio_input", "audio_output", "reasoning_output", "cache_read_input", "cache_creation_input", ], } .. dropdown:: Troubleshooting If this test fails, first verify that your model returns :class:`~langchain_core.messages.ai.UsageMetadata` dicts attached to the returned AIMessage object in `_generate`: .. code-block:: python return ChatResult( generations=[ChatGeneration( message=AIMessage( content="Output text", usage_metadata={ "input_tokens": 350, "output_tokens": 240, "total_tokens": 590, "input_token_details": { "audio": 10, "cache_creation": 200, "cache_read": 100, }, "output_token_details": { "audio": 10, "reasoning": 200, } } ) )] ) Check also that the response includes a ``"model_name"`` key in its ``usage_metadata``. Not implemented.rN input_tokens output_tokens total_tokens model_name audio_inputrinput_token_detailsaudioc3(K|] }|xsd ywrNr:rHvs r<rJz@ChatModelIntegrationTests.test_usage_metadata..=a= audio_outputoutput_token_detailsc3(K|] }|xsd ywrr:rs r<rJz@ChatModelIntegrationTests.test_usage_metadata.. s CaCrreasoning_output reasoningc3(K|] }|xsd ywrr:rs r<rJz@ChatModelIntegrationTests.test_usage_metadata..s>a>rcache_read_input cache_readc3(K|] }|xsd ywrr:rs r<rJz@ChatModelIntegrationTests.test_usage_metadata..!rrcache_creation_inputcache_creationc3(K|] }|xsd ywrr:rs r<rJz@ChatModelIntegrationTests.test_usage_metadata..-rr)returns_usage_metadatapytestskiprrAr usage_metadatariresponse_metadatagetr8 supported_usage_metadata_detailsinvoke_with_audio_inputsumvaluesinvoke_with_audio_outputinvoke_with_reasoning_outputinvoke_with_cache_read_input invoke_with_cache_creation_input)rXrr>rmsgs r<test_usage_metadataz-ChatModelIntegrationTests.test_usage_metadatasj** KK* +g&!!!&),,,$$000&//?EEE&//@#FFF&//?EEE--11,? *c***z DAA(K K..0C%%1 11%%&;<H HHc001FGPRUV VV%%n5=++,ABIIK=:  TBB8L L//1C%%1 11%%&<=I IIc001GHQSVW WWs))/:;sC++,BCJJLC@  !F!Fx!P P335C%%1 11%%&<=I II""#9:;G %%o6#>++,BCJJL>;  !F!Fx!P P335C%%1 11%%&;<H HH""#89,G %%n5=++,ABIIK=:  "T%J%J8%T T779C%%1 11%%&;<H HH""#89:JK %%n5=++,ABIIK=:  Ur;cV|jstjdd}|jdD]j}t |t sJ|r=|j r1|j dr"|j r|j drJd||ntt ||z}lt |t sJ|j Jt |j dtsJt |j dtsJt |j dtsJ|jjd}t |tsJ|sJd |jd vr4|jd }t |j d dtsJd|jd vr4|jd }t |j ddtsJd|jd vr4|jd }t |j ddtsJd|jd vr4|j!d }t |j d dtsJd|jd vr5|j#d }t |j d dtsJyy)a Test to verify that the model returns correct usage metadata in streaming mode. .. versionchanged:: 0.3.17 Additionally check for the presence of `model_name` in the response metadata, which is needed for usage tracking in callback handlers. .. dropdown:: Configuration By default, this test is run. To disable this feature, set `returns_usage_metadata` to False in your test class: .. code-block:: python class TestMyChatModelIntegration(ChatModelIntegrationTests): @property def returns_usage_metadata(self) -> bool: return False This test can also check the format of specific kinds of usage metadata based on the `supported_usage_metadata_details` property. This property should be configured as follows with the types of tokens that the model supports tracking: .. code-block:: python class TestMyChatModelIntegration(ChatModelIntegrationTests): @property def supported_usage_metadata_details(self) -> dict: return { "invoke": [ "audio_input", "audio_output", "reasoning_output", "cache_read_input", "cache_creation_input", ], "stream": [ "audio_input", "audio_output", "reasoning_output", "cache_read_input", "cache_creation_input", ], } .. dropdown:: Troubleshooting If this test fails, first verify that your model yields :class:`~langchain_core.messages.ai.UsageMetadata` dicts attached to the returned AIMessage object in `_stream` that sum up to the total usage metadata. Note that `input_tokens` should only be included on one of the chunks (typically the first or the last chunk), and the rest should have 0 or None to avoid counting input tokens multiple times. `output_tokens` typically count the number of tokens in each chunk, not the sum. This test will pass as long as the sum of `output_tokens` across all chunks is not 0. .. code-block:: python yield ChatResult( generations=[ChatGeneration( message=AIMessage( content="Output text", usage_metadata={ "input_tokens": ( num_input_tokens if is_first_chunk else 0 ), "output_tokens": 11, "total_tokens": ( 11+num_input_tokens if is_first_chunk else 11 ), "input_token_details": { "audio": 10, "cache_creation": 200, "cache_read": 100, }, "output_token_details": { "audio": 10, "reasoning": 200, } } ) )] ) Check also that the aggregated response includes a ``"model_name"`` key in its ``usage_metadata``. rNz+Write me 2 haikus. Only include the haikus.rzDOnly one chunk should set input_tokens, the rest should be 0 or NonerrrrrTrrrrrrrrrrr)rrrrrArrrrirrr8rrrrrr)rXrfullchunkrrs r<test_usage_metadata_streamingz7ChatModelIntegrationTests.test_usage_metadata_streaming2s~** KK* +)-\\"OP QEe^4 44++0C0CN0S,,E4H4H4X4X !L5d>4%<.PD Q$///""...$--n=sCCC$--o>DDD$--n=sCCC++// = *c***z DAA(K K..d.;Cc001FGPRUV VV TBB8L L//t/ ChatResult: hiyou)stoprNr:)rrAr chat_model_classchat_model_params)rXrr> custom_models r<test_stop_sequencez,ChatModelIntegrationTests.test_stop_sequences.d%1&),,,,t,, (( $$T*&),,,r;c|jstjd|jsd}nd}t j |dt j t durtddd|jtg| }d }|j|}t|d}|j|D] }||n||z} t|tsJt|y) a8Test that the model generates tool calls. This test is skipped if the ``has_tool_calling`` property on the test class is set to False. This test is optional and should be skipped if the model does not support tool calling (see Configuration below). .. dropdown:: Configuration To disable tool calling tests, set ``has_tool_calling`` to False in your test class: .. code-block:: python class TestMyChatModelIntegration(ChatModelIntegrationTests): @property def has_tool_calling(self) -> bool: return False .. dropdown:: Troubleshooting If this test fails, check that ``bind_tools`` is implemented to correctly translate LangChain tool objects into the appropriate schema for your chat model. This test may fail if the chat model does not support a ``tool_choice`` parameter. This parameter can be used to force a tool call. If ``tool_choice`` is not supported and the model consistently fails this test, you can ``xfail`` the test: .. code-block:: python @pytest.mark.xfail(reason=("Does not support tool_choice.")) def test_tool_calling(self, model: BaseChatModel) -> None: super().test_tool_calling(model) Otherwise, in the case that only one tool is bound, ensure that ``tool_choice`` supports the string ``"any"`` to force calling that tool. Test requires tool calling.Nanytool_choice_valuez0.3.15a`tool_choice_value` will be removed in version 0.3.20. If a model supports `tool_choice`, it should accept `tool_choice='any' and `tool_choice=`. If the model does not support `tool_choice`, override the `supports_tool_choice` property to return `False`.z0.3.20)rqremoval tool_choice5What is the value of magic_function(3)? Use the tool.)has_tool_callingrrhas_tool_choiceinspectgetattr_staticrr bind_toolsrmrr~rrAr rXrrmodel_with_toolsqueryr>rrs r<test_tool_callingz+ChatModelIntegrationTests.test_tool_callingsN$$ KK5 6## $  %   ! ! % ''(ACVW X 2 ! !++  *;,  H!((/#F+,0%,,U3 ;E L5dUlD ;$ ***#D)r;cl|jr |jstjdtdt dt fd}dD]i}|j t|g|}|jd}t|tsJ|jsJ|dk(sS|jd d dk(riJy ) a.Test that the model can force tool calling via the ``tool_choice`` parameter. This test is skipped if the ``has_tool_choice`` property on the test class is set to False. This test is optional and should be skipped if the model does not support tool calling (see Configuration below). .. dropdown:: Configuration To disable tool calling tests, set ``has_tool_choice`` to False in your test class: .. code-block:: python class TestMyChatModelIntegration(ChatModelIntegrationTests): @property def has_tool_choice(self) -> bool: return False .. dropdown:: Troubleshooting If this test fails, check whether the ``test_tool_calling`` test is passing. If it is not, refer to the troubleshooting steps in that test first. If ``test_tool_calling`` is passing, check that the underlying model supports forced tool calling. If it does, ``bind_tools`` should accept a ``tool_choice`` parameter that can be used to force a tool call. It should accept (1) the string ``"any"`` to force calling the bound tool, and (2) the string name of the tool to force calling that tool. zTest requires tool choice.locationr)cy)zGet weather at a location. It's sunny.r:rs r< get_weatherz?ChatModelIntegrationTests.test_tool_choice..get_weatherZ!r;)rrmrzHello!rmrrtN) rrrrrr8rrmrrAr r|)rXrrrrr>s r<test_tool_choicez*ChatModelIntegrationTests.test_tool_choice6sB##4+@+@ KK4 5  !# !# !  !5 HK$//-; 0  &,,X6Ffi0 00$$ $$..((+F37GGGG Hr;c|K|jstjd|jsd}nd}|j t g|}d}|j |d{}t|d}|j|23d{}||n||z}7576t|tsJt|yw)aPTest that the model generates tool calls. This test is skipped if the ``has_tool_calling`` property on the test class is set to False. This test is optional and should be skipped if the model does not support tool calling (see Configuration below). .. dropdown:: Configuration To disable tool calling tests, set ``has_tool_calling`` to False in your test class: .. code-block:: python class TestMyChatModelIntegration(ChatModelIntegrationTests): @property def has_tool_calling(self) -> bool: return False .. dropdown:: Troubleshooting If this test fails, check that ``bind_tools`` is implemented to correctly translate LangChain tool objects into the appropriate schema for your chat model. This test may fail if the chat model does not support a ``tool_choice`` parameter. This parameter can be used to force a tool call. If ``tool_choice`` is not supported and the model consistently fails this test, you can ``xfail`` the test: .. code-block:: python @pytest.mark.xfail(reason=("Does not support tool_choice.")) async def test_tool_calling_async(self, model: BaseChatModel) -> None: await super().test_tool_calling_async(model) Otherwise, in the case that only one tool is bound, ensure that ``tool_choice`` supports the string ``"any"`` to force calling that tool. rNrrr) rrrrrrmrr~rrAr rs r<test_tool_calling_asyncz1ChatModelIntegrationTests.test_tool_calling_asyncisN$$ KK5 6## $  %  ++  *;,  H'//66#F+,0+33E: ; ;% L5dUlD 7  ;:$ ***#D)s6A!B<#B$"B<B B B B<BB<cP|jstjd|jsd}nd}|j t g|}d}|j |}t|d}|j|D] }||n||z} t|tsJt|y)aTest that the model generates tool calls for tools with no arguments. This test is skipped if the ``has_tool_calling`` property on the test class is set to False. This test is optional and should be skipped if the model does not support tool calling (see Configuration below). .. dropdown:: Configuration To disable tool calling tests, set ``has_tool_calling`` to False in your test class: .. code-block:: python class TestMyChatModelIntegration(ChatModelIntegrationTests): @property def has_tool_calling(self) -> bool: return False .. dropdown:: Troubleshooting If this test fails, check that ``bind_tools`` is implemented to correctly translate LangChain tool objects into the appropriate schema for your chat model. It should correctly handle the case where a tool has no arguments. This test may fail if the chat model does not support a ``tool_choice`` parameter. This parameter can be used to force a tool call. It may also fail if a provider does not support this form of tool. In these cases, you can ``xfail`` the test: .. code-block:: python @pytest.mark.xfail(reason=("Does not support tool_choice.")) def test_tool_calling_with_no_arguments(self, model: BaseChatModel) -> None: super().test_tool_calling_with_no_arguments(model) Otherwise, in the case that only one tool is bound, ensure that ``tool_choice`` supports the string ``"any"`` to force calling that tool. rNrrz bool: return False .. dropdown:: Troubleshooting If this test fails, check that ``bind_tools`` is implemented to correctly translate LangChain tool objects into the appropriate schema for your chat model. This test may fail if the chat model does not support a ``tool_choice`` parameter. This parameter can be used to force a tool call. If ``tool_choice`` is not supported and the model consistently fails this test, you can ``xfail`` the test: .. code-block:: python @pytest.mark.xfail(reason=("Does not support tool_choice.")) def test_bind_runnables_as_tools(self, model: BaseChatModel) -> None: super().test_bind_runnables_as_tools(model) Otherwise, ensure that the ``tool_choice_value`` property is correctly specified on the test class. r)humanz5Hello. Please respond in the style of {answer_style}.z hello matey)r[greeting_generatorz6Generate a greeting in a particular style of speaking.)rtr0rNrz+Using the tool, generate a Pirate greeting.rru answer_stylerxry)rrrr from_messagesr iterras_toolrrrrAr r|r) rXrpromptllmchaintool_rrrr>rys r<test_bind_runnables_as_toolsz6ChatModelIntegrationTests.test_bind_runnables_as_toolss P$$ KK5 6#11 O P #D-,AB 00 %P    ).KK ++UG+M=!((/&),,,    %%a(  $$^444 K///r;r$r%c|jstjdt|\}}|j|fi|j }t }t |_t}|jdd|gi}||t|jdk(sJdt|jdtsJt|jddd tsJ|jddd t|k(sJt} |jdd| giD] } ||   sJt| jdk(sJdt| jdtsJt| jddd tsJ| jddd t|k(sJy ) lTest to verify structured output is generated both on invoke and stream. This test is optional and should be skipped if the model does not support structured output (see Configuration below). .. dropdown:: Configuration To disable structured output tests, set ``has_structured_output`` to False in your test class: .. code-block:: python class TestMyChatModelIntegration(ChatModelIntegrationTests): @property def has_structured_output(self) -> bool: return False By default, ``has_structured_output`` is True if a model overrides the ``with_structured_output`` or ``bind_tools`` methods. .. dropdown:: Troubleshooting If this test fails, ensure that the model's ``bind_tools`` method properly handles both JSON Schema and Pydantic V2 models. ``langchain_core`` implements a utility function that will accommodate most formats: https://python.langchain.com/api_reference/core/utils/langchain_core.utils.function_calling.convert_to_openai_tool.html See example implementation of ``with_structured_output`` here: https://python.langchain.com/api_reference/_modules/langchain_openai/chat_models/base.html#BaseChatOpenAI.with_structured_output Test requires structured output.Tell me a joke about cats. callbacksconfigrs.Expected on_chat_model_start to be called oncerls_structured_output_formatschemaN)has_structured_outputrrrQwith_structured_outputstructured_output_kwargsrr_rSrr{rTrArbrr) rXrr$rvalidation_functionchat mock_callbackinvoke_callbackr>stream_callbackrs r<test_structured_outputz0ChatModelIntegrationTests.test_structured_output#s>)) KK: ;&5k&B##+u++FTd6S6ST! ,5K ).0 (+?P1Q  F#?**+q0 < 0/11!4d;;;  # #A &'D Eh OQU   &&q)*GH  #F +, ,,/0[[ (+?P1Q!  'E  & ' u?**+q0 < 0/11!4d;;;  # #A &'D Eh OQU   &&q)*GH  #F +, ,,r;cK|jstjdt|\}}|j|fi|j }t }|jdd|gid{}||t|jdk(sJdt|jdtsJt|jdd d tsJ|jdd d t|k(sJt }|jdd|gi23d{} || 776 sJt|jdk(sJdt|jdtsJt|jdd d tsJ|jdd d t|k(sJyw) r r rrrNrsrrrr)rrrrQrrrSrr{rTrArbrr) rXrr$rrrainvoke_callbackr>astream_callbackrs r<test_structured_output_asyncz6ChatModelIntegrationTests.test_structured_output_asyncos)B)) KK: ;&5k&B##+u++FTd6S6ST/1|| (+@P?Q1R$   F##++,1 < 1*2215t<<<  $ $Q '(E Fx PRV    ''*+HI  #F +, ,,01<< (+@P?Q1R(  ' '%  &+ $ '  u#++,1 < 1*2215t<<<  $ $Q '(E Fx PRV    ''*+HI  #F +, ,,s8A0G2D73B2G%D;)D9*D;- G9D;;B GrlzTest requires pydantic 2.)reasonc|jstjdGddt}|j|fi|j }|j d}t||sJ|jdD]}t||rJ|j|jfi|j }|j d}t|tsJt|jddhk(sJ|jdD]}t|trJttsJt|jddhk(sJy)aTest to verify we can generate structured output using pydantic.v1.BaseModel. pydantic.v1.BaseModel is available in the pydantic 2 package. This test is optional and should be skipped if the model does not support structured output (see Configuration below). .. dropdown:: Configuration To disable structured output tests, set ``has_structured_output`` to False in your test class: .. code-block:: python class TestMyChatModelIntegration(ChatModelIntegrationTests): @property def has_structured_output(self) -> bool: return False By default, ``has_structured_output`` is True if a model overrides the ``with_structured_output`` or ``bind_tools`` methods. .. dropdown:: Troubleshooting If this test fails, ensure that the model's ``bind_tools`` method properly handles both JSON Schema and Pydantic V1 models. ``langchain_core`` implements a utility function that will accommodate most formats: https://python.langchain.com/api_reference/core/utils/langchain_core.utils.function_calling.convert_to_openai_tool.html See example implementation of ``with_structured_output`` here: https://python.langchain.com/api_reference/_modules/langchain_openai/chat_models/base.html#BaseChatOpenAI.with_structured_output r cJeZdZUdZedZeed<edZeed<y)LChatModelIntegrationTests.test_structured_output_pydantic_2_v1..Joker-r.r/r1r2r3N) r4r5r6r7FieldV1r1r8r9r3r:r;r<r=r%s% $ -HIE3 I$1MNIs Nr;r=rr1r3N) rrr BaseModelV1rrrrArrrbsetrL)rXrr=rr>rs r<$test_structured_output_pydantic_2_v1z>ChatModelIntegrationTests.test_structured_output_pydantic_2_v1sQD)) KK: ; O; O,u++DRD4Q4QR9:&$'''[[!=> +EeT* ** +,u++ KKM !:: 9:&$'''6;;=!g{%;;;;[[!=> +EeT* ** +%&&&5::< Wk$::::r;cT|jstjdGddt}|j|fi|j }|j d}t||sJ|j d}t||sJ|j|jfi|j }|j d}t|tsJGddt}|j|fi|j }|j d}t|tsJy ) aTest to verify we can generate structured output that includes optional parameters. This test is optional and should be skipped if the model does not support structured output (see Configuration below). .. dropdown:: Configuration To disable structured output tests, set ``has_structured_output`` to False in your test class: .. code-block:: python class TestMyChatModelIntegration(ChatModelIntegrationTests): @property def has_structured_output(self) -> bool: return False By default, ``has_structured_output`` is True if a model overrides the ``with_structured_output`` or ``bind_tools`` methods. .. dropdown:: Troubleshooting If this test fails, ensure that the model's ``bind_tools`` method properly handles Pydantic V2 models with optional parameters. ``langchain_core`` implements a utility function that will accommodate most formats: https://python.langchain.com/api_reference/core/utils/langchain_core.utils.function_calling.convert_to_openai_tool.html See example implementation of ``with_structured_output`` here: https://python.langchain.com/api_reference/_modules/langchain_openai/chat_models/base.html#BaseChatOpenAI.with_structured_output r cReZdZUdZedZeed<eddZe eed<y) MChatModelIntegrationTests.test_structured_output_optional_param..Joker-r.r/r1Nr2)defaultr0r3) r4r5r6r7rr1r8r9r3rr:r;r<r=r,s0 $+FGE3 G',*F(Ix} r;r=z5Give me the setup to a joke about cats, no punchline.z1Give me a joke about cats, include the punchline.rcDeZdZUdZeeddfed<eeeddfed<y)QChatModelIntegrationTests.test_structured_output_optional_param..JokeDictr-.r.r1Nr2r3)r4r5r6r7rr8r9rr:r;r<rEr/8s/ $S#'BBC C #6R!RS Sr;rEN) rrrrrrrrArOrbr )rXrr=r setup_result joke_resultr>rEs r<%test_structured_output_optional_paramz?ChatModelIntegrationTests.test_structured_output_optional_params">)) KK: ; 9 ,u++DRD4Q4QR{{ C ,---kk"UV +t,,,,u++  " " $ (,(E(E 9:&$''' Ty T ,u++HV8U8UV9:&$'''r;c|jstjdddlm}ddlmGfdd|}|j |d}d }|j|}t||sJ|j|D]}t||rJ|j |jd}|j|}t|tsJt|jd d hk(sJ|j|D]}t|trJttsJt|jd d hk(sJy ) awTest structured output via `JSON mode. `_ This test is optional and should be skipped if the model does not support the JSON mode feature (see Configuration below). .. dropdown:: Configuration To disable this test, set ``supports_json_mode`` to False in your test class: .. code-block:: python class TestMyChatModelIntegration(ChatModelIntegrationTests): @property def supports_json_mode(self) -> bool: return False .. dropdown:: Troubleshooting See example implementation of ``with_structured_output`` here: https://python.langchain.com/api_reference/_modules/langchain_openai/chat_models/base.html#BaseChatOpenAI.with_structured_output z Test requires json mode support.rrrcPeZdZUdZWdZeed<WdZeed<y)6ChatModelIntegrationTests.test_json_mode..Joker-r.r/r1r2r3N)r4r5r6r7r1r8r9r3) FieldPropersr<r=r5^s& $$1LME3 M(5QRIs Rr;r= json_mode)methodzyTell me a joke about cats. Return the result as a JSON with 'setup' and 'punchline' keys. Return nothing other than JSON.r1r3N)supports_json_moderrr&rrrrrArrOrbr(rL) rXrBaseModelProperr=rrr>rr6s @r<test_json_modez(ChatModelIntegrationTests.test_json_modeBsR,&& KK: ;91 S? S++D+E @ S!&$'''[[% +EeT* ** +++  " " $[, S!&$'''6;;=!g{%;;;;[[% +EeT* ** +%&&&5::< Wk$::::r; my_adder_toolc >|jstjd|j|g}d}ddd}t dt d||dd d g t tjd d i|dg}|j|}t|t sJy)a6Test that message histories are compatible with string tool contents (e.g. OpenAI format). If a model passes this test, it should be compatible with messages generated from providers following OpenAI format. This test should be skipped if the model does not support tool calling (see Configuration below). .. dropdown:: Configuration To disable tool calling tests, set ``has_tool_calling`` to False in your test class: .. code-block:: python class TestMyChatModelIntegration(ChatModelIntegrationTests): @property def has_tool_calling(self) -> bool: return False .. dropdown:: Troubleshooting If this test fails, check that: 1. The model can correctly handle message histories that include AIMessage objects with ``""`` content. 2. The ``tool_calls`` attribute on AIMessage objects is correctly handled and passed to the model in an appropriate format. 3. The model can correctly handle ToolMessage objects with string content and arbitrary string values for ``tool_call_id``. You can ``xfail`` the test if tool calling is implemented but this format is not supported. .. code-block:: python @pytest.mark.xfail(reason=("Not implemented.")) def test_tool_message_histories_string_content(self, *args: Any) -> None: super().test_tool_message_histories_string_content(*args) rr<12ab What is 1 + 2abc123ryrtrurwrxr|r>rvrt tool_call_idN rrrrrr rjsondumpsrrA)rXrr<r function_name function_argsmessages_string_contentresult_string_contents r<*test_tool_message_histories_string_contentzDChatModelIntegrationTests.test_tool_message_histories_string_contentsN$$ KK5 6 ++]O<' !,   ) !. -& +    Ha=)"% # (!1 7 78O P/;;;r;c R|jstjd|j|g}d}ddd}t dt ddd d d ||d g||d d dgt tjddi|d g}|j|}t|t sJy)a{Test that message histories are compatible with list tool contents (e.g. Anthropic format). These message histories will include AIMessage objects with "tool use" and content blocks, e.g., .. code-block:: python [ {"type": "text", "text": "Hmm let me think about that"}, { "type": "tool_use", "input": {"fav_color": "green"}, "id": "foo", "name": "color_picker", }, ] This test should be skipped if the model does not support tool calling (see Configuration below). .. dropdown:: Configuration To disable tool calling tests, set ``has_tool_calling`` to False in your test class: .. code-block:: python class TestMyChatModelIntegration(ChatModelIntegrationTests): @property def has_tool_calling(self) -> bool: return False .. dropdown:: Troubleshooting If this test fails, check that: 1. The model can correctly handle message histories that include AIMessage objects with list content. 2. The ``tool_calls`` attribute on AIMessage objects is correctly handled and passed to the model in an appropriate format. 3. The model can correctly handle ToolMessage objects with string content and arbitrary string values for ``tool_call_id``. You can ``xfail`` the test if tool calling is implemented but this format is not supported. .. code-block:: python @pytest.mark.xfail(reason=("Not implemented.")) def test_tool_message_histories_list_content(self, *args: Any) -> None: super().test_tool_message_histories_list_content(*args) rr<rsrlr@rCrz some textrxrtool_userE)rxrwrtrhryrFrGr>rvrHNrJ)rXrr<rrMrNmessages_list_contentresult_list_contents r<(test_tool_message_histories_list_contentzBChatModelIntegrationTests.test_tool_message_histories_list_contentsn$$ KK5 6 ++]O<' a(   ) #[9 *& -!. !. -& +  &  Ha=)"% -! 8/556KL-y999r;c|jstjd|j|gd}t j ddi}|j }t|trt|tsJtd|dd g|g| }|td gz}|j|}t|tsJy ) aTest that the model can process few-shot examples with tool calls. These are represented as a sequence of messages of the following form: - ``HumanMessage`` with string content; - ``AIMessage`` with the ``tool_calls`` attribute populated; - ``ToolMessage`` with string content; - ``AIMessage`` with string content (an answer); - ``HuamnMessage`` with string content (a follow-up question). This test should be skipped if the model does not support tool calling (see Configuration below). .. dropdown:: Configuration To disable tool calling tests, set ``has_tool_calling`` to False in your test class: .. code-block:: python class TestMyChatModelIntegration(ChatModelIntegrationTests): @property def has_tool_calling(self) -> bool: return False .. dropdown:: Troubleshooting This test uses a utility function in ``langchain_core`` to generate a sequence of messages representing "few-shot" examples: https://python.langchain.com/api_reference/core/utils/langchain_core.utils.function_calling.tool_example_to_messages.html If this test fails, check that the model can correctly handle this sequence of messages. You can ``xfail`` the test if tool calling is implemented but this format is not supported. .. code-block:: python @pytest.mark.xfail(reason=("Not implemented.")) def test_structured_few_shot_examples(self, *args: Any) -> None: super().test_structured_few_shot_examples(*args) rrrr>rvrCrsrlr@) tool_outputs ai_responsez What is 3 + 4N)rrrrrKrLrjrArx issubclassrrrrr ) rXrr<rfunction_result tool_schemafew_shot_messagesr[r>s r<!test_structured_few_shot_examplesz;ChatModelIntegrationTests.test_structured_few_shot_examples sZ$$ KK5 6 ++]O+O**h]3#// +t,K1SSS4  1 " #)*'  % _(E'FF!((2&),,,r;cp|jstjdd}tjt j |jjd}tdddddd |d g}|j|g}tddddd d |d dg}|j|g}y)atTest that the model can process PDF inputs. This test should be skipped (see Configuration below) if the model does not support PDF inputs. These will take the form: .. code-block:: python { "type": "image", "source_type": "base64", "data": "", "mime_type": "application/pdf", } See https://python.langchain.com/docs/concepts/multimodality/ .. dropdown:: Configuration To disable this test, set ``supports_pdf_inputs`` to False in your test class: .. code-block:: python class TestMyChatModelIntegration(ChatModelIntegrationTests): @property def supports_pdf_inputs(self) -> bool: return False .. dropdown:: Troubleshooting If this test fails, check that the model can correctly handle messages with pdf content blocks, including base64-encoded files. Otherwise, set the ``supports_pdf_inputs`` property to False. z"Model does not support PDF inputs.zGhttps://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdfutf-8rzSummarize this document:rSfilebase64zapplication/pdfrx source_type mime_typedataz test file.pdfzdata:application/pdf;base64,)filename file_data)rxrbN) supports_pdf_inputsrrrc b64encodehttpxrrdecoderr)rXrurlpdf_datarq_s r<test_pdf_inputsz)ChatModelIntegrationTests.test_pdf_inputs_sH'' KK< =W##EIIcN$:$:;BB7K#6 ##+!2$     LL' ##6 #$3'CH:%N   LL' #r;cj|jstjdd}tjt j |jjd}tdddddd |d g}|j|g}tdddd |d d dg}|j|g}y)aTest that the model can process audio inputs. This test should be skipped (see Configuration below) if the model does not support audio inputs. These will take the form: .. code-block:: python { "type": "audio", "source_type": "base64", "data": "", "mime_type": "audio/wav", # or appropriate mime-type } See https://python.langchain.com/docs/concepts/multimodality/ .. dropdown:: Configuration To disable this test, set ``supports_audio_inputs`` to False in your test class: .. code-block:: python class TestMyChatModelIntegration(ChatModelIntegrationTests): @property def supports_audio_inputs(self) -> bool: return False .. dropdown:: Troubleshooting If this test fails, check that the model can correctly handle messages with audio content blocks, specifically base64-encoded files. Otherwise, set the ``supports_audio_inputs`` property to False. z$Model does not support audio inputs.zhttps://upload.wikimedia.org/wikipedia/commons/3/3d/Alcal%C3%A1_de_Henares_%28RPS_13-04-2024%29_canto_de_ruise%C3%B1or_%28Luscinia_megarhynchos%29_en_el_Soto_del_Henares.wavrarzDescribe this audio:rSrrcz audio/wavrd input_audiowav)rgformat)rxrsN) supports_audio_inputsrrrcrkrlrrrmrr)rXrrn audio_datarqrps r<test_audio_inputsz+ChatModelIntegrationTests.test_audio_inputssH)) KK> ?~%%eiin&<&<=DDWM #2 $#+!,&     LL' ##2 *,6%#H    LL' #r;c|jstjdd}tjt j |jjd}tdddddd |id g }|j|g}tdddd d d|dg }|j|g}|jr(tdddd d|dg }|j|g}yy)a$Test that the model can process image inputs. This test should be skipped (see Configuration below) if the model does not support image inputs. These will take the form: .. code-block:: python { "type": "image", "source_type": "base64", "data": "", "mime_type": "image/jpeg", # or appropriate mime-type } For backward-compatibility, we must also support OpenAI-style image content blocks: .. code-block:: python [ {"type": "text", "text": "describe the weather in this image"}, { "type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{image_data}"}, }, ] See https://python.langchain.com/docs/concepts/multimodality/ If the property ``supports_image_urls`` is set to True, the test will also check that we can process content blocks of the form: .. code-block:: python { "type": "image", "source_type": "url", "url": "", } .. dropdown:: Configuration To disable this test, set ``supports_image_inputs`` to False in your test class: .. code-block:: python class TestMyChatModelIntegration(ChatModelIntegrationTests): @property def supports_image_inputs(self) -> bool: return False # Can also explicitly disable testing image URLs: @property def supports_image_urls(self) -> bool: return False .. dropdown:: Troubleshooting If this test fails, check that the model can correctly handle messages with image content blocks, including base64-encoded images. Otherwise, set the ``supports_image_inputs`` property to False. z%Model does not support image message.https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpgrarz"describe the weather in this imagerS image_urlrndata:image/jpeg;base64,rxr{)rimagerc image/jpegrd)rxrernN) supports_image_inputsrrrcrkrlrrrmrrsupports_image_urls)rXrr{ image_datarqrps r<test_image_inputsz+ChatModelIntegrationTests.test_image_inputss@)) KK? @u %%eii &:&B&BCJJ7S )MN'"'+B:,)O!P  LL' #)MN##+!-&    LL' #  # #"#-QR '',( G gY'A $r;c |jstjdd}tjt j |jjd}tddd|idgdd }td d |d dgdd }||fD]L}tdtgddd idg|g}dtfd}|j|gj|} Ny)aTest that the model can process ToolMessages with image inputs. This test should be skipped if the model does not support messages of the form: .. code-block:: python ToolMessage( content=[ { "type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{image_data}"}, }, ], tool_call_id="1", name="random_image", ) containing image content blocks in OpenAI Chat Completions format, in addition to messages of the form: .. code-block:: python ToolMessage( content=[ { "type": "image", "source_type": "base64", "data": image_data, "mime_type": "image/jpeg", }, ], tool_call_id="1", name="random_image", ) containing image content blocks in standard format. This test can be skipped by setting the ``supports_image_tool_message`` property to False (see Configuration below). .. dropdown:: Configuration To disable this test, set ``supports_image_tool_message`` to False in your test class: .. code-block:: python class TestMyChatModelIntegration(ChatModelIntegrationTests): @property def supports_image_tool_message(self) -> bool: return False .. dropdown:: Troubleshooting If this test fails, check that the model can correctly handle messages with image content blocks in ToolMessages, including base64-encoded images. Otherwise, set the ``supports_image_tool_message`` property to False. z*Model does not support image tool message.rzrar{rnr|r}r> random_image)rrIrtr~rcr)rxrergrfz:get a random image using the tool and describe the weatherry)rxrwrtrurGr)cy)zReturn a random image.rDr:r:r;r<rzGChatModelIntegrationTests.test_image_tool_message..random_image sr;N)supports_image_tool_messagerrrcrkrlrrrmrrr r8rr) rXrr{roai_format_messagestandard_format_message tool_messager[rrps r<test_image_tool_messagez1ChatModelIntegrationTests.test_image_tool_message_ s.z// KKD Eu %%eii &:&B&BCJJ7S )("'+B:,)O!P   #.$#+&!-  # 01HI BLP%0"%$2$&   H$ #   ,077AA/ Br;c |jstjdGddt}dddg}|jr[d}t j tj|jjd}|jd d d |d d tdt|tddddddidddgdddidddgtddg}|j!|gj#|}t%|tsJtdddgtdddddddgtdd dgg}|j#|}t%|tsJy!)"a Test that model can process Anthropic-style message histories. These message histories will include ``AIMessage`` objects with ``tool_use`` content blocks, e.g., .. code-block:: python AIMessage( [ {"type": "text", "text": "Hmm let me think about that"}, { "type": "tool_use", "input": {"fav_color": "green"}, "id": "foo", "name": "color_picker", }, ] ) as well as ``HumanMessage`` objects containing ``tool_result`` content blocks: .. code-block:: python HumanMessage( [ { "type": "tool_result", "tool_use_id": "foo", "content": [ { "type": "text", "text": "green is a great pick! that's my sister's favorite color", # noqa: E501 } ], "is_error": False, }, {"type": "text", "text": "what's my sister's favorite color"}, ] ) This test should be skipped if the model does not support messages of this form (or doesn't support tool calling generally). See Configuration below. .. dropdown:: Configuration To disable this test, set ``supports_anthropic_inputs`` to False in your test class: .. code-block:: python class TestMyChatModelIntegration(ChatModelIntegrationTests): @property def supports_anthropic_inputs(self) -> bool: return False .. dropdown:: Troubleshooting If this test fails, check that: 1. The model can correctly handle message histories that include message objects with list content. 2. The ``tool_calls`` attribute on AIMessage objects is correctly handled and passed to the model in an appropriate format. 3. HumanMessages with "tool_result" content blocks are correctly handled. Otherwise, if Anthropic tool call and result formats are not supported, set the ``supports_anthropic_inputs`` property to False. z3Model does not explicitly support Anthropic inputs.ceZdZUdZeed<y)EChatModelIntegrationTests.test_anthropic_inputs..color_pickerz4Input your fav color and get a random fact about it. fav_colorN)r4r5r6r7r8r9r:r;r< color_pickerr s FNr;rrz(what's your favorite color in this imagerSrzrar~rcr)rx media_typerg)rxsourcezyou're a good assistantzHmm let me think about thatrTrgreenfoo)rxrhrwrtryrFrGzThat's a great pick!)rIrthinkingzI'm thinking...rE)rxr signaturezHello, how are you?z Well, thanks.N)supports_anthropic_inputsrrr'rrcrkrlrrrmr^rrr rrrrA)rXrr human_contentr{rr[responses r<test_anthropic_inputsz/ChatModelIntegrationTests.test_anthropic_inputs sF-- KKM N ; B %  % %yI))%))I*>*F*FGNNwWJ  # (&2 *  3 4  ' #-JK *"-w!7# . !/!,g 6# +  & .U C- 0##\N3::8D(I... !' '  !+$5%- !' 5   !' / - ><<)(I...r;c  |jstjd|j|g}t dt ddddiddd g t d ddd g}|j|}t|t sJy)aTest that ToolMessage with ``status="error"`` can be handled. These messages may take the form: .. code-block:: python ToolMessage( "Error: Missing required argument 'b'.", name="my_adder_tool", tool_call_id="abc123", status="error", ) If possible, the ``status`` field should be parsed and passed appropriately to the model. This test is optional and should be skipped if the model does not support tool calling (see Configuration below). .. dropdown:: Configuration To disable tool calling tests, set ``has_tool_calling`` to False in your test class: .. code-block:: python class TestMyChatModelIntegration(ChatModelIntegrationTests): @property def has_tool_calling(self) -> bool: return False .. dropdown:: Troubleshooting If this test fails, check that the ``status`` field on ``ToolMessage`` objects is either ignored or passed to the model appropriately. rrCrDr<rArsrEryrFrGz%Error: Missing required argument 'b'.error)rtrIstatusN) rrrrrr rrrA)rXrr<rr[r>s r<test_tool_message_error_statusz8ChatModelIntegrationTests.test_tool_message_error_statuso sN$$ KK5 6 ++]O<  ) !0!$a& +   7$%   ("((2&),,,r;c|jtddg}|Jt|tsJt|j t sJt |jdkDsJy)a%Test that HumanMessage with values for the ``name`` field can be handled. These messages may take the form: .. code-block:: python HumanMessage("hello", name="example_user") If possible, the ``name`` field should be parsed and passed appropriately to the model. Otherwise, it should be ignored. .. dropdown:: Troubleshooting If this test fails, check that the ``name`` field on ``HumanMessage`` objects is either ignored or passed to the model appropriately. r example_user)rtNr)rrrAr rr8r{rrs r<test_message_with_namez0ChatModelIntegrationTests.test_message_with_name se"|G.IJK!!!&),,,&++----6>>"Q&&&r;c|jstjdtdtdtfd}|j |g}t d}|j|g}t|tsJ|j}t|dk(sJ|d}|j|}t|tsJ|j|||g} t| tsJy) aTest that the model supports a simple ReAct agent loop. This test is skipped if the ``has_tool_calling`` property on the test class is set to False. This test is optional and should be skipped if the model does not support tool calling (see Configuration below). .. dropdown:: Configuration To disable tool calling tests, set ``has_tool_calling`` to False in your test class: .. code-block:: python class TestMyChatModelIntegration(ChatModelIntegrationTests): @property def has_tool_calling(self) -> bool: return False .. dropdown:: Troubleshooting If this test fails, check that ``bind_tools`` is implemented to correctly translate LangChain tool objects into the appropriate schema for your chat model. Check also that all required information (e.g., tool calling identifiers) from AIMessage objects is propagated correctly to model payloads. This test may fail if the chat model does not consistently generate tool calls in response to an appropriate query. In these cases you can ``xfail`` the test: .. code-block:: python @pytest.mark.xfail(reason=("Does not support tool_choice.")) def test_agent_loop(self, model: BaseChatModel) -> None: super().test_agent_loop(model) rrr)cy)zCall to surf the web.rr:rs r<rz>ChatModelIntegrationTests.test_agent_loop..get_weather rr;z)What is the weather in San Francisco, CA?rsrN) rrrrr8rrrrAr r|r{r) rXrrllm_with_tools input_messagetool_call_messager|ryrrs r<test_agent_loopz)ChatModelIntegrationTests.test_agent_loop sN$$ KK5 6  !# !# !  !));-8$%PQ *11=/B+Y777&11 :!###qM ")))4 , 444!((!  (I...r; benchmarkvcrc|jstjddfd }|js|y||y)a`Test that streaming does not introduce undue overhead. See ``enable_vcr_tests`` dropdown :class:`above ` for more information. .. dropdown:: Configuration This test can be enabled or disabled using the ``enable_vcr_tests`` property. For example, to disable the test, set this property to ``False``: .. code-block:: python @property def enable_vcr_tests(self) -> bool: return False .. important:: VCR will by default record authentication headers and other sensitive information in cassettes. See ``enable_vcr_tests`` dropdown :class:`above ` for how to configure what information is recorded in cassettes. zVCR not set up.Nc2jdD]}y)NzWrite a story about a cat.r)rprs r<_runz8ChatModelIntegrationTests.test_stream_time.._run( s\\">?  r;r`)enable_vcr_testsrr responses)rXrrrrs ` r<test_stream_timez*ChatModelIntegrationTests.test_stream_time s5:$$ KK) * }} F dOr;FrrctrNotImplementedErrorrXrs r<rz1ChatModelIntegrationTests.invoke_with_audio_input1 !##r;ctrrrs r<rz2ChatModelIntegrationTests.invoke_with_audio_output5 rr;ctrrrs r<rz6ChatModelIntegrationTests.invoke_with_reasoning_output9 rr;ctrrrs r<rz6ChatModelIntegrationTests.invoke_with_cache_read_input= rr;ctrrrs r<rz:ChatModelIntegrationTests.invoke_with_cache_creation_inputA rr;);r4r5r6r7propertyrbrr rrrrrrrrrrrrrrrr rmark parametrizer8rr!skipifr#r)r2r;rrQrWr_rqrxrrrrrrrrrr!rrNr rrrrrr:r;r<rrsJSjD''4'2' '$'@4@! !$!F+ +$+6+}++:'}'':$'}$'$'LZZ4ZxS=STSj!- !-$!-FJ*}J*J*X1Hm1H1Hf;*=;*T;*z:2:24:2x?0-?0D?0B [[],TUI,MI,I,PTI,VI,V [[],TUJ,"J,14J, J,VJ,X [[.!3$$)$:?$$$9$>C$d$y$>C$d$y$BG$$$9$r;r)@rcrrKtypingrrrrr unittest.mockrrlrlangchain_core._apir langchain_core.callbacksr langchain_core.language_modelsr r langchain_core.messagesr rrrrrrlangchain_core.output_parsersrlangchain_core.promptsrlangchain_core.toolsrr%langchain_core.utils.function_callingrrr&rr pydantic.v1r'r&pytest_benchmark.fixturertyping_extensionsrr vcr.cassetter!&langchain_tests.unit_tests.chat_modelsr"langchain_tests.utils.pydanticr#rQrSrerirmrpr~rrr:r;r<rs  55# /8N:5/&0(52!B"0?@"0"0J%.%$/9/&'##(    ,,,,,,*$*$r;