diff --git a/newrelic/config.py b/newrelic/config.py index babaa9779b..e6913e5e48 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -3208,6 +3208,19 @@ def _process_module_builtin_defaults(): "newrelic.hooks.mlmodel_autogen", "instrument_autogen_agentchat_agents__assistant_agent", ) + _process_module_definition( + "crewai.tools.tool_usage", "newrelic.hooks.mlmodel_crewai", "instrument_crewai_tools_tool_usage" + ) + _process_module_definition( + "crewai.agents.crew_agent_executor", + "newrelic.hooks.mlmodel_crewai", + "instrument_crewai_agents_crew_agent_executor", + ) + _process_module_definition( + "crewai.events.types.tool_usage_events", + "newrelic.hooks.mlmodel_crewai", + "instrument_crewai_events_types_tool_usage_events", + ) _process_module_definition( "google.adk.agents.llm_agent", "newrelic.hooks.mlmodel_googleadk", "instrument_googleadk_agents_llm_agent" ) diff --git a/newrelic/hooks/mlmodel_crewai.py b/newrelic/hooks/mlmodel_crewai.py new file mode 100644 index 0000000000..5c92da9bb5 --- /dev/null +++ b/newrelic/hooks/mlmodel_crewai.py @@ -0,0 +1,296 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import logging +import sys +import uuid + +from newrelic.api.function_trace import FunctionTrace +from newrelic.api.time_trace import get_trace_linking_metadata +from newrelic.api.transaction import current_transaction +from newrelic.common.llm_utils import _get_llm_metadata +from newrelic.common.object_names import callable_name +from newrelic.common.object_wrapper import wrap_function_wrapper +from newrelic.common.package_version_utils import get_package_version +from newrelic.common.signature import bind_args +from newrelic.core.config import global_settings + +CREWAI_VERSION = get_package_version("crewai") + +RECORD_EVENTS_FAILURE_LOG_MESSAGE = "Exception occurred in CrewAI instrumentation: Failed to record LLM events. Please report this issue to New Relic Support.\n%s" +TOOL_EXTRACTOR_FAILURE_LOG_MESSAGE = "Exception occurred in CrewAI instrumentation: Failed to extract tool information. If the issue persists, report this issue to New Relic Support.\n" + +_logger = logging.getLogger(__name__) + + +def _get_tool_name(tool, calling): + # The tool name lives on both the resolved tool and the calling object + return getattr(tool, "name", None) or getattr(calling, "tool_name", None) or "tool" + + +def _construct_base_tool_event_dict(instance, tool, calling, tool_id, transaction, settings, linking_metadata): + try: + _input = getattr(calling, "arguments", None) + tool_input = str(_input) if _input else None + tool_name = _get_tool_name(tool, calling) + agent_name = getattr(getattr(instance, "agent", None), "role", "agent") + + tool_event_dict = { + "id": tool_id, + "name": tool_name, + "span_id": linking_metadata.get("span.id"), + "trace_id": linking_metadata.get("trace.id"), + "agent_name": agent_name, + "vendor": "crewai", + "ingest_source": "Python", + } + if settings.ai_monitoring.record_content.enabled: + tool_event_dict["input"] = tool_input + tool_event_dict.update(_get_llm_metadata(transaction)) + except Exception: + tool_event_dict = {} + _logger.warning(RECORD_EVENTS_FAILURE_LOG_MESSAGE, exc_info=True) + + return tool_event_dict + + +def _start_tool_trace(wrapped, instance, tool, calling, transaction): + transaction.add_ml_model_info("CrewAI", CREWAI_VERSION) + transaction._add_agent_attribute("llm", True) + + tool_name = _get_tool_name(tool, calling) + func_name = callable_name(wrapped) + agentic_subcomponent_data = {"type": "APM-AI_TOOL", "name": tool_name} + + ft = FunctionTrace(name=f"{func_name}/{tool_name}", group="Llm/tool/CrewAI") + ft.__enter__() + ft._add_agent_attribute("subcomponent", json.dumps(agentic_subcomponent_data)) + return ft + + +def wrap_tool_usage__use(wrapped, instance, args, kwargs): + transaction = current_transaction() + if not transaction: + return wrapped(*args, **kwargs) + + settings = transaction.settings or global_settings() + if not settings.ai_monitoring.enabled: + return wrapped(*args, **kwargs) + + try: + bound_args = bind_args(wrapped, args, kwargs) + tool = bound_args.get("tool") + calling = bound_args.get("calling") + except Exception: + tool = calling = None + _logger.warning(TOOL_EXTRACTOR_FAILURE_LOG_MESSAGE, exc_info=True) + + tool_id = str(uuid.uuid4()) + ft = _start_tool_trace(wrapped, instance, tool, calling, transaction) + linking_metadata = get_trace_linking_metadata() + tool_event_dict = _construct_base_tool_event_dict( + instance, tool, calling, tool_id, transaction, settings, linking_metadata + ) + + try: + return_val = wrapped(*args, **kwargs) + except Exception: + ft.notice_error(attributes={"tool_id": tool_id}) + ft.__exit__(*sys.exc_info()) + tool_event_dict.update({"duration": ft.duration * 1000, "error": True}) + transaction.record_custom_event("LlmTool", tool_event_dict) + raise + + ft.__exit__(None, None, None) + _record_tool_success(transaction, settings, tool_event_dict, ft, return_val) + return return_val + + +async def wrap_tool_usage__ause(wrapped, instance, args, kwargs): + transaction = current_transaction() + if not transaction: + return await wrapped(*args, **kwargs) + + settings = transaction.settings or global_settings() + if not settings.ai_monitoring.enabled: + return await wrapped(*args, **kwargs) + + try: + bound_args = bind_args(wrapped, args, kwargs) + tool = bound_args.get("tool") + calling = bound_args.get("calling") + except Exception: + tool = calling = None + _logger.warning(TOOL_EXTRACTOR_FAILURE_LOG_MESSAGE, exc_info=True) + + tool_id = str(uuid.uuid4()) + ft = _start_tool_trace(wrapped, instance, tool, calling, transaction) + linking_metadata = get_trace_linking_metadata() + tool_event_dict = _construct_base_tool_event_dict( + instance, tool, calling, tool_id, transaction, settings, linking_metadata + ) + + try: + return_val = await wrapped(*args, **kwargs) + except Exception: + ft.notice_error(attributes={"tool_id": tool_id}) + ft.__exit__(*sys.exc_info()) + tool_event_dict.update({"duration": ft.duration * 1000, "error": True}) + transaction.record_custom_event("LlmTool", tool_event_dict) + raise + + ft.__exit__(None, None, None) + _record_tool_success(transaction, settings, tool_event_dict, ft, return_val) + return return_val + + +def _record_tool_success(transaction, settings, tool_event_dict, ft, return_val): + try: + tool_event_dict.update({"duration": ft.duration * 1000}) + # _use/_ause return the formatted result string (tool output) + if settings.ai_monitoring.record_content.enabled: + tool_event_dict["output"] = str(return_val) if return_val else None + transaction.record_custom_event("LlmTool", tool_event_dict) + except Exception: + _logger.warning(RECORD_EVENTS_FAILURE_LOG_MESSAGE, exc_info=True) + + +def wrap_tool_usage_event_init(wrapped, instance, args, kwargs): + wrapped(*args, **kwargs) + + transaction = current_transaction() + if not transaction: + return + + captured_events = getattr(transaction, "_nr_crewai_native_tool_events", None) + if captured_events is None: + return + + if type(instance).__name__ in ("ToolUsageFinishedEvent", "ToolUsageErrorEvent"): + captured_events.append(instance) + + +def _construct_native_tool_event_dict(event, tool_id, transaction, settings, linking_metadata): + try: + tool_name = (getattr(event, "tool_name", None) if event else None) or "tool" + tool_input = getattr(event, "tool_args", None) if event else None + tool_input = str(tool_input) if tool_input else None + agent_name = (getattr(event, "agent_role", None) if event else None) or "agent" + + tool_event_dict = { + "id": tool_id, + "name": tool_name, + "span_id": linking_metadata.get("span.id"), + "trace_id": linking_metadata.get("trace.id"), + "agent_name": agent_name, + "vendor": "crewai", + "ingest_source": "Python", + } + if settings.ai_monitoring.record_content.enabled: + tool_event_dict["input"] = tool_input + tool_event_dict.update(_get_llm_metadata(transaction)) + except Exception: + tool_event_dict = {} + _logger.warning(RECORD_EVENTS_FAILURE_LOG_MESSAGE, exc_info=True) + + return tool_event_dict + + +def wrap_crew_agent_executor__handle_native_tool_calls(wrapped, instance, args, kwargs): + # Covers the native function-calling tool path, which is the default for OpenAI/Anthropic/ + # Gemini/Azure/Bedrock models in current CrewAI versions and bypasses ToolUsage entirely + transaction = current_transaction() + if not transaction: + return wrapped(*args, **kwargs) + + settings = transaction.settings or global_settings() + if not settings.ai_monitoring.enabled: + return wrapped(*args, **kwargs) + + transaction.add_ml_model_info("CrewAI", CREWAI_VERSION) + transaction._add_agent_attribute("llm", True) + + tool_id = str(uuid.uuid4()) + func_name = callable_name(wrapped) + linking_metadata = get_trace_linking_metadata() + + ft = FunctionTrace(name=func_name, group="Llm/tool/CrewAI") + ft.__enter__() + + # Save/restore rather than blindly clearing, in case of reentrant native tool calls + # within the same transaction (e.g. an agent delegating to another agent). + previous_events = getattr(transaction, "_nr_crewai_native_tool_events", None) + transaction._nr_crewai_native_tool_events = [] + try: + return_val = wrapped(*args, **kwargs) + except Exception: + ft.notice_error(attributes={"tool_id": tool_id}) + ft.__exit__(*sys.exc_info()) + if previous_events is None: + del transaction._nr_crewai_native_tool_events + else: + transaction._nr_crewai_native_tool_events = previous_events + raise + + captured_events = transaction._nr_crewai_native_tool_events + if previous_events is None: + del transaction._nr_crewai_native_tool_events + else: + transaction._nr_crewai_native_tool_events = previous_events + + # _handle_native_tool_calls emits ToolUsageErrorEvent and ToolUsageFinishedEvent + error_event = next((e for e in captured_events if type(e).__name__ == "ToolUsageErrorEvent"), None) + finished_event = error_event or next( + (e for e in captured_events if type(e).__name__ == "ToolUsageFinishedEvent"), None + ) + + tool_name = (getattr(finished_event, "tool_name", None) if finished_event is not None else None) or "tool" + ft.name = f"{func_name}/{tool_name}" + agentic_subcomponent_data = {"type": "APM-AI_TOOL", "name": tool_name} + ft._add_agent_attribute("subcomponent", json.dumps(agentic_subcomponent_data)) + ft.__exit__(None, None, None) + + tool_event_dict = _construct_native_tool_event_dict( + finished_event, tool_id, transaction, settings, linking_metadata + ) + tool_event_dict["duration"] = ft.duration * 1000 + if error_event is not None: + tool_event_dict["error"] = True + elif settings.ai_monitoring.record_content.enabled and finished_event is not None: + output = getattr(finished_event, "output", None) + tool_event_dict["output"] = str(output) if output is not None else None + + transaction.record_custom_event("LlmTool", tool_event_dict) + return return_val + + +def instrument_crewai_events_types_tool_usage_events(module): + if hasattr(module, "ToolUsageEvent"): + wrap_function_wrapper(module, "ToolUsageEvent.__init__", wrap_tool_usage_event_init) + + +def instrument_crewai_agents_crew_agent_executor(module): + if hasattr(module, "CrewAgentExecutor") and hasattr(module.CrewAgentExecutor, "_handle_native_tool_calls"): + wrap_function_wrapper( + module, "CrewAgentExecutor._handle_native_tool_calls", wrap_crew_agent_executor__handle_native_tool_calls + ) + + +def instrument_crewai_tools_tool_usage(module): + if hasattr(module, "ToolUsage"): + if hasattr(module.ToolUsage, "_use"): + wrap_function_wrapper(module, "ToolUsage._use", wrap_tool_usage__use) + if hasattr(module.ToolUsage, "_ause"): + wrap_function_wrapper(module, "ToolUsage._ause", wrap_tool_usage__ause) diff --git a/tests/mlmodel_crewai/_test_tools.py b/tests/mlmodel_crewai/_test_tools.py new file mode 100644 index 0000000000..3a3393e0cf --- /dev/null +++ b/tests/mlmodel_crewai/_test_tools.py @@ -0,0 +1,72 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import functools + +from conftest import AGENT_NAME +from crewai.tools import tool + +TOOL_NAME = "get_capital" + + +def _get_capital(country: str) -> str: + """Return a country's capital.""" + capitals = {"France": "Paris", "Japan": "Tokyo"} + return capitals.get(country, "Unknown") + + +@functools.wraps(_get_capital) # Make tool name and description match +def _raising_capital(country: str) -> str: + raise ValueError("intentional tool failure") + + +get_capital = tool(TOOL_NAME)(_get_capital) +raising_capital = tool(TOOL_NAME)(_raising_capital) + +EXPECTED_TOOL_INPUT_STR = "{'country': 'France'}" +EXPECTED_TOOL_OUTPUT_STR = "Paris" + + +def tool_recorded_event(record_content: bool, output: str = EXPECTED_TOOL_OUTPUT_STR): + base = { + "id": None, + "name": TOOL_NAME, + "span_id": None, + "trace_id": "trace-id", + "agent_name": AGENT_NAME, + "vendor": "crewai", + "ingest_source": "Python", + "duration": None, + } + if record_content: + base["input"] = EXPECTED_TOOL_INPUT_STR + base["output"] = output + return [({"type": "LlmTool"}, base)] + + +def tool_recorded_event_error(record_content: bool): + base = { + "id": None, + "name": TOOL_NAME, + "span_id": None, + "trace_id": "trace-id", + "agent_name": AGENT_NAME, + "vendor": "crewai", + "ingest_source": "Python", + "duration": None, + "error": True, + } + if record_content: + base["input"] = EXPECTED_TOOL_INPUT_STR + return [({"type": "LlmTool"}, base)] diff --git a/tests/mlmodel_crewai/cassette.yaml b/tests/mlmodel_crewai/cassette.yaml new file mode 100644 index 0000000000..d40d4141bd --- /dev/null +++ b/tests/mlmodel_crewai/cassette.yaml @@ -0,0 +1,423 @@ +interactions: +- request: + body: '{"messages":[{"role":"system","content":"You are my_agent. A concise assistant.\nYour + personal goal is: Answer in one word.\nYou ONLY have access to the following + tools, and should NEVER make up tools that are not listed here:\n\nTool Name: + get_capital\nTool Arguments: {\n \"properties\": {\n \"country\": {\n \"title\": + \"Country\",\n \"type\": \"string\"\n }\n },\n \"required\": [\n \"country\"\n ],\n \"title\": + \"Get_Capital\",\n \"type\": \"object\",\n \"additionalProperties\": false\n}\nTool + Description: Return a country''s capital.\n\nIMPORTANT: Use the following format + in your response:\n\n```\nThought: you should always think about what to do\nAction: + the action to take, only one name of [get_capital], just the name, exactly as + it''s written.\nAction Input: the input to the action, just a simple JSON object, + enclosed in curly braces, using \" to wrap keys and values.\nObservation: the + result of the action\n```\n\nOnce all necessary information is gathered, return + the following format:\n\n```\nThought: I now know the final answer\nFinal Answer: + the final answer to the original input question\n```"},{"role":"user","content":"\nCurrent + Task: Use get_capital on France. Answer with only the city name.\n\nThis is + the expected criteria for your final answer: One word.\nyou MUST return the + actual complete content as the final answer, not a summary.\n\nBegin! This is + VERY important to you, use the tools available and give your best Final Answer, + your job depends on it!\n\nThought:"}],"model":"gpt-4o-mini","seed":42,"temperature":0.0}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + authorization: + - XXXXXX + connection: + - keep-alive + content-type: + - application/json + host: + - api.openai.com + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: '{"id": "chatcmpl-EEfelAcpD2NNUqCpgGDFqBJNnzI90", "object": "chat.completion", + "created": 1787165427, "model": "gpt-4o-mini-2024-07-18", "choices": [{"index": + 0, "message": {"role": "assistant", "content": "Thought: I need to find the + capital of France.\nAction: get_capital\nAction Input: {\"country\":\"France\"}\nObservation: + The result of the action is \"Paris\".\n```\nThought: I now know the final + answer\nFinal Answer: Paris\n```", "refusal": null, "annotations": []}, "logprobs": + null, "finish_reason": "stop"}], "usage": {"prompt_tokens": 330, "completion_tokens": + 53, "total_tokens": 383, "prompt_tokens_details": {"cached_tokens": 0, "audio_tokens": + 0}, "completion_tokens_details": {"reasoning_tokens": 0, "audio_tokens": 0, + "accepted_prediction_tokens": 0, "rejected_prediction_tokens": 0}}, "service_tier": + "default", "system_fingerprint": "fp_c73dd82b09"}' + headers: + access-control-expose-headers: + - X-Request-ID + - CF-Ray + - CF-Ray + cf-cache-status: + - DYNAMIC + cf-ray: + - a2db580f8969e17a-SEA + connection: + - keep-alive + content-type: + - application/json + date: + - Wed, 19 Aug 2026 18:50:28 GMT + openai-organization: + - nr-test-org + openai-processing-ms: + - '948' + openai-project: + - nr-test-project + openai-version: + - '2020-10-01' + server: + - cloudflare + transfer-encoding: + - chunked + x-ratelimit-limit-requests: + - '10000' + x-ratelimit-limit-tokens: + - '50000000' + x-ratelimit-remaining-requests: + - '9999' + x-ratelimit-remaining-tokens: + - '49999975' + x-ratelimit-reset-requests: + - 6ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_84b5e4407c9d45edb07c78dc1552b509 + status: + code: 200 + message: OK +- request: + body: '{"messages":[{"role":"system","content":"You are my_agent. A concise assistant.\nYour + personal goal is: Answer in one word.\nYou ONLY have access to the following + tools, and should NEVER make up tools that are not listed here:\n\nTool Name: + get_capital\nTool Arguments: {\n \"properties\": {\n \"country\": {\n \"title\": + \"Country\",\n \"type\": \"string\"\n }\n },\n \"required\": [\n \"country\"\n ],\n \"title\": + \"Get_Capital\",\n \"type\": \"object\",\n \"additionalProperties\": false\n}\nTool + Description: Return a country''s capital.\n\nIMPORTANT: Use the following format + in your response:\n\n```\nThought: you should always think about what to do\nAction: + the action to take, only one name of [get_capital], just the name, exactly as + it''s written.\nAction Input: the input to the action, just a simple JSON object, + enclosed in curly braces, using \" to wrap keys and values.\nObservation: the + result of the action\n```\n\nOnce all necessary information is gathered, return + the following format:\n\n```\nThought: I now know the final answer\nFinal Answer: + the final answer to the original input question\n```"},{"role":"user","content":"\nCurrent + Task: Use get_capital on France. Answer with only the city name.\n\nThis is + the expected criteria for your final answer: One word.\nyou MUST return the + actual complete content as the final answer, not a summary.\n\nBegin! This is + VERY important to you, use the tools available and give your best Final Answer, + your job depends on it!\n\nThought:"},{"role":"assistant","content":"Thought: + I need to find the capital of France.\nAction: get_capital\nAction Input: {\"country\":\"France\"}\nObservation: + Paris"}],"model":"gpt-4o-mini","seed":42,"temperature":0.0}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + authorization: + - XXXXXX + connection: + - keep-alive + content-type: + - application/json + host: + - api.openai.com + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: '{"id": "chatcmpl-EEfemdJBVmfa5wVij7I8OJDSMKZzJ", "object": "chat.completion", + "created": 1787165428, "model": "gpt-4o-mini-2024-07-18", "choices": [{"index": + 0, "message": {"role": "assistant", "content": "Thought: I now know the final + answer\nFinal Answer: Paris", "refusal": null, "annotations": []}, "logprobs": + null, "finish_reason": "stop"}], "usage": {"prompt_tokens": 362, "completion_tokens": + 13, "total_tokens": 375, "prompt_tokens_details": {"cached_tokens": 0, "audio_tokens": + 0}, "completion_tokens_details": {"reasoning_tokens": 0, "audio_tokens": 0, + "accepted_prediction_tokens": 0, "rejected_prediction_tokens": 0}}, "service_tier": + "default", "system_fingerprint": "fp_c73dd82b09"}' + headers: + access-control-expose-headers: + - X-Request-ID + - CF-Ray + - CF-Ray + cf-cache-status: + - DYNAMIC + cf-ray: + - a2db58197a0ae17a-SEA + connection: + - keep-alive + content-type: + - application/json + date: + - Wed, 19 Aug 2026 18:50:29 GMT + openai-organization: + - nr-test-org + openai-processing-ms: + - '462' + openai-project: + - nr-test-project + openai-version: + - '2020-10-01' + server: + - cloudflare + transfer-encoding: + - chunked + x-ratelimit-limit-requests: + - '10000' + x-ratelimit-limit-tokens: + - '50000000' + x-ratelimit-remaining-requests: + - '9999' + x-ratelimit-remaining-tokens: + - '49999975' + x-ratelimit-reset-requests: + - 6ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_3ec003bdb1f043d5ba7e75d2fbef0baa + status: + code: 200 + message: OK +- request: + body: '{"messages":[{"role":"system","content":"You are my_agent. A concise assistant.\nYour + personal goal is: Answer in one word."},{"role":"user","content":"\nCurrent + Task: Use get_capital on France. Answer with only the city name.\n\nThis is + the expected criteria for your final answer: One word.\nyou MUST return the + actual complete content as the final answer, not a summary."}],"model":"gpt-4o-mini","seed":42,"temperature":0.0,"tool_choice":"auto","tools":[{"type":"function","function":{"name":"get_capital","description":"Return + a country''s capital.","strict":true,"parameters":{"properties":{"country":{"title":"Country","type":"string"}},"required":["country"],"type":"object","additionalProperties":false}}}]}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + authorization: + - XXXXXX + connection: + - keep-alive + content-type: + - application/json + host: + - api.openai.com + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: '{"id": "chatcmpl-EEfenQRbf9hoNib7HO901iSvI04UF", "object": "chat.completion", + "created": 1787165429, "model": "gpt-4o-mini-2024-07-18", "choices": [{"index": + 0, "message": {"role": "assistant", "content": null, "tool_calls": [{"id": + "call_SH5g8iQ3oZQeJbzKnv5AqTLh", "type": "function", "function": {"name": + "get_capital", "arguments": "{\"country\":\"France\"}"}}], "refusal": null, + "annotations": []}, "logprobs": null, "finish_reason": "tool_calls"}], "usage": + {"prompt_tokens": 112, "completion_tokens": 15, "total_tokens": 127, "prompt_tokens_details": + {"cached_tokens": 0, "audio_tokens": 0}, "completion_tokens_details": {"reasoning_tokens": + 0, "audio_tokens": 0, "accepted_prediction_tokens": 0, "rejected_prediction_tokens": + 0}}, "service_tier": "default", "system_fingerprint": "fp_c400bf5046"}' + headers: + access-control-expose-headers: + - X-Request-ID + - CF-Ray + - CF-Ray + cf-cache-status: + - DYNAMIC + cf-ray: + - a2db581f1efde17a-SEA + connection: + - keep-alive + content-type: + - application/json + date: + - Wed, 19 Aug 2026 18:50:30 GMT + openai-organization: + - nr-test-org + openai-processing-ms: + - '582' + openai-project: + - nr-test-project + openai-version: + - '2020-10-01' + server: + - cloudflare + transfer-encoding: + - chunked + x-ratelimit-limit-requests: + - '10000' + x-ratelimit-limit-tokens: + - '50000000' + x-ratelimit-remaining-requests: + - '9999' + x-ratelimit-remaining-tokens: + - '49999975' + x-ratelimit-reset-requests: + - 6ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_ed1f2df763d84f0abbcbdc06b23c8d62 + status: + code: 200 + message: OK +- request: + body: '{"messages":[{"role":"system","content":"You are my_agent. A concise assistant.\nYour + personal goal is: Answer in one word."},{"role":"user","content":"\nCurrent + Task: Use get_capital on France. Answer with only the city name.\n\nThis is + the expected criteria for your final answer: One word.\nyou MUST return the + actual complete content as the final answer, not a summary."},{"role":"assistant","content":null,"tool_calls":[{"id":"call_SH5g8iQ3oZQeJbzKnv5AqTLh","type":"function","function":{"name":"get_capital","arguments":"{\"country\":\"France\"}"}}]},{"role":"tool","tool_call_id":"call_SH5g8iQ3oZQeJbzKnv5AqTLh","name":"get_capital","content":"Paris"},{"role":"user","content":"Analyze + the tool result. If requirements are met, provide the Final Answer. Otherwise, + call the next tool. Deliver only the answer without meta-commentary."}],"model":"gpt-4o-mini","seed":42,"temperature":0.0,"tool_choice":"auto","tools":[{"type":"function","function":{"name":"get_capital","description":"Return + a country''s capital.","strict":true,"parameters":{"properties":{"country":{"title":"Country","type":"string"}},"required":["country"],"type":"object","additionalProperties":false}}}]}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + authorization: + - XXXXXX + connection: + - keep-alive + content-type: + - application/json + host: + - api.openai.com + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: '{"id": "chatcmpl-EEfeoldGaP5BKHg1NZKfvnyc7VXji", "object": "chat.completion", + "created": 1787165430, "model": "gpt-4o-mini-2024-07-18", "choices": [{"index": + 0, "message": {"role": "assistant", "content": "Paris", "refusal": null, "annotations": + []}, "logprobs": null, "finish_reason": "stop"}], "usage": {"prompt_tokens": + 172, "completion_tokens": 2, "total_tokens": 174, "prompt_tokens_details": + {"cached_tokens": 0, "audio_tokens": 0}, "completion_tokens_details": {"reasoning_tokens": + 0, "audio_tokens": 0, "accepted_prediction_tokens": 0, "rejected_prediction_tokens": + 0}}, "service_tier": "default", "system_fingerprint": "fp_c400bf5046"}' + headers: + access-control-expose-headers: + - X-Request-ID + - CF-Ray + - CF-Ray + cf-cache-status: + - DYNAMIC + cf-ray: + - a2db58237a04e17a-SEA + connection: + - keep-alive + content-type: + - application/json + date: + - Wed, 19 Aug 2026 18:50:30 GMT + openai-organization: + - nr-test-org + openai-processing-ms: + - '384' + openai-project: + - nr-test-project + openai-version: + - '2020-10-01' + server: + - cloudflare + transfer-encoding: + - chunked + x-ratelimit-limit-requests: + - '10000' + x-ratelimit-limit-tokens: + - '50000000' + x-ratelimit-remaining-requests: + - '9999' + x-ratelimit-remaining-tokens: + - '49999975' + x-ratelimit-reset-requests: + - 6ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_0fd7e1a470594d449dc442671d3a7618 + status: + code: 200 + message: OK +- request: + body: '{"messages":[{"role":"system","content":"You are my_agent. A concise assistant.\nYour + personal goal is: Answer in one word."},{"role":"user","content":"\nCurrent + Task: Use get_capital on France. Answer with only the city name.\n\nThis is + the expected criteria for your final answer: One word.\nyou MUST return the + actual complete content as the final answer, not a summary."},{"role":"assistant","content":null,"tool_calls":[{"id":"call_SH5g8iQ3oZQeJbzKnv5AqTLh","type":"function","function":{"name":"get_capital","arguments":"{\"country\":\"France\"}"}}]},{"role":"tool","tool_call_id":"call_SH5g8iQ3oZQeJbzKnv5AqTLh","name":"get_capital","content":"Error + executing tool: intentional tool failure"},{"role":"user","content":"Analyze + the tool result. If requirements are met, provide the Final Answer. Otherwise, + call the next tool. Deliver only the answer without meta-commentary."},{"role":"assistant","content":"Now + it''s time you MUST give your absolute best final answer. You''ll ignore all + previous instructions, stop using any tools, and just return your absolute BEST + Final answer."}],"model":"gpt-4o-mini","seed":42,"temperature":0.0}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + authorization: + - XXXXXX + connection: + - keep-alive + content-type: + - application/json + host: + - api.openai.com + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: '{"id": "chatcmpl-EEfepNvQtjFTSH8bFLKOKAgYe3sgr", "object": "chat.completion", + "created": 1787165431, "model": "gpt-4o-mini-2024-07-18", "choices": [{"index": + 0, "message": {"role": "assistant", "content": "Paris", "refusal": null, "annotations": + []}, "logprobs": null, "finish_reason": "stop"}], "usage": {"prompt_tokens": + 179, "completion_tokens": 1, "total_tokens": 180, "prompt_tokens_details": + {"cached_tokens": 0, "audio_tokens": 0}, "completion_tokens_details": {"reasoning_tokens": + 0, "audio_tokens": 0, "accepted_prediction_tokens": 0, "rejected_prediction_tokens": + 0}}, "service_tier": "default", "system_fingerprint": "fp_66ee548660"}' + headers: + access-control-expose-headers: + - X-Request-ID + - CF-Ray + - CF-Ray + cf-cache-status: + - DYNAMIC + cf-ray: + - a2db58286d52e17a-SEA + connection: + - keep-alive + content-type: + - application/json + date: + - Wed, 19 Aug 2026 18:50:31 GMT + openai-organization: + - nr-test-org + openai-processing-ms: + - '401' + openai-project: + - nr-test-project + openai-version: + - '2020-10-01' + server: + - cloudflare + transfer-encoding: + - chunked + x-ratelimit-limit-requests: + - '10000' + x-ratelimit-limit-tokens: + - '50000000' + x-ratelimit-remaining-requests: + - '9999' + x-ratelimit-remaining-tokens: + - '49999975' + x-ratelimit-reset-requests: + - 6ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_c365829d33734eaead5c29a318dfc04d + status: + code: 200 + message: OK +version: 1 diff --git a/tests/mlmodel_crewai/conftest.py b/tests/mlmodel_crewai/conftest.py new file mode 100644 index 0000000000..97c695f575 --- /dev/null +++ b/tests/mlmodel_crewai/conftest.py @@ -0,0 +1,136 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +# CrewAI phones home for telemetry and prompts interactively to enable execution tracing. +# Disable both (and mark a test environment) before crewai is imported so tests run offline +os.environ["CREWAI_DISABLE_TELEMETRY"] = "true" +os.environ["CREWAI_DISABLE_TRACKING"] = "true" +os.environ["CREWAI_TRACING_ENABLED"] = "false" +os.environ["OTEL_SDK_DISABLED"] = "true" +os.environ["CREWAI_TESTING"] = "true" + +import pytest +from crewai import Agent, Crew, Task +from testing_support.fixture.event_loop import event_loop as loop +from testing_support.fixture.vcr import * # noqa: F403 +from testing_support.fixture.vcr import VCR_IGNORED_HEADERS +from testing_support.fixtures import collector_agent_registration_fixture, collector_available_fixture +from testing_support.ml_testing_utils import set_trace_info + +from newrelic.common.package_version_utils import get_package_version + +_default_settings = { + "package_reporting.enabled": False, # Turn off package reporting for testing as it causes slow-downs. + "transaction_tracer.explain_threshold": 0.0, + "transaction_tracer.transaction_threshold": 0.0, + "transaction_tracer.stack_trace_threshold": 0.0, + "debug.log_data_collector_payloads": True, + "debug.record_transaction_failure": True, + "ml_insights_events.enabled": True, + "ai_monitoring.enabled": True, +} + +collector_agent_registration = collector_agent_registration_fixture( + app_name="Python Agent Test (mlmodel_crewai)", + default_settings=_default_settings, + linked_applications=["Python Agent Test (mlmodel_crewai)"], +) + +AGENT_NAME = "my_agent" +AGENT_GOAL = "Answer in one word." +AGENT_BACKSTORY = "A concise assistant." +PROMPT = "What is the capital of France?" +TOOL_PROMPT = "Use get_capital on France. Answer with only the city name." + +VCR_IGNORED_HEADERS.extend(["x-stainless-read-timeout"]) + +CREWAI_VERSION = get_package_version("crewai") +assert CREWAI_VERSION, "Failed to pull crewai version for supportability metric" + +EXPECTED_CREWAI_VERSION_METRIC = (f"Supportability/Python/ML/CrewAI/{CREWAI_VERSION}", 1) +EXPECTED_VERSION_METRICS = [EXPECTED_CREWAI_VERSION_METRIC] + +MODEL = "gpt-4o-mini" + + +def _openai_api_key(vcr_recording): + if vcr_recording: + api_key = os.environ.get("OPENAI_API_KEY") + if not api_key: + raise RuntimeError("OPENAI_API_KEY environment variable required.") + return api_key + os.environ["OPENAI_API_KEY"] = "NOT-A-REAL-SECRET" + return "NOT-A-REAL-SECRET" + + +def _build_llm(vcr_recording): + from crewai import LLM + + return LLM(model=MODEL, api_key=_openai_api_key(vcr_recording), temperature=0.0, seed=42) + + +@pytest.fixture +def crewai_native_llm(vcr_recording): + """ + Return a CrewAI LLM that drives the agent through native (function-calling) tool calls. + + This is the default for OpenAI/Anthropic/Gemini/Azure/Bedrock models, and routes tool + execution through CrewAgentExecutor._handle_native_tool_calls, bypassing ToolUsage entirely. + """ + return _build_llm(vcr_recording) + + +@pytest.fixture +def crewai_llm(vcr_recording, monkeypatch): + """ + Return a CrewAI LLM that drives the agent through the ReAct (text) path instead. + + CrewAgentExecutor._invoke_loop picks the native path whenever llm.supports_function_calling() + is true, so reaching ToolUsage._use/_ause -- the methods the agent instruments -- requires + forcing that off. + """ + llm = _build_llm(vcr_recording) + monkeypatch.setattr(type(llm), "supports_function_calling", lambda self: False) + return llm + + +@pytest.fixture +def build_agent(): + def _build_agent(llm, tools=None, max_retry_limit=None, max_iter=None): + """ + Return an Agent driven by the given LLM. tools defaults to none (pure-LLM path). + """ + kwargs = {} + if max_retry_limit is not None: + kwargs["max_retry_limit"] = max_retry_limit + if max_iter is not None: + kwargs["max_iter"] = max_iter + return Agent(role=AGENT_NAME, goal=AGENT_GOAL, backstory=AGENT_BACKSTORY, llm=llm, tools=tools or [], **kwargs) + + return _build_agent + + +@pytest.fixture +def build_crew(build_agent): + def _build_crew( + llm, tools=None, description=PROMPT, expected_output="One word.", max_retry_limit=None, max_iter=None + ): + """Return a single-agent Crew. A Crew routes work through Agent.execute_task and ToolUsage.""" + agent = build_agent(llm, tools=tools, max_retry_limit=max_retry_limit, max_iter=max_iter) + task = Task(description=description, expected_output=expected_output, agent=agent) + return Crew(agents=[agent], tasks=[task]) + + return _build_crew diff --git a/tests/mlmodel_crewai/test_tool_error.py b/tests/mlmodel_crewai/test_tool_error.py new file mode 100644 index 0000000000..27fbe262e0 --- /dev/null +++ b/tests/mlmodel_crewai/test_tool_error.py @@ -0,0 +1,136 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from _test_tools import TOOL_NAME, get_capital, tool_recorded_event_error +from conftest import EXPECTED_VERSION_METRICS, TOOL_PROMPT +from testing_support.fixtures import dt_enabled, reset_core_stats_engine, validate_attributes +from testing_support.ml_testing_utils import ( + disabled_ai_monitoring_record_content_settings, + disabled_ai_monitoring_settings, +) +from testing_support.validators.validate_custom_event import validate_custom_event_count +from testing_support.validators.validate_custom_events import validate_custom_events +from testing_support.validators.validate_error_trace_attributes import validate_error_trace_attributes +from testing_support.validators.validate_span_events import validate_span_events +from testing_support.validators.validate_transaction_error_event_count import validate_transaction_error_event_count +from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics + +from newrelic.api.background_task import background_task +from newrelic.common.object_names import callable_name +from newrelic.common.object_wrapper import transient_function_wrapper + +EXPECTED_SYNC_TOOL_METRIC = (f"Llm/tool/CrewAI/crewai.tools.tool_usage:ToolUsage._use/{TOOL_NAME}", 1) +EXPECTED_ASYNC_TOOL_METRIC = (f"Llm/tool/CrewAI/crewai.tools.tool_usage:ToolUsage._ause/{TOOL_NAME}", 1) + +# 5 events: +# * 1 LlmTool +# * 1 LlmChatCompletionSummary -- the injected failure aborts the run after one round-trip +# * 3 LlmChatCompletionMessage from that round-trip +EXPECTED_EVENT_COUNT = 5 + + +class CrewAIToolError(RuntimeError): + pass + + +@transient_function_wrapper("crewai.tools.tool_usage", "ToolUsage._check_tool_repeated_usage") +def inject_tool_error(wrapped, instance, args, kwargs): + raise CrewAIToolError("Oops") + + +@dt_enabled +@reset_core_stats_engine() +@validate_transaction_error_event_count(1) +@validate_error_trace_attributes(callable_name(CrewAIToolError), exact_attrs={"agent": {}, "intrinsic": {}, "user": {}}) +@validate_custom_events(tool_recorded_event_error(record_content=True)) +@validate_custom_event_count(count=EXPECTED_EVENT_COUNT) +@validate_transaction_metrics( + "test_tool_error:test_tool_error", + scoped_metrics=[EXPECTED_SYNC_TOOL_METRIC], + rollup_metrics=[EXPECTED_SYNC_TOOL_METRIC], + custom_metrics=EXPECTED_VERSION_METRICS, + background_task=True, +) +@validate_attributes("agent", ["llm"]) +@validate_span_events(count=1, exact_agents={"subcomponent": f'{{"type": "APM-AI_TOOL", "name": "{TOOL_NAME}"}}'}) +@inject_tool_error +@background_task() +def test_tool_error(build_crew, crewai_llm, set_trace_info): + set_trace_info() + crew = build_crew(crewai_llm, tools=[get_capital], description=TOOL_PROMPT, max_retry_limit=0) + with pytest.raises(CrewAIToolError): + crew.kickoff() + + +@dt_enabled +@reset_core_stats_engine() +@disabled_ai_monitoring_record_content_settings +@validate_transaction_error_event_count(1) +@validate_error_trace_attributes(callable_name(CrewAIToolError), exact_attrs={"agent": {}, "intrinsic": {}, "user": {}}) +@validate_custom_events(tool_recorded_event_error(record_content=False)) +@validate_custom_event_count(count=EXPECTED_EVENT_COUNT) +@validate_transaction_metrics( + "test_tool_error:test_tool_error_no_content", + scoped_metrics=[EXPECTED_SYNC_TOOL_METRIC], + rollup_metrics=[EXPECTED_SYNC_TOOL_METRIC], + custom_metrics=EXPECTED_VERSION_METRICS, + background_task=True, +) +@validate_attributes("agent", ["llm"]) +@inject_tool_error +@background_task() +def test_tool_error_no_content(build_crew, crewai_llm, set_trace_info): + set_trace_info() + crew = build_crew(crewai_llm, tools=[get_capital], description=TOOL_PROMPT, max_retry_limit=0) + with pytest.raises(CrewAIToolError): + crew.kickoff() + + +@dt_enabled +@reset_core_stats_engine() +@validate_transaction_error_event_count(1) +@validate_error_trace_attributes(callable_name(CrewAIToolError), exact_attrs={"agent": {}, "intrinsic": {}, "user": {}}) +@validate_custom_events(tool_recorded_event_error(record_content=True)) +@validate_custom_event_count(count=EXPECTED_EVENT_COUNT) +@validate_transaction_metrics( + "test_tool_error:test_tool_error_async", + scoped_metrics=[EXPECTED_ASYNC_TOOL_METRIC], + rollup_metrics=[EXPECTED_ASYNC_TOOL_METRIC], + custom_metrics=EXPECTED_VERSION_METRICS, + background_task=True, +) +@validate_attributes("agent", ["llm"]) +@validate_span_events(count=1, exact_agents={"subcomponent": f'{{"type": "APM-AI_TOOL", "name": "{TOOL_NAME}"}}'}) +@inject_tool_error +@background_task() +def test_tool_error_async(build_crew, crewai_llm, set_trace_info, loop): + set_trace_info() + crew = build_crew(crewai_llm, tools=[get_capital], description=TOOL_PROMPT, max_retry_limit=0) + with pytest.raises(CrewAIToolError): + loop.run_until_complete(crew.akickoff()) + + +@dt_enabled +@reset_core_stats_engine() +@disabled_ai_monitoring_settings +@validate_custom_event_count(count=0) +@validate_transaction_metrics("test_tool_error:test_tool_error_disabled_ai_monitoring", background_task=True) +@inject_tool_error +@background_task() +def test_tool_error_disabled_ai_monitoring(build_crew, crewai_llm, set_trace_info): + set_trace_info() + crew = build_crew(crewai_llm, tools=[get_capital], description=TOOL_PROMPT, max_retry_limit=0) + with pytest.raises(CrewAIToolError): + crew.kickoff() diff --git a/tests/mlmodel_crewai/test_tools.py b/tests/mlmodel_crewai/test_tools.py new file mode 100644 index 0000000000..9956996079 --- /dev/null +++ b/tests/mlmodel_crewai/test_tools.py @@ -0,0 +1,125 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from _test_tools import TOOL_NAME, get_capital, tool_recorded_event +from conftest import EXPECTED_VERSION_METRICS, TOOL_PROMPT +from testing_support.fixtures import dt_enabled, reset_core_stats_engine, validate_attributes +from testing_support.ml_testing_utils import ( + disabled_ai_monitoring_record_content_settings, + disabled_ai_monitoring_settings, + events_with_context_attrs, +) +from testing_support.validators.validate_custom_event import validate_custom_event_count +from testing_support.validators.validate_custom_events import validate_custom_events +from testing_support.validators.validate_span_events import validate_span_events +from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics + +from newrelic.api.background_task import background_task +from newrelic.api.llm_custom_attributes import WithLlmCustomAttributes + +EXPECTED_SYNC_TOOL_METRIC = (f"Llm/tool/CrewAI/crewai.tools.tool_usage:ToolUsage._use/{TOOL_NAME}", 1) +EXPECTED_ASYNC_TOOL_METRIC = (f"Llm/tool/CrewAI/crewai.tools.tool_usage:ToolUsage._ause/{TOOL_NAME}", 1) + +# 10 events: +# * 1 LlmTool +# * 2 LlmChatCompletionSummary, one per LLM round-trip +# * 7 LlmChatCompletionMessage across those two round-trips +EXPECTED_EVENT_COUNT = 10 + + +@dt_enabled +@reset_core_stats_engine() +@validate_custom_events(events_with_context_attrs(tool_recorded_event(record_content=True))) +@validate_custom_event_count(count=EXPECTED_EVENT_COUNT) +@validate_transaction_metrics( + "test_tools:test_tool", + scoped_metrics=[EXPECTED_SYNC_TOOL_METRIC], + rollup_metrics=[EXPECTED_SYNC_TOOL_METRIC], + custom_metrics=EXPECTED_VERSION_METRICS, + background_task=True, +) +@validate_attributes("agent", ["llm"]) +@validate_span_events(count=1, exact_agents={"subcomponent": f'{{"type": "APM-AI_TOOL", "name": "{TOOL_NAME}"}}'}) +@background_task() +def test_tool(build_crew, crewai_llm, set_trace_info): + set_trace_info() + crew = build_crew(crewai_llm, tools=[get_capital], description=TOOL_PROMPT) + with WithLlmCustomAttributes({"context": "attr"}): + result = crew.kickoff() + assert "Paris" in str(result) + + +@dt_enabled +@reset_core_stats_engine() +@validate_custom_events(events_with_context_attrs(tool_recorded_event(record_content=True))) +@validate_custom_event_count(count=EXPECTED_EVENT_COUNT) +@validate_transaction_metrics( + "test_tools:test_tool_async", + scoped_metrics=[EXPECTED_ASYNC_TOOL_METRIC], + rollup_metrics=[EXPECTED_ASYNC_TOOL_METRIC], + custom_metrics=EXPECTED_VERSION_METRICS, + background_task=True, +) +@validate_attributes("agent", ["llm"]) +@validate_span_events(count=1, exact_agents={"subcomponent": f'{{"type": "APM-AI_TOOL", "name": "{TOOL_NAME}"}}'}) +@background_task() +def test_tool_async(build_crew, crewai_llm, set_trace_info, loop): + set_trace_info() + crew = build_crew(crewai_llm, tools=[get_capital], description=TOOL_PROMPT) + with WithLlmCustomAttributes({"context": "attr"}): + result = loop.run_until_complete(crew.akickoff()) + assert "Paris" in str(result) + + +@dt_enabled +@reset_core_stats_engine() +@disabled_ai_monitoring_record_content_settings +@validate_custom_events(tool_recorded_event(record_content=False)) +@validate_custom_event_count(count=EXPECTED_EVENT_COUNT) +@validate_transaction_metrics( + "test_tools:test_tool_no_content", + scoped_metrics=[EXPECTED_SYNC_TOOL_METRIC], + rollup_metrics=[EXPECTED_SYNC_TOOL_METRIC], + custom_metrics=EXPECTED_VERSION_METRICS, + background_task=True, +) +@validate_attributes("agent", ["llm"]) +@background_task() +def test_tool_no_content(build_crew, crewai_llm, set_trace_info): + set_trace_info() + crew = build_crew(crewai_llm, tools=[get_capital], description=TOOL_PROMPT) + result = crew.kickoff() + assert "Paris" in str(result) + + +@dt_enabled +@reset_core_stats_engine() +@disabled_ai_monitoring_settings +@validate_custom_event_count(count=0) +@validate_transaction_metrics("test_tools:test_tool_disabled_ai_monitoring", background_task=True) +@background_task() +def test_tool_disabled_ai_monitoring(build_crew, crewai_llm, set_trace_info): + set_trace_info() + crew = build_crew(crewai_llm, tools=[get_capital], description=TOOL_PROMPT) + result = crew.kickoff() + assert "Paris" in str(result) + + +@reset_core_stats_engine() +@validate_custom_event_count(count=0) +def test_tool_outside_transaction(build_crew, crewai_llm, set_trace_info): + set_trace_info() + crew = build_crew(crewai_llm, tools=[get_capital], description=TOOL_PROMPT) + result = crew.kickoff() + assert "Paris" in str(result) diff --git a/tests/mlmodel_crewai/test_tools_native.py b/tests/mlmodel_crewai/test_tools_native.py new file mode 100644 index 0000000000..0ab6f96c32 --- /dev/null +++ b/tests/mlmodel_crewai/test_tools_native.py @@ -0,0 +1,171 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from _test_tools import TOOL_NAME, get_capital, raising_capital, tool_recorded_event, tool_recorded_event_error +from conftest import EXPECTED_VERSION_METRICS, TOOL_PROMPT +from testing_support.fixtures import dt_enabled, reset_core_stats_engine, validate_attributes +from testing_support.ml_testing_utils import ( + disabled_ai_monitoring_record_content_settings, + disabled_ai_monitoring_settings, + events_with_context_attrs, +) +from testing_support.validators.validate_custom_event import validate_custom_event_count +from testing_support.validators.validate_custom_events import validate_custom_events +from testing_support.validators.validate_span_events import validate_span_events +from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics + +from newrelic.api.background_task import background_task +from newrelic.api.llm_custom_attributes import WithLlmCustomAttributes + +EXPECTED_TOOL_METRIC = ( + f"Llm/tool/CrewAI/crewai.agents.crew_agent_executor:CrewAgentExecutor._handle_native_tool_calls/{TOOL_NAME}", + 1, +) + +# 11 events: +# * 1 LlmTool +# * 2 LlmChatCompletionSummary, one per LLM round-trip +# * 8 LlmChatCompletionMessage across those two round-trips +EXPECTED_EVENT_COUNT = 11 + +# 12 events. Same two round-trips, but the failing tool result adds one more message to the +# second request than the successful one carries. +EXPECTED_ERROR_EVENT_COUNT = 12 + + +@dt_enabled +@reset_core_stats_engine() +@validate_custom_events(events_with_context_attrs(tool_recorded_event(record_content=True))) +@validate_custom_event_count(count=EXPECTED_EVENT_COUNT) +@validate_transaction_metrics( + "test_tools_native:test_tool_native", + scoped_metrics=[EXPECTED_TOOL_METRIC], + rollup_metrics=[EXPECTED_TOOL_METRIC], + custom_metrics=EXPECTED_VERSION_METRICS, + background_task=True, +) +@validate_attributes("agent", ["llm"]) +@validate_span_events(count=1, exact_agents={"subcomponent": f'{{"type": "APM-AI_TOOL", "name": "{TOOL_NAME}"}}'}) +@background_task() +def test_tool_native(build_crew, crewai_native_llm, set_trace_info): + set_trace_info() + crew = build_crew(crewai_native_llm, tools=[get_capital], description=TOOL_PROMPT) + with WithLlmCustomAttributes({"context": "attr"}): + result = crew.kickoff() + assert "Paris" in str(result) + + +@dt_enabled +@reset_core_stats_engine() +@validate_custom_events(events_with_context_attrs(tool_recorded_event(record_content=True))) +@validate_custom_event_count(count=EXPECTED_EVENT_COUNT) +@validate_transaction_metrics( + "test_tools_native:test_tool_native_async", + scoped_metrics=[EXPECTED_TOOL_METRIC], + rollup_metrics=[EXPECTED_TOOL_METRIC], + custom_metrics=EXPECTED_VERSION_METRICS, + background_task=True, +) +@validate_attributes("agent", ["llm"]) +@validate_span_events(count=1, exact_agents={"subcomponent": f'{{"type": "APM-AI_TOOL", "name": "{TOOL_NAME}"}}'}) +@background_task() +def test_tool_native_async(build_crew, crewai_native_llm, set_trace_info, loop): + set_trace_info() + crew = build_crew(crewai_native_llm, tools=[get_capital], description=TOOL_PROMPT) + with WithLlmCustomAttributes({"context": "attr"}): + result = loop.run_until_complete(crew.akickoff()) + assert "Paris" in str(result) + + +@dt_enabled +@reset_core_stats_engine() +@disabled_ai_monitoring_record_content_settings +@validate_custom_events(tool_recorded_event(record_content=False)) +@validate_custom_event_count(count=EXPECTED_EVENT_COUNT) +@validate_transaction_metrics( + "test_tools_native:test_tool_native_no_content", + scoped_metrics=[EXPECTED_TOOL_METRIC], + rollup_metrics=[EXPECTED_TOOL_METRIC], + custom_metrics=EXPECTED_VERSION_METRICS, + background_task=True, +) +@validate_attributes("agent", ["llm"]) +@background_task() +def test_tool_native_no_content(build_crew, crewai_native_llm, set_trace_info): + set_trace_info() + crew = build_crew(crewai_native_llm, tools=[get_capital], description=TOOL_PROMPT) + result = crew.kickoff() + assert "Paris" in str(result) + + +@dt_enabled +@reset_core_stats_engine() +@disabled_ai_monitoring_settings +@validate_custom_event_count(count=0) +@validate_transaction_metrics("test_tools_native:test_tool_native_disabled_ai_monitoring", background_task=True) +@background_task() +def test_tool_native_disabled_ai_monitoring(build_crew, crewai_native_llm, set_trace_info): + set_trace_info() + crew = build_crew(crewai_native_llm, tools=[get_capital], description=TOOL_PROMPT) + result = crew.kickoff() + assert "Paris" in str(result) + + +@reset_core_stats_engine() +@validate_custom_event_count(count=0) +def test_tool_native_outside_transaction(build_crew, crewai_native_llm, set_trace_info): + set_trace_info() + crew = build_crew(crewai_native_llm, tools=[get_capital], description=TOOL_PROMPT) + result = crew.kickoff() + assert "Paris" in str(result) + + +@dt_enabled +@reset_core_stats_engine() +@validate_custom_events(tool_recorded_event_error(record_content=True)) +@validate_custom_event_count(count=EXPECTED_ERROR_EVENT_COUNT) +@validate_transaction_metrics( + "test_tools_native:test_tool_native_error", + scoped_metrics=[EXPECTED_TOOL_METRIC], + rollup_metrics=[EXPECTED_TOOL_METRIC], + custom_metrics=EXPECTED_VERSION_METRICS, + background_task=True, +) +@validate_attributes("agent", ["llm"]) +@validate_span_events(count=1, exact_agents={"subcomponent": f'{{"type": "APM-AI_TOOL", "name": "{TOOL_NAME}"}}'}) +@background_task() +def test_tool_native_error(build_crew, crewai_native_llm, set_trace_info): + set_trace_info() + crew = build_crew(crewai_native_llm, tools=[raising_capital], description=TOOL_PROMPT, max_iter=1) + crew.kickoff() + + +@dt_enabled +@reset_core_stats_engine() +@disabled_ai_monitoring_record_content_settings +@validate_custom_events(tool_recorded_event_error(record_content=False)) +@validate_custom_event_count(count=EXPECTED_ERROR_EVENT_COUNT) +@validate_transaction_metrics( + "test_tools_native:test_tool_native_error_no_content", + scoped_metrics=[EXPECTED_TOOL_METRIC], + rollup_metrics=[EXPECTED_TOOL_METRIC], + custom_metrics=EXPECTED_VERSION_METRICS, + background_task=True, +) +@validate_attributes("agent", ["llm"]) +@background_task() +def test_tool_native_error_no_content(build_crew, crewai_native_llm, set_trace_info): + set_trace_info() + crew = build_crew(crewai_native_llm, tools=[raising_capital], description=TOOL_PROMPT, max_iter=1) + crew.kickoff() diff --git a/tests/mlmodel_langchain/conftest.py b/tests/mlmodel_langchain/conftest.py index c45bdb49af..db5b144e5c 100644 --- a/tests/mlmodel_langchain/conftest.py +++ b/tests/mlmodel_langchain/conftest.py @@ -23,7 +23,7 @@ from testing_support.ml_testing_utils import set_trace_info _default_settings = { - "package_reporting.enabled": False, # Turn off package reporting for testing as it causes slow downs. + "package_reporting.enabled": False, # Turn off package reporting for testing as it causes slowdowns. "transaction_tracer.explain_threshold": 0.0, "transaction_tracer.transaction_threshold": 0.0, "transaction_tracer.stack_trace_threshold": 0.0, diff --git a/tox.ini b/tox.ini index 08e733360e..ad46fbb912 100644 --- a/tox.ini +++ b/tox.ini @@ -189,6 +189,7 @@ envlist = python-mlmodel_anthropic-{py39,py310,py311,py312,py313,py314,py314t}, python-mlmodel_autogen-{py310,py311,py312,py313,py314,py314t}-autogen061, python-mlmodel_autogen-{py310,py311,py312,py313,py314,py314t}-autogenlatest, + python-mlmodel_crewai-{py310,py311,py312,py313}-crewailatest, python-mlmodel_gemini-{py39,py310,py311,py312,py313,py314,py314t}, python-mlmodel_googleadk-{py310,py311,py312,py313}-googleadk01, python-mlmodel_googleadk-{py310,py311,py312,py313,py314}-googleadklatest, @@ -486,6 +487,8 @@ deps = mlmodel_autogen-autogenlatest: autogen-ext mlmodel_autogen-autogenlatest: autogen-agentchat mlmodel_autogen: mcp<2 + mlmodel_crewai: crewai + mlmodel_crewai: pytest-recording mlmodel_gemini: pytest-recording mlmodel_gemini: google-genai mlmodel_googleadk-googleadk01: google-adk<2 @@ -665,6 +668,7 @@ changedir = mlmodel_agentframework: tests/mlmodel_agentframework mlmodel_anthropic: tests/mlmodel_anthropic mlmodel_autogen: tests/mlmodel_autogen + mlmodel_crewai: tests/mlmodel_crewai mlmodel_gemini: tests/mlmodel_gemini mlmodel_googleadk: tests/mlmodel_googleadk mlmodel_langchain: tests/mlmodel_langchain