From 4edcda4098dddd0f1e511c7159bebbb59cf02093 Mon Sep 17 00:00:00 2001 From: Ian Arawjo Date: Mon, 16 Dec 2024 16:23:58 -0500 Subject: [PATCH] Update function calling example --- .../examples/basic-function-calls.cforge | 6252 +++++++++++------ chainforge/react-server/src/backend/utils.ts | 7 +- 2 files changed, 4295 insertions(+), 1964 deletions(-) diff --git a/chainforge/examples/basic-function-calls.cforge b/chainforge/examples/basic-function-calls.cforge index 62b6fe9..d586bde 100644 --- a/chainforge/examples/basic-function-calls.cforge +++ b/chainforge/examples/basic-function-calls.cforge @@ -3,81 +3,92 @@ "nodes": [ { "width": 312, - "height": 370, + "height": 410, "data": { "fields": [ { - "text": "[[FUNCTION]] get_current_weather\\{\n \"format\": \"celsius\",\n \"location\": \"Boston, MA\"\n\\}", + "text": "[[TOOLS]] get_current_weather \\{\"location\":\"Boston, MA\",\"format\":\"fahrenheit\"\\}", "prompt": "What's the weather like in Boston?", "fill_history": { "prompt": "What's the weather like in Boston?" }, "llm": { + "key": "1725e2ec-cd4b-4211-a73a-3d6ef4482451", + "name": "gpt-4-turbo", + "emoji": "🤖", + "model": "gpt-4-turbo", "base_model": "gpt-3.5-turbo", - "emoji": "🌦️", - "formData": { - "frequency_penalty": 0, - "function_call": "auto", - "functions": "[\n {\n \"name\": \"get_current_weather\",\n \"description\": \"Get the current weather\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"location\": {\n \"type\": \"string\",\n \"description\": \"The city and state, e.g. San Francisco, CA\"\n },\n \"format\": {\n \"type\": \"string\",\n \"enum\": [\"celsius\", \"fahrenheit\"],\n \"description\": \"The temperature unit to use. Infer this from the users location.\"\n }\n },\n \"required\": [\"location\", \"format\"]\n }\n }\n]", - "model": "gpt-3.5-turbo-0613", - "presence_penalty": 0, - "shortname": "gpt3.5-turbo-0613", - "stop": "", + "temp": 1, + "settings": { "system_msg": "You are a helpful assistant.", "temperature": 1, - "top_p": 1 - }, - "key": "af7031af-96ae-44b8-bcc4-3538f3174b73", - "model": "gpt-3.5-turbo-0613", - "name": "gpt3.5-turbo-0613", - "settings": { - "frequency_penalty": 0, - "function_call": "auto", - "functions": [ + "response_format": { + "type": "text" + }, + "tools": [ { - "description": "Get the current weather", - "name": "get_current_weather", - "parameters": { - "properties": { - "format": { - "description": "The temperature unit to use. Infer this from the users location.", - "enum": [ - "celsius", - "fahrenheit" - ], - "type": "string" + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA" + }, + "format": { + "type": "string", + "enum": [ + "celsius", + "fahrenheit" + ], + "description": "The temperature unit to use. Infer this from the users location." + } }, - "location": { - "description": "The city and state, e.g. San Francisco, CA", - "type": "string" - } - }, - "required": [ - "location", - "format" - ], - "type": "object" + "required": [ + "location", + "format" + ] + } } } ], + "tool_choice": "auto", + "parallel_tool_calls": true, + "top_p": 1, + "stop": [], "presence_penalty": 0, - "temperature": 1, - "top_p": 1 + "frequency_penalty": 0 + }, + "formData": { + "shortname": "gpt-4-turbo", + "model": "gpt-4-turbo", + "system_msg": "You are a helpful assistant.", + "temperature": 1, + "response_format": "text", + "tools": "[{ \"type\": \"function\",\n \"function\": {\n \"name\": \"get_current_weather\",\n \"description\": \"Get the current weather\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"location\": {\n \"type\": \"string\",\n \"description\": \"The city and state, e.g. San Francisco, CA\"\n },\n \"format\": {\n \"type\": \"string\",\n \"enum\": [\"celsius\", \"fahrenheit\"],\n \"description\": \"The temperature unit to use. Infer this from the users location.\"\n }\n },\n \"required\": [\"location\", \"format\"]\n }\n }\n}]", + "tool_choice": "auto", + "parallel_tool_calls": true, + "top_p": 1, + "stop": "", + "presence_penalty": 0, + "frequency_penalty": 0 }, - "temp": 1, "progress": { - "success": 0, + "success": 80, "error": 0 } }, - "batch_id": "cfa8f743-32c7-4a24-8141-9e1fa264af49", + "uid": "2e2d2d3f-d162-4f6a-af6a-23f2628748e5", "metavars": { "__pt": "{prompt}", - "LLM_0": "gpt3.5-turbo-0613" + "LLM_0": "gpt-4-turbo" } }, { - "text": "I'm sorry, I cannot provide real-time information as I do not have access to live data. However, you can check reliable weather websites or applications for up-to-date information about the weather in Boston.", + "text": "I'm sorry, but I don't have real-time data access. You can check the weather in Boston by using a weather website or app, or by asking a voice assistant like Siri or Google Assistant for the current weather in Boston.", "prompt": "What's the weather like in Boston?", "fill_history": { "prompt": "What's the weather like in Boston?" @@ -86,36 +97,130 @@ "base_model": "gpt-3.5-turbo", "emoji": "🙂", "formData": { - "frequency_penalty": 0, - "function_call": "", - "functions": "", + "shortname": "no-tools-gpt3.5", "model": "gpt-3.5-turbo", - "presence_penalty": 0, - "shortname": "old-gpt3.5", - "stop": "", "system_msg": "You are a helpful assistant.", "temperature": 1, - "top_p": 1 + "response_format": "text", + "tools": "", + "tool_choice": "", + "parallel_tool_calls": true, + "top_p": 1, + "stop": "", + "presence_penalty": 0, + "frequency_penalty": 0, + "function_call": "", + "functions": "" }, "key": "fead74ce-615a-4812-869d-9735d9d985de", "model": "gpt-3.5-turbo", - "name": "old-gpt3.5", + "name": "no-tools-gpt3.5", "settings": { - "frequency_penalty": 0, - "presence_penalty": 0, + "system_msg": "You are a helpful assistant.", "temperature": 1, - "top_p": 1 + "response_format": { + "type": "text" + }, + "tools": [], + "tool_choice": "", + "parallel_tool_calls": true, + "top_p": 1, + "stop": [], + "presence_penalty": 0, + "frequency_penalty": 0, + "function_call": "", + "functions": [] }, "temp": 1, "progress": { - "success": 0, + "success": 100, "error": 0 } }, - "batch_id": "e342daaf-acde-4988-a759-c1c753efd670", + "uid": "64f90d18-939b-4e59-867e-5c433edd5103", "metavars": { "__pt": "{prompt}", - "LLM_0": "old-gpt3.5" + "LLM_0": "no-tools-gpt3.5" + } + }, + { + "text": "[[TOOLS]] \\{\"name\":\"get_current_weather\",\"input\":\\{\"format\":\"fahrenheit\",\"location\":\"Boston, MA\"\\}\\}", + "prompt": "What's the weather like in Boston?", + "fill_history": { + "prompt": "What's the weather like in Boston?" + }, + "llm": { + "key": "4caee194-fa12-4ca8-bb9e-1c8d7d5f4a08", + "name": "Claude", + "emoji": "📚", + "model": "claude-3-5-sonnet-latest", + "base_model": "claude-v1", + "temp": 0.5, + "settings": { + "system_msg": "", + "temperature": 1, + "tools": [ + { + "name": "get_current_weather", + "description": "Get the current weather", + "input_schema": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA" + }, + "format": { + "type": "string", + "enum": [ + "celsius", + "fahrenheit" + ], + "description": "The temperature unit to use. Infer this from the users location." + } + }, + "required": [ + "location", + "format" + ] + } + } + ], + "tool_choice": { + "type": "auto" + }, + "parallel_tool_calls": true, + "max_tokens_to_sample": 1024, + "custom_prompt_wrapper": "", + "stop_sequences": [ + "\n\nHuman:" + ], + "top_k": -1, + "top_p": -1 + }, + "formData": { + "shortname": "Claude", + "model": "claude-3-5-sonnet-latest", + "system_msg": "", + "temperature": 1, + "tools": "[{\n \"name\": \"get_current_weather\",\n \"description\": \"Get the current weather\",\n \"input_schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"location\": {\n \"type\": \"string\",\n \"description\": \"The city and state, e.g. San Francisco, CA\"\n },\n \"format\": {\n \"type\": \"string\",\n \"enum\": [\"celsius\", \"fahrenheit\"],\n \"description\": \"The temperature unit to use. Infer this from the users location.\"\n }\n },\n \"required\": [\"location\", \"format\"]\n }\n }\n]", + "tool_choice": "auto", + "parallel_tool_calls": true, + "max_tokens_to_sample": 1024, + "custom_prompt_wrapper": "", + "stop_sequences": "\"\n\nHuman:\"", + "top_k": -1, + "top_p": -1 + }, + "progress": { + "success": 100, + "error": 0 + } + }, + "uid": "eb2ba7da-fc44-4b3e-9cf9-6ee684e84ac6", + "metavars": { + "__pt": "{prompt}", + "LLM_0": "Claude" } }, { @@ -125,71 +230,82 @@ "prompt": "Who directed the movie Twister?" }, "llm": { + "key": "1725e2ec-cd4b-4211-a73a-3d6ef4482451", + "name": "gpt-4-turbo", + "emoji": "🤖", + "model": "gpt-4-turbo", "base_model": "gpt-3.5-turbo", - "emoji": "🌦️", - "formData": { - "frequency_penalty": 0, - "function_call": "auto", - "functions": "[\n {\n \"name\": \"get_current_weather\",\n \"description\": \"Get the current weather\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"location\": {\n \"type\": \"string\",\n \"description\": \"The city and state, e.g. San Francisco, CA\"\n },\n \"format\": {\n \"type\": \"string\",\n \"enum\": [\"celsius\", \"fahrenheit\"],\n \"description\": \"The temperature unit to use. Infer this from the users location.\"\n }\n },\n \"required\": [\"location\", \"format\"]\n }\n }\n]", - "model": "gpt-3.5-turbo-0613", - "presence_penalty": 0, - "shortname": "gpt3.5-turbo-0613", - "stop": "", + "temp": 1, + "settings": { "system_msg": "You are a helpful assistant.", "temperature": 1, - "top_p": 1 - }, - "key": "af7031af-96ae-44b8-bcc4-3538f3174b73", - "model": "gpt-3.5-turbo-0613", - "name": "gpt3.5-turbo-0613", - "settings": { - "frequency_penalty": 0, - "function_call": "auto", - "functions": [ + "response_format": { + "type": "text" + }, + "tools": [ { - "description": "Get the current weather", - "name": "get_current_weather", - "parameters": { - "properties": { - "format": { - "description": "The temperature unit to use. Infer this from the users location.", - "enum": [ - "celsius", - "fahrenheit" - ], - "type": "string" + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA" + }, + "format": { + "type": "string", + "enum": [ + "celsius", + "fahrenheit" + ], + "description": "The temperature unit to use. Infer this from the users location." + } }, - "location": { - "description": "The city and state, e.g. San Francisco, CA", - "type": "string" - } - }, - "required": [ - "location", - "format" - ], - "type": "object" + "required": [ + "location", + "format" + ] + } } } ], + "tool_choice": "auto", + "parallel_tool_calls": true, + "top_p": 1, + "stop": [], "presence_penalty": 0, - "temperature": 1, - "top_p": 1 + "frequency_penalty": 0 + }, + "formData": { + "shortname": "gpt-4-turbo", + "model": "gpt-4-turbo", + "system_msg": "You are a helpful assistant.", + "temperature": 1, + "response_format": "text", + "tools": "[{ \"type\": \"function\",\n \"function\": {\n \"name\": \"get_current_weather\",\n \"description\": \"Get the current weather\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"location\": {\n \"type\": \"string\",\n \"description\": \"The city and state, e.g. San Francisco, CA\"\n },\n \"format\": {\n \"type\": \"string\",\n \"enum\": [\"celsius\", \"fahrenheit\"],\n \"description\": \"The temperature unit to use. Infer this from the users location.\"\n }\n },\n \"required\": [\"location\", \"format\"]\n }\n }\n}]", + "tool_choice": "auto", + "parallel_tool_calls": true, + "top_p": 1, + "stop": "", + "presence_penalty": 0, + "frequency_penalty": 0 }, - "temp": 1, "progress": { - "success": 0, + "success": 80, "error": 0 } }, - "batch_id": "ebcba322-b11c-4680-8170-98b0f3bebbe5", + "uid": "423a06d4-1111-48a8-9897-5348ea362ebc", "metavars": { "__pt": "{prompt}", - "LLM_0": "gpt3.5-turbo-0613" + "LLM_0": "gpt-4-turbo" } }, { - "text": "The movie \"Twister\" was directed by Jan de Bont.", + "text": "The movie Twister was directed by Jan de Bont. It was released in 1996 and stars Helen Hunt and Bill Paxton.", "prompt": "Who directed the movie Twister?", "fill_history": { "prompt": "Who directed the movie Twister?" @@ -198,110 +314,215 @@ "base_model": "gpt-3.5-turbo", "emoji": "🙂", "formData": { - "frequency_penalty": 0, - "function_call": "", - "functions": "", + "shortname": "no-tools-gpt3.5", "model": "gpt-3.5-turbo", - "presence_penalty": 0, - "shortname": "old-gpt3.5", - "stop": "", "system_msg": "You are a helpful assistant.", "temperature": 1, - "top_p": 1 + "response_format": "text", + "tools": "", + "tool_choice": "", + "parallel_tool_calls": true, + "top_p": 1, + "stop": "", + "presence_penalty": 0, + "frequency_penalty": 0, + "function_call": "", + "functions": "" }, "key": "fead74ce-615a-4812-869d-9735d9d985de", "model": "gpt-3.5-turbo", - "name": "old-gpt3.5", + "name": "no-tools-gpt3.5", "settings": { - "frequency_penalty": 0, - "presence_penalty": 0, + "system_msg": "You are a helpful assistant.", "temperature": 1, - "top_p": 1 + "response_format": { + "type": "text" + }, + "tools": [], + "tool_choice": "", + "parallel_tool_calls": true, + "top_p": 1, + "stop": [], + "presence_penalty": 0, + "frequency_penalty": 0, + "function_call": "", + "functions": [] }, "temp": 1, "progress": { - "success": 0, + "success": 100, "error": 0 } }, - "batch_id": "1e884f42-d130-4a33-9039-22490772902d", + "uid": "53b45f6f-12fe-4d2a-998b-4fc199c28c1d", "metavars": { "__pt": "{prompt}", - "LLM_0": "old-gpt3.5" + "LLM_0": "no-tools-gpt3.5" } }, { - "text": "Thunderstorms are caused by the rapid upward movement of warm, moist air, which then cools and condenses, forming cumulonimbus clouds. These clouds contain large amounts of water vapor, ice, and electrically charged particles. As the air continues to rise, the water droplets and ice particles within the clouds collide, leading to the development of strong updrafts and downdrafts. The movement of these particles within the cloud generates electrical charges, resulting in the formation of lightning. The rapid expansion and contraction of the air around the lightning bolt creates a shock wave that we hear as thunder.", - "prompt": "What causes thunderstorms?", + "text": "[[TOOLS]] \\{\"name\":\"get_current_weather\",\"input\":\\{\"format\":\"celsius\",\"location\":\"\"\\}\\}", + "prompt": "Who directed the movie Twister?", "fill_history": { - "prompt": "What causes thunderstorms?" + "prompt": "Who directed the movie Twister?" }, "llm": { - "base_model": "gpt-3.5-turbo", - "emoji": "🌦️", - "formData": { - "frequency_penalty": 0, - "function_call": "auto", - "functions": "[\n {\n \"name\": \"get_current_weather\",\n \"description\": \"Get the current weather\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"location\": {\n \"type\": \"string\",\n \"description\": \"The city and state, e.g. San Francisco, CA\"\n },\n \"format\": {\n \"type\": \"string\",\n \"enum\": [\"celsius\", \"fahrenheit\"],\n \"description\": \"The temperature unit to use. Infer this from the users location.\"\n }\n },\n \"required\": [\"location\", \"format\"]\n }\n }\n]", - "model": "gpt-3.5-turbo-0613", - "presence_penalty": 0, - "shortname": "gpt3.5-turbo-0613", - "stop": "", - "system_msg": "You are a helpful assistant.", - "temperature": 1, - "top_p": 1 - }, - "key": "af7031af-96ae-44b8-bcc4-3538f3174b73", - "model": "gpt-3.5-turbo-0613", - "name": "gpt3.5-turbo-0613", + "key": "4caee194-fa12-4ca8-bb9e-1c8d7d5f4a08", + "name": "Claude", + "emoji": "📚", + "model": "claude-3-5-sonnet-latest", + "base_model": "claude-v1", + "temp": 0.5, "settings": { - "frequency_penalty": 0, - "function_call": "auto", - "functions": [ + "system_msg": "", + "temperature": 1, + "tools": [ { - "description": "Get the current weather", "name": "get_current_weather", - "parameters": { + "description": "Get the current weather", + "input_schema": { + "type": "object", "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA" + }, "format": { - "description": "The temperature unit to use. Infer this from the users location.", + "type": "string", "enum": [ "celsius", "fahrenheit" ], - "type": "string" - }, - "location": { - "description": "The city and state, e.g. San Francisco, CA", - "type": "string" + "description": "The temperature unit to use. Infer this from the users location." } }, "required": [ "location", "format" - ], - "type": "object" + ] } } ], - "presence_penalty": 0, - "temperature": 1, - "top_p": 1 + "tool_choice": { + "type": "auto" + }, + "parallel_tool_calls": true, + "max_tokens_to_sample": 1024, + "custom_prompt_wrapper": "", + "stop_sequences": [ + "\n\nHuman:" + ], + "top_k": -1, + "top_p": -1 + }, + "formData": { + "shortname": "Claude", + "model": "claude-3-5-sonnet-latest", + "system_msg": "", + "temperature": 1, + "tools": "[{\n \"name\": \"get_current_weather\",\n \"description\": \"Get the current weather\",\n \"input_schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"location\": {\n \"type\": \"string\",\n \"description\": \"The city and state, e.g. San Francisco, CA\"\n },\n \"format\": {\n \"type\": \"string\",\n \"enum\": [\"celsius\", \"fahrenheit\"],\n \"description\": \"The temperature unit to use. Infer this from the users location.\"\n }\n },\n \"required\": [\"location\", \"format\"]\n }\n }\n]", + "tool_choice": "auto", + "parallel_tool_calls": true, + "max_tokens_to_sample": 1024, + "custom_prompt_wrapper": "", + "stop_sequences": "\"\n\nHuman:\"", + "top_k": -1, + "top_p": -1 }, - "temp": 1, "progress": { - "success": 0, + "success": 100, "error": 0 } }, - "batch_id": "ba600f1c-ea3c-4c74-b5e5-bf03739d6d1c", + "uid": "19109992-fdef-479d-92fd-7a48c0b4b527", "metavars": { "__pt": "{prompt}", - "LLM_0": "gpt3.5-turbo-0613" + "LLM_0": "Claude" } }, { - "text": "Thunderstorms are caused by the rapid upward movement of warm, moist air, which creates unstable atmospheric conditions. As the warm air rises, it cools and forms cumulonimbus clouds, which are the towering dark clouds that typically come to mind when thinking of a thunderstorm. Within these clouds, a variety of atmospheric conditions exist that can cause the development of lightning, thunder, heavy rain, hail, and tornadoes. The exact cause of a thunderstorm can vary depending on the location, time of year, and other environmental factors.", + "text": "Thunderstorms are caused by the combination of moisture, rapidly rising warm air, and a lifting mechanism. Here is a breakdown of the factors and processes involved:\n\n1. **Moisture**: Moisture is necessary to form clouds and rain. It comes primarily from bodies of water like oceans and lakes. Warmer air can hold more moisture than cooler air.\n\n2. **Rapidly Rising Warm Air**: When the surface air is warmed by the sun, it becomes lighter and rises quickly. As the air rises, it cools, and the moisture it carries condenses into a cumulus cloud. If the rising continues and conditions are favorable, these clouds can grow into cumulonimbus clouds, which are the towering clouds typically associated with thunderstorms.\n\n3. **Lifting Mechanism**: A lifting mechanism is needed to help or force the warm air to rise. This can be caused by a variety of geographical and meteorological factors:\n - Frontal systems (where cold and warm air masses collide)\n - Sea breezes\n - Mountains that force the air to rise (orographic lift)\n - Variations in temperature (thermal differences) on the surface caused by features such as urban settings, forests, and crops.\n\nOnce these elements are in place, a thunderstorm can develop. Within the storm, electrical charges become separated, with positive charges often accumulating near the top of the storm and negative charges near the bottom. This charge separation can eventually lead to lightning, which is the sudden neutralization of these charges. The rapid expansion of air heated by the lightning causes thunder.\n\nThroughout a thunderstorm, various types of precipitation (rain, hail) and wind phenomena (gusts, downdrafts) can occur, depending on the intensity and maturity of the storm.", + "prompt": "What causes thunderstorms?", + "fill_history": { + "prompt": "What causes thunderstorms?" + }, + "llm": { + "key": "1725e2ec-cd4b-4211-a73a-3d6ef4482451", + "name": "gpt-4-turbo", + "emoji": "🤖", + "model": "gpt-4-turbo", + "base_model": "gpt-3.5-turbo", + "temp": 1, + "settings": { + "system_msg": "You are a helpful assistant.", + "temperature": 1, + "response_format": { + "type": "text" + }, + "tools": [ + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA" + }, + "format": { + "type": "string", + "enum": [ + "celsius", + "fahrenheit" + ], + "description": "The temperature unit to use. Infer this from the users location." + } + }, + "required": [ + "location", + "format" + ] + } + } + } + ], + "tool_choice": "auto", + "parallel_tool_calls": true, + "top_p": 1, + "stop": [], + "presence_penalty": 0, + "frequency_penalty": 0 + }, + "formData": { + "shortname": "gpt-4-turbo", + "model": "gpt-4-turbo", + "system_msg": "You are a helpful assistant.", + "temperature": 1, + "response_format": "text", + "tools": "[{ \"type\": \"function\",\n \"function\": {\n \"name\": \"get_current_weather\",\n \"description\": \"Get the current weather\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"location\": {\n \"type\": \"string\",\n \"description\": \"The city and state, e.g. San Francisco, CA\"\n },\n \"format\": {\n \"type\": \"string\",\n \"enum\": [\"celsius\", \"fahrenheit\"],\n \"description\": \"The temperature unit to use. Infer this from the users location.\"\n }\n },\n \"required\": [\"location\", \"format\"]\n }\n }\n}]", + "tool_choice": "auto", + "parallel_tool_calls": true, + "top_p": 1, + "stop": "", + "presence_penalty": 0, + "frequency_penalty": 0 + }, + "progress": { + "success": 80, + "error": 0 + } + }, + "uid": "0068affe-862a-4864-b4ef-376913a78d4a", + "metavars": { + "__pt": "{prompt}", + "LLM_0": "gpt-4-turbo" + } + }, + { + "text": "Thunderstorms are caused by the rapid upward movement of warm, moist air that cools and condenses into clouds. This process leads to the formation of cumulonimbus clouds, which can grow to great heights and create unstable atmospheric conditions. As the clouds continue to grow, water droplets and ice particles collide and create an electrical charge. The difference in charge between the clouds and the ground or between different parts of the cloud leads to the discharge of lightning, which is then followed by the sound of thunder. This entire process of rapid uplift, condensation, and electrical discharge is what we experience as a thunderstorm.", "prompt": "What causes thunderstorms?", "fill_history": { "prompt": "What causes thunderstorms?" @@ -310,110 +531,215 @@ "base_model": "gpt-3.5-turbo", "emoji": "🙂", "formData": { - "frequency_penalty": 0, - "function_call": "", - "functions": "", + "shortname": "no-tools-gpt3.5", "model": "gpt-3.5-turbo", - "presence_penalty": 0, - "shortname": "old-gpt3.5", - "stop": "", "system_msg": "You are a helpful assistant.", "temperature": 1, - "top_p": 1 + "response_format": "text", + "tools": "", + "tool_choice": "", + "parallel_tool_calls": true, + "top_p": 1, + "stop": "", + "presence_penalty": 0, + "frequency_penalty": 0, + "function_call": "", + "functions": "" }, "key": "fead74ce-615a-4812-869d-9735d9d985de", "model": "gpt-3.5-turbo", - "name": "old-gpt3.5", + "name": "no-tools-gpt3.5", "settings": { - "frequency_penalty": 0, - "presence_penalty": 0, + "system_msg": "You are a helpful assistant.", "temperature": 1, - "top_p": 1 + "response_format": { + "type": "text" + }, + "tools": [], + "tool_choice": "", + "parallel_tool_calls": true, + "top_p": 1, + "stop": [], + "presence_penalty": 0, + "frequency_penalty": 0, + "function_call": "", + "functions": [] }, "temp": 1, "progress": { - "success": 0, + "success": 100, "error": 0 } }, - "batch_id": "8c2e8fb8-76dc-455d-8ba7-252f99594b36", + "uid": "6056ad0a-a1c6-4c21-9788-f1c9143f6f85", "metavars": { "__pt": "{prompt}", - "LLM_0": "old-gpt3.5" + "LLM_0": "no-tools-gpt3.5" } }, { - "text": "[[FUNCTION]] get_current_weather\\{\n \"format\": \"celsius\",\n \"location\": \"Kenya\"\n\\}", - "prompt": "What is the typical temperature in Kenya in June?", + "text": "[[TOOLS]] \\{\"name\":\"get_current_weather\",\"input\":\\{\"format\":\"celsius\",\"location\":\"\"\\}\\}", + "prompt": "What causes thunderstorms?", "fill_history": { - "prompt": "What is the typical temperature in Kenya in June?" + "prompt": "What causes thunderstorms?" }, "llm": { - "base_model": "gpt-3.5-turbo", - "emoji": "🌦️", - "formData": { - "frequency_penalty": 0, - "function_call": "auto", - "functions": "[\n {\n \"name\": \"get_current_weather\",\n \"description\": \"Get the current weather\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"location\": {\n \"type\": \"string\",\n \"description\": \"The city and state, e.g. San Francisco, CA\"\n },\n \"format\": {\n \"type\": \"string\",\n \"enum\": [\"celsius\", \"fahrenheit\"],\n \"description\": \"The temperature unit to use. Infer this from the users location.\"\n }\n },\n \"required\": [\"location\", \"format\"]\n }\n }\n]", - "model": "gpt-3.5-turbo-0613", - "presence_penalty": 0, - "shortname": "gpt3.5-turbo-0613", - "stop": "", - "system_msg": "You are a helpful assistant.", - "temperature": 1, - "top_p": 1 - }, - "key": "af7031af-96ae-44b8-bcc4-3538f3174b73", - "model": "gpt-3.5-turbo-0613", - "name": "gpt3.5-turbo-0613", + "key": "4caee194-fa12-4ca8-bb9e-1c8d7d5f4a08", + "name": "Claude", + "emoji": "📚", + "model": "claude-3-5-sonnet-latest", + "base_model": "claude-v1", + "temp": 0.5, "settings": { - "frequency_penalty": 0, - "function_call": "auto", - "functions": [ + "system_msg": "", + "temperature": 1, + "tools": [ { - "description": "Get the current weather", "name": "get_current_weather", - "parameters": { + "description": "Get the current weather", + "input_schema": { + "type": "object", "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA" + }, "format": { - "description": "The temperature unit to use. Infer this from the users location.", + "type": "string", "enum": [ "celsius", "fahrenheit" ], - "type": "string" - }, - "location": { - "description": "The city and state, e.g. San Francisco, CA", - "type": "string" + "description": "The temperature unit to use. Infer this from the users location." } }, "required": [ "location", "format" - ], - "type": "object" + ] } } ], - "presence_penalty": 0, - "temperature": 1, - "top_p": 1 + "tool_choice": { + "type": "auto" + }, + "parallel_tool_calls": true, + "max_tokens_to_sample": 1024, + "custom_prompt_wrapper": "", + "stop_sequences": [ + "\n\nHuman:" + ], + "top_k": -1, + "top_p": -1 + }, + "formData": { + "shortname": "Claude", + "model": "claude-3-5-sonnet-latest", + "system_msg": "", + "temperature": 1, + "tools": "[{\n \"name\": \"get_current_weather\",\n \"description\": \"Get the current weather\",\n \"input_schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"location\": {\n \"type\": \"string\",\n \"description\": \"The city and state, e.g. San Francisco, CA\"\n },\n \"format\": {\n \"type\": \"string\",\n \"enum\": [\"celsius\", \"fahrenheit\"],\n \"description\": \"The temperature unit to use. Infer this from the users location.\"\n }\n },\n \"required\": [\"location\", \"format\"]\n }\n }\n]", + "tool_choice": "auto", + "parallel_tool_calls": true, + "max_tokens_to_sample": 1024, + "custom_prompt_wrapper": "", + "stop_sequences": "\"\n\nHuman:\"", + "top_k": -1, + "top_p": -1 }, - "temp": 1, "progress": { - "success": 0, + "success": 100, "error": 0 } }, - "batch_id": "2a8ff935-4c25-4222-8d37-22634b8c6208", + "uid": "cd5e5b7f-7cf6-4ab3-91c1-ce9ab6052d07", "metavars": { "__pt": "{prompt}", - "LLM_0": "gpt3.5-turbo-0613" + "LLM_0": "Claude" } }, { - "text": "The typical temperature in Kenya in June varies depending on the region, but generally, temperatures range from 19°C (66°F) to 30°C (86°F). In some parts of the country, particularly in areas of higher elevation, temperatures may be cooler, while in areas closer to the equator temperatures can be quite hot.", + "text": "In June, Kenya typically experiences variable temperatures depending on the region. Due to its diverse geography, which includes coastal areas, highlands, and savannahs, temperatures can differ significantly:\n\n1. **Coastal areas (e.g., Mombasa):** The temperatures are warmer with highs around 28°C (82°F) and lows about 20°C (68°F).\n \n2. **Highlands (e.g., Nairobi):** Cooler temperatures prevail, with daytime highs averaging 21°C (70°F) and nights around 12°C (54°F).\n\n3. **Western and Lake Victoria region:** Warm and humid with temperatures similar to the coastal areas but can be a bit cooler especially during the night.\n\nJune marks the start of the cooler season in Kenya, making it a pleasant time to visit, especially for those who prefer milder weather away from the heat and humidity typical of tropical climates.", + "prompt": "What is the typical temperature in Kenya in June?", + "fill_history": { + "prompt": "What is the typical temperature in Kenya in June?" + }, + "llm": { + "key": "1725e2ec-cd4b-4211-a73a-3d6ef4482451", + "name": "gpt-4-turbo", + "emoji": "🤖", + "model": "gpt-4-turbo", + "base_model": "gpt-3.5-turbo", + "temp": 1, + "settings": { + "system_msg": "You are a helpful assistant.", + "temperature": 1, + "response_format": { + "type": "text" + }, + "tools": [ + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA" + }, + "format": { + "type": "string", + "enum": [ + "celsius", + "fahrenheit" + ], + "description": "The temperature unit to use. Infer this from the users location." + } + }, + "required": [ + "location", + "format" + ] + } + } + } + ], + "tool_choice": "auto", + "parallel_tool_calls": true, + "top_p": 1, + "stop": [], + "presence_penalty": 0, + "frequency_penalty": 0 + }, + "formData": { + "shortname": "gpt-4-turbo", + "model": "gpt-4-turbo", + "system_msg": "You are a helpful assistant.", + "temperature": 1, + "response_format": "text", + "tools": "[{ \"type\": \"function\",\n \"function\": {\n \"name\": \"get_current_weather\",\n \"description\": \"Get the current weather\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"location\": {\n \"type\": \"string\",\n \"description\": \"The city and state, e.g. San Francisco, CA\"\n },\n \"format\": {\n \"type\": \"string\",\n \"enum\": [\"celsius\", \"fahrenheit\"],\n \"description\": \"The temperature unit to use. Infer this from the users location.\"\n }\n },\n \"required\": [\"location\", \"format\"]\n }\n }\n}]", + "tool_choice": "auto", + "parallel_tool_calls": true, + "top_p": 1, + "stop": "", + "presence_penalty": 0, + "frequency_penalty": 0 + }, + "progress": { + "success": 80, + "error": 0 + } + }, + "uid": "b3e061b4-b81d-4904-97b3-5627d4e64824", + "metavars": { + "__pt": "{prompt}", + "LLM_0": "gpt-4-turbo" + } + }, + { + "text": "In Kenya, June is during the dry season, with temperatures averaging around 18-23°C (64-73°F) in the central highlands and around 24-30°C (75-86°F) along the coast and in lower elevations. However, temperatures can vary depending on the specific region and altitude within the country.", "prompt": "What is the typical temperature in Kenya in June?", "fill_history": { "prompt": "What is the typical temperature in Kenya in June?" @@ -422,110 +748,215 @@ "base_model": "gpt-3.5-turbo", "emoji": "🙂", "formData": { - "frequency_penalty": 0, - "function_call": "", - "functions": "", + "shortname": "no-tools-gpt3.5", "model": "gpt-3.5-turbo", - "presence_penalty": 0, - "shortname": "old-gpt3.5", - "stop": "", "system_msg": "You are a helpful assistant.", "temperature": 1, - "top_p": 1 + "response_format": "text", + "tools": "", + "tool_choice": "", + "parallel_tool_calls": true, + "top_p": 1, + "stop": "", + "presence_penalty": 0, + "frequency_penalty": 0, + "function_call": "", + "functions": "" }, "key": "fead74ce-615a-4812-869d-9735d9d985de", "model": "gpt-3.5-turbo", - "name": "old-gpt3.5", + "name": "no-tools-gpt3.5", "settings": { - "frequency_penalty": 0, - "presence_penalty": 0, + "system_msg": "You are a helpful assistant.", "temperature": 1, - "top_p": 1 + "response_format": { + "type": "text" + }, + "tools": [], + "tool_choice": "", + "parallel_tool_calls": true, + "top_p": 1, + "stop": [], + "presence_penalty": 0, + "frequency_penalty": 0, + "function_call": "", + "functions": [] }, "temp": 1, "progress": { - "success": 0, + "success": 100, "error": 0 } }, - "batch_id": "a993b28a-df28-462e-9e2b-e7afd6742824", + "uid": "7f84b658-031a-4bc8-9e2f-37e1db9b21e0", "metavars": { "__pt": "{prompt}", - "LLM_0": "old-gpt3.5" + "LLM_0": "no-tools-gpt3.5" } }, { - "text": "[[FUNCTION]] get_current_weather\\{\n \"format\": \"celsius\",\n \"location\": \"Honolulu, HI\"\n\\}", - "prompt": "What is the average temperature in Hawaii in December?", + "text": "[[TOOLS]] \\{\"name\":\"get_current_weather\",\"input\":\\{\"format\":\"celsius\",\"location\":\"Nairobi, Kenya\"\\}\\}", + "prompt": "What is the typical temperature in Kenya in June?", "fill_history": { - "prompt": "What is the average temperature in Hawaii in December?" + "prompt": "What is the typical temperature in Kenya in June?" }, "llm": { - "base_model": "gpt-3.5-turbo", - "emoji": "🌦️", - "formData": { - "frequency_penalty": 0, - "function_call": "auto", - "functions": "[\n {\n \"name\": \"get_current_weather\",\n \"description\": \"Get the current weather\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"location\": {\n \"type\": \"string\",\n \"description\": \"The city and state, e.g. San Francisco, CA\"\n },\n \"format\": {\n \"type\": \"string\",\n \"enum\": [\"celsius\", \"fahrenheit\"],\n \"description\": \"The temperature unit to use. Infer this from the users location.\"\n }\n },\n \"required\": [\"location\", \"format\"]\n }\n }\n]", - "model": "gpt-3.5-turbo-0613", - "presence_penalty": 0, - "shortname": "gpt3.5-turbo-0613", - "stop": "", - "system_msg": "You are a helpful assistant.", - "temperature": 1, - "top_p": 1 - }, - "key": "af7031af-96ae-44b8-bcc4-3538f3174b73", - "model": "gpt-3.5-turbo-0613", - "name": "gpt3.5-turbo-0613", + "key": "4caee194-fa12-4ca8-bb9e-1c8d7d5f4a08", + "name": "Claude", + "emoji": "📚", + "model": "claude-3-5-sonnet-latest", + "base_model": "claude-v1", + "temp": 0.5, "settings": { - "frequency_penalty": 0, - "function_call": "auto", - "functions": [ + "system_msg": "", + "temperature": 1, + "tools": [ { - "description": "Get the current weather", "name": "get_current_weather", - "parameters": { + "description": "Get the current weather", + "input_schema": { + "type": "object", "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA" + }, "format": { - "description": "The temperature unit to use. Infer this from the users location.", + "type": "string", "enum": [ "celsius", "fahrenheit" ], - "type": "string" - }, - "location": { - "description": "The city and state, e.g. San Francisco, CA", - "type": "string" + "description": "The temperature unit to use. Infer this from the users location." } }, "required": [ "location", "format" - ], - "type": "object" + ] } } ], - "presence_penalty": 0, - "temperature": 1, - "top_p": 1 + "tool_choice": { + "type": "auto" + }, + "parallel_tool_calls": true, + "max_tokens_to_sample": 1024, + "custom_prompt_wrapper": "", + "stop_sequences": [ + "\n\nHuman:" + ], + "top_k": -1, + "top_p": -1 + }, + "formData": { + "shortname": "Claude", + "model": "claude-3-5-sonnet-latest", + "system_msg": "", + "temperature": 1, + "tools": "[{\n \"name\": \"get_current_weather\",\n \"description\": \"Get the current weather\",\n \"input_schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"location\": {\n \"type\": \"string\",\n \"description\": \"The city and state, e.g. San Francisco, CA\"\n },\n \"format\": {\n \"type\": \"string\",\n \"enum\": [\"celsius\", \"fahrenheit\"],\n \"description\": \"The temperature unit to use. Infer this from the users location.\"\n }\n },\n \"required\": [\"location\", \"format\"]\n }\n }\n]", + "tool_choice": "auto", + "parallel_tool_calls": true, + "max_tokens_to_sample": 1024, + "custom_prompt_wrapper": "", + "stop_sequences": "\"\n\nHuman:\"", + "top_k": -1, + "top_p": -1 }, - "temp": 1, "progress": { - "success": 0, + "success": 100, "error": 0 } }, - "batch_id": "4b818464-a297-429e-b081-19c61b12bae7", + "uid": "eac6af65-6e4c-40ac-a8ad-b07e13c36ac5", "metavars": { "__pt": "{prompt}", - "LLM_0": "gpt3.5-turbo-0613" + "LLM_0": "Claude" } }, { - "text": "The average temperature in Hawaii in December is around 77°F (25°C) during the day and 66°F (19°C) at night.", + "text": "The average temperature in Hawaii in December typically ranges from about 68°F (20°C) to 81°F (27°C). This offers pleasant, warm weather which is why Hawaii is such a popular winter destination. Local variations across the different islands can, of course, result in slightly different temperatures due to geography and elevation.", + "prompt": "What is the average temperature in Hawaii in December?", + "fill_history": { + "prompt": "What is the average temperature in Hawaii in December?" + }, + "llm": { + "key": "1725e2ec-cd4b-4211-a73a-3d6ef4482451", + "name": "gpt-4-turbo", + "emoji": "🤖", + "model": "gpt-4-turbo", + "base_model": "gpt-3.5-turbo", + "temp": 1, + "settings": { + "system_msg": "You are a helpful assistant.", + "temperature": 1, + "response_format": { + "type": "text" + }, + "tools": [ + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA" + }, + "format": { + "type": "string", + "enum": [ + "celsius", + "fahrenheit" + ], + "description": "The temperature unit to use. Infer this from the users location." + } + }, + "required": [ + "location", + "format" + ] + } + } + } + ], + "tool_choice": "auto", + "parallel_tool_calls": true, + "top_p": 1, + "stop": [], + "presence_penalty": 0, + "frequency_penalty": 0 + }, + "formData": { + "shortname": "gpt-4-turbo", + "model": "gpt-4-turbo", + "system_msg": "You are a helpful assistant.", + "temperature": 1, + "response_format": "text", + "tools": "[{ \"type\": \"function\",\n \"function\": {\n \"name\": \"get_current_weather\",\n \"description\": \"Get the current weather\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"location\": {\n \"type\": \"string\",\n \"description\": \"The city and state, e.g. San Francisco, CA\"\n },\n \"format\": {\n \"type\": \"string\",\n \"enum\": [\"celsius\", \"fahrenheit\"],\n \"description\": \"The temperature unit to use. Infer this from the users location.\"\n }\n },\n \"required\": [\"location\", \"format\"]\n }\n }\n}]", + "tool_choice": "auto", + "parallel_tool_calls": true, + "top_p": 1, + "stop": "", + "presence_penalty": 0, + "frequency_penalty": 0 + }, + "progress": { + "success": 80, + "error": 0 + } + }, + "uid": "de3fd081-62b5-4943-bd4d-36987a43004d", + "metavars": { + "__pt": "{prompt}", + "LLM_0": "gpt-4-turbo" + } + }, + { + "text": "In December, the average temperature in Hawaii typically ranges from 75°F to 80°F (24°C to 27°C). However, the exact temperature can vary depending on the specific island and location within Hawaii.", "prompt": "What is the average temperature in Hawaii in December?", "fill_history": { "prompt": "What is the average temperature in Hawaii in December?" @@ -534,119 +965,302 @@ "base_model": "gpt-3.5-turbo", "emoji": "🙂", "formData": { - "frequency_penalty": 0, - "function_call": "", - "functions": "", + "shortname": "no-tools-gpt3.5", "model": "gpt-3.5-turbo", - "presence_penalty": 0, - "shortname": "old-gpt3.5", - "stop": "", "system_msg": "You are a helpful assistant.", "temperature": 1, - "top_p": 1 + "response_format": "text", + "tools": "", + "tool_choice": "", + "parallel_tool_calls": true, + "top_p": 1, + "stop": "", + "presence_penalty": 0, + "frequency_penalty": 0, + "function_call": "", + "functions": "" }, "key": "fead74ce-615a-4812-869d-9735d9d985de", "model": "gpt-3.5-turbo", - "name": "old-gpt3.5", + "name": "no-tools-gpt3.5", "settings": { - "frequency_penalty": 0, - "presence_penalty": 0, + "system_msg": "You are a helpful assistant.", "temperature": 1, - "top_p": 1 + "response_format": { + "type": "text" + }, + "tools": [], + "tool_choice": "", + "parallel_tool_calls": true, + "top_p": 1, + "stop": [], + "presence_penalty": 0, + "frequency_penalty": 0, + "function_call": "", + "functions": [] }, "temp": 1, "progress": { - "success": 0, + "success": 100, "error": 0 } }, - "batch_id": "426b0d32-7afd-46a8-a9db-b0e893e7e3b7", + "uid": "f3aa7139-def5-4397-8b87-ee075ffe315f", "metavars": { "__pt": "{prompt}", - "LLM_0": "old-gpt3.5" + "LLM_0": "no-tools-gpt3.5" + } + }, + { + "text": "[[TOOLS]] \\{\"name\":\"get_current_weather\",\"input\":\\{\"format\":\"fahrenheit\",\"location\":\"Honolulu, HI\"\\}\\}", + "prompt": "What is the average temperature in Hawaii in December?", + "fill_history": { + "prompt": "What is the average temperature in Hawaii in December?" + }, + "llm": { + "key": "4caee194-fa12-4ca8-bb9e-1c8d7d5f4a08", + "name": "Claude", + "emoji": "📚", + "model": "claude-3-5-sonnet-latest", + "base_model": "claude-v1", + "temp": 0.5, + "settings": { + "system_msg": "", + "temperature": 1, + "tools": [ + { + "name": "get_current_weather", + "description": "Get the current weather", + "input_schema": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA" + }, + "format": { + "type": "string", + "enum": [ + "celsius", + "fahrenheit" + ], + "description": "The temperature unit to use. Infer this from the users location." + } + }, + "required": [ + "location", + "format" + ] + } + } + ], + "tool_choice": { + "type": "auto" + }, + "parallel_tool_calls": true, + "max_tokens_to_sample": 1024, + "custom_prompt_wrapper": "", + "stop_sequences": [ + "\n\nHuman:" + ], + "top_k": -1, + "top_p": -1 + }, + "formData": { + "shortname": "Claude", + "model": "claude-3-5-sonnet-latest", + "system_msg": "", + "temperature": 1, + "tools": "[{\n \"name\": \"get_current_weather\",\n \"description\": \"Get the current weather\",\n \"input_schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"location\": {\n \"type\": \"string\",\n \"description\": \"The city and state, e.g. San Francisco, CA\"\n },\n \"format\": {\n \"type\": \"string\",\n \"enum\": [\"celsius\", \"fahrenheit\"],\n \"description\": \"The temperature unit to use. Infer this from the users location.\"\n }\n },\n \"required\": [\"location\", \"format\"]\n }\n }\n]", + "tool_choice": "auto", + "parallel_tool_calls": true, + "max_tokens_to_sample": 1024, + "custom_prompt_wrapper": "", + "stop_sequences": "\"\n\nHuman:\"", + "top_k": -1, + "top_p": -1 + }, + "progress": { + "success": 100, + "error": 0 + } + }, + "uid": "58de1cb7-3514-4ecc-ae42-6e0900b91ff5", + "metavars": { + "__pt": "{prompt}", + "LLM_0": "Claude" } } ], "llms": [ { + "key": "1725e2ec-cd4b-4211-a73a-3d6ef4482451", + "name": "gpt-4-turbo", + "emoji": "🤖", + "model": "gpt-4-turbo", "base_model": "gpt-3.5-turbo", - "emoji": "🌦️", - "formData": { - "frequency_penalty": 0, - "function_call": "auto", - "functions": "[\n {\n \"name\": \"get_current_weather\",\n \"description\": \"Get the current weather\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"location\": {\n \"type\": \"string\",\n \"description\": \"The city and state, e.g. San Francisco, CA\"\n },\n \"format\": {\n \"type\": \"string\",\n \"enum\": [\"celsius\", \"fahrenheit\"],\n \"description\": \"The temperature unit to use. Infer this from the users location.\"\n }\n },\n \"required\": [\"location\", \"format\"]\n }\n }\n]", - "model": "gpt-3.5-turbo-0613", - "presence_penalty": 0, - "shortname": "gpt3.5-turbo-0613", - "stop": "", + "temp": 1, + "settings": { "system_msg": "You are a helpful assistant.", "temperature": 1, - "top_p": 1 - }, - "key": "af7031af-96ae-44b8-bcc4-3538f3174b73", - "model": "gpt-3.5-turbo-0613", - "name": "gpt3.5-turbo-0613", - "settings": { - "frequency_penalty": 0, - "function_call": "auto", - "functions": [ + "response_format": { + "type": "text" + }, + "tools": [ { - "description": "Get the current weather", - "name": "get_current_weather", - "parameters": { - "properties": { - "format": { - "description": "The temperature unit to use. Infer this from the users location.", - "enum": [ - "celsius", - "fahrenheit" - ], - "type": "string" + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA" + }, + "format": { + "type": "string", + "enum": [ + "celsius", + "fahrenheit" + ], + "description": "The temperature unit to use. Infer this from the users location." + } }, - "location": { - "description": "The city and state, e.g. San Francisco, CA", - "type": "string" - } - }, - "required": [ - "location", - "format" - ], - "type": "object" + "required": [ + "location", + "format" + ] + } } } ], + "tool_choice": "auto", + "parallel_tool_calls": true, + "top_p": 1, + "stop": [], "presence_penalty": 0, - "temperature": 1, - "top_p": 1 + "frequency_penalty": 0 }, - "temp": 1 + "formData": { + "shortname": "gpt-4-turbo", + "model": "gpt-4-turbo", + "system_msg": "You are a helpful assistant.", + "temperature": 1, + "response_format": "text", + "tools": "[{ \"type\": \"function\",\n \"function\": {\n \"name\": \"get_current_weather\",\n \"description\": \"Get the current weather\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"location\": {\n \"type\": \"string\",\n \"description\": \"The city and state, e.g. San Francisco, CA\"\n },\n \"format\": {\n \"type\": \"string\",\n \"enum\": [\"celsius\", \"fahrenheit\"],\n \"description\": \"The temperature unit to use. Infer this from the users location.\"\n }\n },\n \"required\": [\"location\", \"format\"]\n }\n }\n}]", + "tool_choice": "auto", + "parallel_tool_calls": true, + "top_p": 1, + "stop": "", + "presence_penalty": 0, + "frequency_penalty": 0 + } }, { "base_model": "gpt-3.5-turbo", "emoji": "🙂", "formData": { - "frequency_penalty": 0, - "function_call": "", - "functions": "", + "shortname": "no-tools-gpt3.5", "model": "gpt-3.5-turbo", - "presence_penalty": 0, - "shortname": "old-gpt3.5", - "stop": "", "system_msg": "You are a helpful assistant.", "temperature": 1, - "top_p": 1 + "response_format": "text", + "tools": "", + "tool_choice": "", + "parallel_tool_calls": true, + "top_p": 1, + "stop": "", + "presence_penalty": 0, + "frequency_penalty": 0, + "function_call": "", + "functions": "" }, "key": "fead74ce-615a-4812-869d-9735d9d985de", "model": "gpt-3.5-turbo", - "name": "old-gpt3.5", + "name": "no-tools-gpt3.5", "settings": { - "frequency_penalty": 0, - "presence_penalty": 0, + "system_msg": "You are a helpful assistant.", "temperature": 1, - "top_p": 1 + "response_format": { + "type": "text" + }, + "tools": [], + "tool_choice": "", + "parallel_tool_calls": true, + "top_p": 1, + "stop": [], + "presence_penalty": 0, + "frequency_penalty": 0, + "function_call": "", + "functions": [] }, "temp": 1 + }, + { + "key": "4caee194-fa12-4ca8-bb9e-1c8d7d5f4a08", + "name": "Claude", + "emoji": "📚", + "model": "claude-3-5-sonnet-latest", + "base_model": "claude-v1", + "temp": 0.5, + "settings": { + "system_msg": "", + "temperature": 1, + "tools": [ + { + "name": "get_current_weather", + "description": "Get the current weather", + "input_schema": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA" + }, + "format": { + "type": "string", + "enum": [ + "celsius", + "fahrenheit" + ], + "description": "The temperature unit to use. Infer this from the users location." + } + }, + "required": [ + "location", + "format" + ] + } + } + ], + "tool_choice": { + "type": "auto" + }, + "parallel_tool_calls": true, + "max_tokens_to_sample": 1024, + "custom_prompt_wrapper": "", + "stop_sequences": [ + "\n\nHuman:" + ], + "top_k": -1, + "top_p": -1 + }, + "formData": { + "shortname": "Claude", + "model": "claude-3-5-sonnet-latest", + "system_msg": "", + "temperature": 1, + "tools": "[{\n \"name\": \"get_current_weather\",\n \"description\": \"Get the current weather\",\n \"input_schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"location\": {\n \"type\": \"string\",\n \"description\": \"The city and state, e.g. San Francisco, CA\"\n },\n \"format\": {\n \"type\": \"string\",\n \"enum\": [\"celsius\", \"fahrenheit\"],\n \"description\": \"The temperature unit to use. Infer this from the users location.\"\n }\n },\n \"required\": [\"location\", \"format\"]\n }\n }\n]", + "tool_choice": "auto", + "parallel_tool_calls": true, + "max_tokens_to_sample": 1024, + "custom_prompt_wrapper": "", + "stop_sequences": "\"\n\nHuman:\"", + "top_k": -1, + "top_p": -1 + } } ], "n": 1, @@ -666,40 +1280,27 @@ "type": "prompt" }, { - "width": 722, - "height": 557, + "width": 649, + "height": 533, "data": { "input": "evalNode-1688060142717", "llm_groups": [ { "value": "LLM", - "label": "LLMs (last)" - }, - { - "value": "__pt", - "label": "LLMs #NaN" - }, - { - "value": "LLM_0", - "label": "LLMs #1" + "label": "LLM" } ], "refresh": false, "selected_vars": [], "title": "Whether the prompt triggered the function call 'get_current_weather'", "vars": [ - "LLM (default)", + { + "value": "LLM (default)", + "label": "LLM (default)" + }, { "value": "prompt", "label": "prompt" - }, - { - "value": "__meta___pt", - "label": "__pt (meta)" - }, - { - "value": "__meta_LLM_0", - "label": "LLM_0 (meta)" } ] }, @@ -717,8 +1318,8 @@ "type": "vis" }, { - "width": 302, - "height": 319, + "width": 344, + "height": 494, "data": { "input": "prompt-1686689549227", "refresh": false @@ -752,12 +1353,12 @@ "dragging": false, "id": "textFieldsNode-1686690525665", "position": { - "x": 112, - "y": 224 + "x": 80, + "y": 160 }, "positionAbsolute": { - "x": 112, - "y": 224 + "x": 80, + "y": 160 }, "selected": false, "type": "textfields" @@ -766,119 +1367,224 @@ "width": 359, "height": 221, "data": { - "code": "function evaluate(response) {\n return response.text.startsWith(\n \"[[FUNCTION]]\"\n );\n}", + "code": "function evaluate(response) {\n return response.text.startsWith(\n \"[[TOOLS]]\"\n );\n}", "fields": [ { - "text": "[[FUNCTION]] get_current_weather\\\\{\n \"format\": \"celsius\",\n \"location\": \"Boston, MA\"\n\\\\}", + "text": "[[TOOLS]] get_current_weather \\{\"location\":\"Boston, MA\",\"format\":\"fahrenheit\"\\}", "prompt": "What's the weather like in Boston?", "fill_history": { "prompt": "What's the weather like in Boston?" }, "metavars": { "__pt": "{prompt}", - "LLM_0": "gpt3.5-turbo-0613" + "LLM_0": "gpt-4-turbo" }, "llm": { + "key": "1725e2ec-cd4b-4211-a73a-3d6ef4482451", + "name": "gpt-4-turbo", + "emoji": "🤖", + "model": "gpt-4-turbo", "base_model": "gpt-3.5-turbo", - "emoji": "🌦️", - "formData": { - "frequency_penalty": 0, - "function_call": "auto", - "functions": "[\n {\n \"name\": \"get_current_weather\",\n \"description\": \"Get the current weather\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"location\": {\n \"type\": \"string\",\n \"description\": \"The city and state, e.g. San Francisco, CA\"\n },\n \"format\": {\n \"type\": \"string\",\n \"enum\": [\"celsius\", \"fahrenheit\"],\n \"description\": \"The temperature unit to use. Infer this from the users location.\"\n }\n },\n \"required\": [\"location\", \"format\"]\n }\n }\n]", - "model": "gpt-3.5-turbo-0613", - "presence_penalty": 0, - "shortname": "gpt3.5-turbo-0613", - "stop": "", + "temp": 1, + "settings": { "system_msg": "You are a helpful assistant.", "temperature": 1, - "top_p": 1 - }, - "key": "af7031af-96ae-44b8-bcc4-3538f3174b73", - "model": "gpt-3.5-turbo-0613", - "name": "gpt3.5-turbo-0613", - "settings": { - "frequency_penalty": 0, - "function_call": "auto", - "functions": [ + "response_format": { + "type": "text" + }, + "tools": [ { - "description": "Get the current weather", - "name": "get_current_weather", - "parameters": { - "properties": { - "format": { - "description": "The temperature unit to use. Infer this from the users location.", - "enum": [ - "celsius", - "fahrenheit" - ], - "type": "string" + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA" + }, + "format": { + "type": "string", + "enum": [ + "celsius", + "fahrenheit" + ], + "description": "The temperature unit to use. Infer this from the users location." + } }, - "location": { - "description": "The city and state, e.g. San Francisco, CA", - "type": "string" - } - }, - "required": [ - "location", - "format" - ], - "type": "object" + "required": [ + "location", + "format" + ] + } } } ], + "tool_choice": "auto", + "parallel_tool_calls": true, + "top_p": 1, + "stop": [], "presence_penalty": 0, - "temperature": 1, - "top_p": 1 + "frequency_penalty": 0 + }, + "formData": { + "shortname": "gpt-4-turbo", + "model": "gpt-4-turbo", + "system_msg": "You are a helpful assistant.", + "temperature": 1, + "response_format": "text", + "tools": "[{ \"type\": \"function\",\n \"function\": {\n \"name\": \"get_current_weather\",\n \"description\": \"Get the current weather\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"location\": {\n \"type\": \"string\",\n \"description\": \"The city and state, e.g. San Francisco, CA\"\n },\n \"format\": {\n \"type\": \"string\",\n \"enum\": [\"celsius\", \"fahrenheit\"],\n \"description\": \"The temperature unit to use. Infer this from the users location.\"\n }\n },\n \"required\": [\"location\", \"format\"]\n }\n }\n}]", + "tool_choice": "auto", + "parallel_tool_calls": true, + "top_p": 1, + "stop": "", + "presence_penalty": 0, + "frequency_penalty": 0 }, - "temp": 1, "progress": { - "success": 0, + "success": 80, "error": 0 } }, - "batch_id": "cfa8f743-32c7-4a24-8141-9e1fa264af49" + "uid": "2e2d2d3f-d162-4f6a-af6a-23f2628748e5" }, { - "text": "I'm sorry, I cannot provide real-time information as I do not have access to live data. However, you can check reliable weather websites or applications for up-to-date information about the weather in Boston.", + "text": "I'm sorry, but I don't have real-time data access. You can check the weather in Boston by using a weather website or app, or by asking a voice assistant like Siri or Google Assistant for the current weather in Boston.", "prompt": "What's the weather like in Boston?", "fill_history": { "prompt": "What's the weather like in Boston?" }, "metavars": { "__pt": "{prompt}", - "LLM_0": "old-gpt3.5" + "LLM_0": "no-tools-gpt3.5" }, "llm": { "base_model": "gpt-3.5-turbo", "emoji": "🙂", "formData": { - "frequency_penalty": 0, - "function_call": "", - "functions": "", + "shortname": "no-tools-gpt3.5", "model": "gpt-3.5-turbo", - "presence_penalty": 0, - "shortname": "old-gpt3.5", - "stop": "", "system_msg": "You are a helpful assistant.", "temperature": 1, - "top_p": 1 + "response_format": "text", + "tools": "", + "tool_choice": "", + "parallel_tool_calls": true, + "top_p": 1, + "stop": "", + "presence_penalty": 0, + "frequency_penalty": 0, + "function_call": "", + "functions": "" }, "key": "fead74ce-615a-4812-869d-9735d9d985de", "model": "gpt-3.5-turbo", - "name": "old-gpt3.5", + "name": "no-tools-gpt3.5", "settings": { - "frequency_penalty": 0, - "presence_penalty": 0, + "system_msg": "You are a helpful assistant.", "temperature": 1, - "top_p": 1 + "response_format": { + "type": "text" + }, + "tools": [], + "tool_choice": "", + "parallel_tool_calls": true, + "top_p": 1, + "stop": [], + "presence_penalty": 0, + "frequency_penalty": 0, + "function_call": "", + "functions": [] }, "temp": 1, "progress": { - "success": 0, + "success": 100, "error": 0 } }, - "batch_id": "e342daaf-acde-4988-a759-c1c753efd670" + "uid": "64f90d18-939b-4e59-867e-5c433edd5103" + }, + { + "text": "[[TOOLS]] \\{\"name\":\"get_current_weather\",\"input\":\\{\"format\":\"fahrenheit\",\"location\":\"Boston, MA\"\\}\\}", + "prompt": "What's the weather like in Boston?", + "fill_history": { + "prompt": "What's the weather like in Boston?" + }, + "metavars": { + "__pt": "{prompt}", + "LLM_0": "Claude" + }, + "llm": { + "key": "4caee194-fa12-4ca8-bb9e-1c8d7d5f4a08", + "name": "Claude", + "emoji": "📚", + "model": "claude-3-5-sonnet-latest", + "base_model": "claude-v1", + "temp": 0.5, + "settings": { + "system_msg": "", + "temperature": 1, + "tools": [ + { + "name": "get_current_weather", + "description": "Get the current weather", + "input_schema": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA" + }, + "format": { + "type": "string", + "enum": [ + "celsius", + "fahrenheit" + ], + "description": "The temperature unit to use. Infer this from the users location." + } + }, + "required": [ + "location", + "format" + ] + } + } + ], + "tool_choice": { + "type": "auto" + }, + "parallel_tool_calls": true, + "max_tokens_to_sample": 1024, + "custom_prompt_wrapper": "", + "stop_sequences": [ + "\n\nHuman:" + ], + "top_k": -1, + "top_p": -1 + }, + "formData": { + "shortname": "Claude", + "model": "claude-3-5-sonnet-latest", + "system_msg": "", + "temperature": 1, + "tools": "[{\n \"name\": \"get_current_weather\",\n \"description\": \"Get the current weather\",\n \"input_schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"location\": {\n \"type\": \"string\",\n \"description\": \"The city and state, e.g. San Francisco, CA\"\n },\n \"format\": {\n \"type\": \"string\",\n \"enum\": [\"celsius\", \"fahrenheit\"],\n \"description\": \"The temperature unit to use. Infer this from the users location.\"\n }\n },\n \"required\": [\"location\", \"format\"]\n }\n }\n]", + "tool_choice": "auto", + "parallel_tool_calls": true, + "max_tokens_to_sample": 1024, + "custom_prompt_wrapper": "", + "stop_sequences": "\"\n\nHuman:\"", + "top_k": -1, + "top_p": -1 + }, + "progress": { + "success": 100, + "error": 0 + } + }, + "uid": "eb2ba7da-fc44-4b3e-9cf9-6ee684e84ac6" }, { "text": "The movie \"Twister\" was directed by Jan de Bont.", @@ -888,445 +1594,865 @@ }, "metavars": { "__pt": "{prompt}", - "LLM_0": "gpt3.5-turbo-0613" + "LLM_0": "gpt-4-turbo" }, "llm": { + "key": "1725e2ec-cd4b-4211-a73a-3d6ef4482451", + "name": "gpt-4-turbo", + "emoji": "🤖", + "model": "gpt-4-turbo", "base_model": "gpt-3.5-turbo", - "emoji": "🌦️", - "formData": { - "frequency_penalty": 0, - "function_call": "auto", - "functions": "[\n {\n \"name\": \"get_current_weather\",\n \"description\": \"Get the current weather\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"location\": {\n \"type\": \"string\",\n \"description\": \"The city and state, e.g. San Francisco, CA\"\n },\n \"format\": {\n \"type\": \"string\",\n \"enum\": [\"celsius\", \"fahrenheit\"],\n \"description\": \"The temperature unit to use. Infer this from the users location.\"\n }\n },\n \"required\": [\"location\", \"format\"]\n }\n }\n]", - "model": "gpt-3.5-turbo-0613", - "presence_penalty": 0, - "shortname": "gpt3.5-turbo-0613", - "stop": "", + "temp": 1, + "settings": { "system_msg": "You are a helpful assistant.", "temperature": 1, - "top_p": 1 - }, - "key": "af7031af-96ae-44b8-bcc4-3538f3174b73", - "model": "gpt-3.5-turbo-0613", - "name": "gpt3.5-turbo-0613", - "settings": { - "frequency_penalty": 0, - "function_call": "auto", - "functions": [ + "response_format": { + "type": "text" + }, + "tools": [ { - "description": "Get the current weather", - "name": "get_current_weather", - "parameters": { - "properties": { - "format": { - "description": "The temperature unit to use. Infer this from the users location.", - "enum": [ - "celsius", - "fahrenheit" - ], - "type": "string" + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA" + }, + "format": { + "type": "string", + "enum": [ + "celsius", + "fahrenheit" + ], + "description": "The temperature unit to use. Infer this from the users location." + } }, - "location": { - "description": "The city and state, e.g. San Francisco, CA", - "type": "string" - } - }, - "required": [ - "location", - "format" - ], - "type": "object" + "required": [ + "location", + "format" + ] + } } } ], + "tool_choice": "auto", + "parallel_tool_calls": true, + "top_p": 1, + "stop": [], "presence_penalty": 0, - "temperature": 1, - "top_p": 1 + "frequency_penalty": 0 + }, + "formData": { + "shortname": "gpt-4-turbo", + "model": "gpt-4-turbo", + "system_msg": "You are a helpful assistant.", + "temperature": 1, + "response_format": "text", + "tools": "[{ \"type\": \"function\",\n \"function\": {\n \"name\": \"get_current_weather\",\n \"description\": \"Get the current weather\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"location\": {\n \"type\": \"string\",\n \"description\": \"The city and state, e.g. San Francisco, CA\"\n },\n \"format\": {\n \"type\": \"string\",\n \"enum\": [\"celsius\", \"fahrenheit\"],\n \"description\": \"The temperature unit to use. Infer this from the users location.\"\n }\n },\n \"required\": [\"location\", \"format\"]\n }\n }\n}]", + "tool_choice": "auto", + "parallel_tool_calls": true, + "top_p": 1, + "stop": "", + "presence_penalty": 0, + "frequency_penalty": 0 }, - "temp": 1, "progress": { - "success": 0, + "success": 80, "error": 0 } }, - "batch_id": "ebcba322-b11c-4680-8170-98b0f3bebbe5" + "uid": "423a06d4-1111-48a8-9897-5348ea362ebc" }, { - "text": "The movie \"Twister\" was directed by Jan de Bont.", + "text": "The movie Twister was directed by Jan de Bont. It was released in 1996 and stars Helen Hunt and Bill Paxton.", "prompt": "Who directed the movie Twister?", "fill_history": { "prompt": "Who directed the movie Twister?" }, "metavars": { "__pt": "{prompt}", - "LLM_0": "old-gpt3.5" + "LLM_0": "no-tools-gpt3.5" }, "llm": { "base_model": "gpt-3.5-turbo", "emoji": "🙂", "formData": { - "frequency_penalty": 0, - "function_call": "", - "functions": "", + "shortname": "no-tools-gpt3.5", "model": "gpt-3.5-turbo", - "presence_penalty": 0, - "shortname": "old-gpt3.5", - "stop": "", "system_msg": "You are a helpful assistant.", "temperature": 1, - "top_p": 1 + "response_format": "text", + "tools": "", + "tool_choice": "", + "parallel_tool_calls": true, + "top_p": 1, + "stop": "", + "presence_penalty": 0, + "frequency_penalty": 0, + "function_call": "", + "functions": "" }, "key": "fead74ce-615a-4812-869d-9735d9d985de", "model": "gpt-3.5-turbo", - "name": "old-gpt3.5", + "name": "no-tools-gpt3.5", "settings": { - "frequency_penalty": 0, - "presence_penalty": 0, + "system_msg": "You are a helpful assistant.", "temperature": 1, - "top_p": 1 + "response_format": { + "type": "text" + }, + "tools": [], + "tool_choice": "", + "parallel_tool_calls": true, + "top_p": 1, + "stop": [], + "presence_penalty": 0, + "frequency_penalty": 0, + "function_call": "", + "functions": [] }, "temp": 1, "progress": { - "success": 0, + "success": 100, "error": 0 } }, - "batch_id": "1e884f42-d130-4a33-9039-22490772902d" + "uid": "53b45f6f-12fe-4d2a-998b-4fc199c28c1d" }, { - "text": "Thunderstorms are caused by the rapid upward movement of warm, moist air, which then cools and condenses, forming cumulonimbus clouds. These clouds contain large amounts of water vapor, ice, and electrically charged particles. As the air continues to rise, the water droplets and ice particles within the clouds collide, leading to the development of strong updrafts and downdrafts. The movement of these particles within the cloud generates electrical charges, resulting in the formation of lightning. The rapid expansion and contraction of the air around the lightning bolt creates a shock wave that we hear as thunder.", + "text": "[[TOOLS]] \\{\"name\":\"get_current_weather\",\"input\":\\{\"format\":\"celsius\",\"location\":\"\"\\}\\}", + "prompt": "Who directed the movie Twister?", + "fill_history": { + "prompt": "Who directed the movie Twister?" + }, + "metavars": { + "__pt": "{prompt}", + "LLM_0": "Claude" + }, + "llm": { + "key": "4caee194-fa12-4ca8-bb9e-1c8d7d5f4a08", + "name": "Claude", + "emoji": "📚", + "model": "claude-3-5-sonnet-latest", + "base_model": "claude-v1", + "temp": 0.5, + "settings": { + "system_msg": "", + "temperature": 1, + "tools": [ + { + "name": "get_current_weather", + "description": "Get the current weather", + "input_schema": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA" + }, + "format": { + "type": "string", + "enum": [ + "celsius", + "fahrenheit" + ], + "description": "The temperature unit to use. Infer this from the users location." + } + }, + "required": [ + "location", + "format" + ] + } + } + ], + "tool_choice": { + "type": "auto" + }, + "parallel_tool_calls": true, + "max_tokens_to_sample": 1024, + "custom_prompt_wrapper": "", + "stop_sequences": [ + "\n\nHuman:" + ], + "top_k": -1, + "top_p": -1 + }, + "formData": { + "shortname": "Claude", + "model": "claude-3-5-sonnet-latest", + "system_msg": "", + "temperature": 1, + "tools": "[{\n \"name\": \"get_current_weather\",\n \"description\": \"Get the current weather\",\n \"input_schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"location\": {\n \"type\": \"string\",\n \"description\": \"The city and state, e.g. San Francisco, CA\"\n },\n \"format\": {\n \"type\": \"string\",\n \"enum\": [\"celsius\", \"fahrenheit\"],\n \"description\": \"The temperature unit to use. Infer this from the users location.\"\n }\n },\n \"required\": [\"location\", \"format\"]\n }\n }\n]", + "tool_choice": "auto", + "parallel_tool_calls": true, + "max_tokens_to_sample": 1024, + "custom_prompt_wrapper": "", + "stop_sequences": "\"\n\nHuman:\"", + "top_k": -1, + "top_p": -1 + }, + "progress": { + "success": 100, + "error": 0 + } + }, + "uid": "19109992-fdef-479d-92fd-7a48c0b4b527" + }, + { + "text": "Thunderstorms are caused by the combination of moisture, rapidly rising warm air, and a lifting mechanism. Here is a breakdown of the factors and processes involved:\n\n1. **Moisture**: Moisture is necessary to form clouds and rain. It comes primarily from bodies of water like oceans and lakes. Warmer air can hold more moisture than cooler air.\n\n2. **Rapidly Rising Warm Air**: When the surface air is warmed by the sun, it becomes lighter and rises quickly. As the air rises, it cools, and the moisture it carries condenses into a cumulus cloud. If the rising continues and conditions are favorable, these clouds can grow into cumulonimbus clouds, which are the towering clouds typically associated with thunderstorms.\n\n3. **Lifting Mechanism**: A lifting mechanism is needed to help or force the warm air to rise. This can be caused by a variety of geographical and meteorological factors:\n - Frontal systems (where cold and warm air masses collide)\n - Sea breezes\n - Mountains that force the air to rise (orographic lift)\n - Variations in temperature (thermal differences) on the surface caused by features such as urban settings, forests, and crops.\n\nOnce these elements are in place, a thunderstorm can develop. Within the storm, electrical charges become separated, with positive charges often accumulating near the top of the storm and negative charges near the bottom. This charge separation can eventually lead to lightning, which is the sudden neutralization of these charges. The rapid expansion of air heated by the lightning causes thunder.\n\nThroughout a thunderstorm, various types of precipitation (rain, hail) and wind phenomena (gusts, downdrafts) can occur, depending on the intensity and maturity of the storm.", "prompt": "What causes thunderstorms?", "fill_history": { "prompt": "What causes thunderstorms?" }, "metavars": { "__pt": "{prompt}", - "LLM_0": "gpt3.5-turbo-0613" + "LLM_0": "gpt-4-turbo" }, "llm": { + "key": "1725e2ec-cd4b-4211-a73a-3d6ef4482451", + "name": "gpt-4-turbo", + "emoji": "🤖", + "model": "gpt-4-turbo", "base_model": "gpt-3.5-turbo", - "emoji": "🌦️", - "formData": { - "frequency_penalty": 0, - "function_call": "auto", - "functions": "[\n {\n \"name\": \"get_current_weather\",\n \"description\": \"Get the current weather\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"location\": {\n \"type\": \"string\",\n \"description\": \"The city and state, e.g. San Francisco, CA\"\n },\n \"format\": {\n \"type\": \"string\",\n \"enum\": [\"celsius\", \"fahrenheit\"],\n \"description\": \"The temperature unit to use. Infer this from the users location.\"\n }\n },\n \"required\": [\"location\", \"format\"]\n }\n }\n]", - "model": "gpt-3.5-turbo-0613", - "presence_penalty": 0, - "shortname": "gpt3.5-turbo-0613", - "stop": "", + "temp": 1, + "settings": { "system_msg": "You are a helpful assistant.", "temperature": 1, - "top_p": 1 - }, - "key": "af7031af-96ae-44b8-bcc4-3538f3174b73", - "model": "gpt-3.5-turbo-0613", - "name": "gpt3.5-turbo-0613", - "settings": { - "frequency_penalty": 0, - "function_call": "auto", - "functions": [ + "response_format": { + "type": "text" + }, + "tools": [ { - "description": "Get the current weather", - "name": "get_current_weather", - "parameters": { - "properties": { - "format": { - "description": "The temperature unit to use. Infer this from the users location.", - "enum": [ - "celsius", - "fahrenheit" - ], - "type": "string" + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA" + }, + "format": { + "type": "string", + "enum": [ + "celsius", + "fahrenheit" + ], + "description": "The temperature unit to use. Infer this from the users location." + } }, - "location": { - "description": "The city and state, e.g. San Francisco, CA", - "type": "string" - } - }, - "required": [ - "location", - "format" - ], - "type": "object" + "required": [ + "location", + "format" + ] + } } } ], + "tool_choice": "auto", + "parallel_tool_calls": true, + "top_p": 1, + "stop": [], "presence_penalty": 0, - "temperature": 1, - "top_p": 1 + "frequency_penalty": 0 + }, + "formData": { + "shortname": "gpt-4-turbo", + "model": "gpt-4-turbo", + "system_msg": "You are a helpful assistant.", + "temperature": 1, + "response_format": "text", + "tools": "[{ \"type\": \"function\",\n \"function\": {\n \"name\": \"get_current_weather\",\n \"description\": \"Get the current weather\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"location\": {\n \"type\": \"string\",\n \"description\": \"The city and state, e.g. San Francisco, CA\"\n },\n \"format\": {\n \"type\": \"string\",\n \"enum\": [\"celsius\", \"fahrenheit\"],\n \"description\": \"The temperature unit to use. Infer this from the users location.\"\n }\n },\n \"required\": [\"location\", \"format\"]\n }\n }\n}]", + "tool_choice": "auto", + "parallel_tool_calls": true, + "top_p": 1, + "stop": "", + "presence_penalty": 0, + "frequency_penalty": 0 }, - "temp": 1, "progress": { - "success": 0, + "success": 80, "error": 0 } }, - "batch_id": "ba600f1c-ea3c-4c74-b5e5-bf03739d6d1c" + "uid": "0068affe-862a-4864-b4ef-376913a78d4a" }, { - "text": "Thunderstorms are caused by the rapid upward movement of warm, moist air, which creates unstable atmospheric conditions. As the warm air rises, it cools and forms cumulonimbus clouds, which are the towering dark clouds that typically come to mind when thinking of a thunderstorm. Within these clouds, a variety of atmospheric conditions exist that can cause the development of lightning, thunder, heavy rain, hail, and tornadoes. The exact cause of a thunderstorm can vary depending on the location, time of year, and other environmental factors.", + "text": "Thunderstorms are caused by the rapid upward movement of warm, moist air that cools and condenses into clouds. This process leads to the formation of cumulonimbus clouds, which can grow to great heights and create unstable atmospheric conditions. As the clouds continue to grow, water droplets and ice particles collide and create an electrical charge. The difference in charge between the clouds and the ground or between different parts of the cloud leads to the discharge of lightning, which is then followed by the sound of thunder. This entire process of rapid uplift, condensation, and electrical discharge is what we experience as a thunderstorm.", "prompt": "What causes thunderstorms?", "fill_history": { "prompt": "What causes thunderstorms?" }, "metavars": { "__pt": "{prompt}", - "LLM_0": "old-gpt3.5" + "LLM_0": "no-tools-gpt3.5" }, "llm": { "base_model": "gpt-3.5-turbo", "emoji": "🙂", "formData": { - "frequency_penalty": 0, - "function_call": "", - "functions": "", + "shortname": "no-tools-gpt3.5", "model": "gpt-3.5-turbo", - "presence_penalty": 0, - "shortname": "old-gpt3.5", - "stop": "", "system_msg": "You are a helpful assistant.", "temperature": 1, - "top_p": 1 + "response_format": "text", + "tools": "", + "tool_choice": "", + "parallel_tool_calls": true, + "top_p": 1, + "stop": "", + "presence_penalty": 0, + "frequency_penalty": 0, + "function_call": "", + "functions": "" }, "key": "fead74ce-615a-4812-869d-9735d9d985de", "model": "gpt-3.5-turbo", - "name": "old-gpt3.5", + "name": "no-tools-gpt3.5", "settings": { - "frequency_penalty": 0, - "presence_penalty": 0, + "system_msg": "You are a helpful assistant.", "temperature": 1, - "top_p": 1 + "response_format": { + "type": "text" + }, + "tools": [], + "tool_choice": "", + "parallel_tool_calls": true, + "top_p": 1, + "stop": [], + "presence_penalty": 0, + "frequency_penalty": 0, + "function_call": "", + "functions": [] }, "temp": 1, "progress": { - "success": 0, + "success": 100, "error": 0 } }, - "batch_id": "8c2e8fb8-76dc-455d-8ba7-252f99594b36" + "uid": "6056ad0a-a1c6-4c21-9788-f1c9143f6f85" }, { - "text": "[[FUNCTION]] get_current_weather\\\\{\n \"format\": \"celsius\",\n \"location\": \"Kenya\"\n\\\\}", + "text": "[[TOOLS]] \\{\"name\":\"get_current_weather\",\"input\":\\{\"format\":\"celsius\",\"location\":\"\"\\}\\}", + "prompt": "What causes thunderstorms?", + "fill_history": { + "prompt": "What causes thunderstorms?" + }, + "metavars": { + "__pt": "{prompt}", + "LLM_0": "Claude" + }, + "llm": { + "key": "4caee194-fa12-4ca8-bb9e-1c8d7d5f4a08", + "name": "Claude", + "emoji": "📚", + "model": "claude-3-5-sonnet-latest", + "base_model": "claude-v1", + "temp": 0.5, + "settings": { + "system_msg": "", + "temperature": 1, + "tools": [ + { + "name": "get_current_weather", + "description": "Get the current weather", + "input_schema": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA" + }, + "format": { + "type": "string", + "enum": [ + "celsius", + "fahrenheit" + ], + "description": "The temperature unit to use. Infer this from the users location." + } + }, + "required": [ + "location", + "format" + ] + } + } + ], + "tool_choice": { + "type": "auto" + }, + "parallel_tool_calls": true, + "max_tokens_to_sample": 1024, + "custom_prompt_wrapper": "", + "stop_sequences": [ + "\n\nHuman:" + ], + "top_k": -1, + "top_p": -1 + }, + "formData": { + "shortname": "Claude", + "model": "claude-3-5-sonnet-latest", + "system_msg": "", + "temperature": 1, + "tools": "[{\n \"name\": \"get_current_weather\",\n \"description\": \"Get the current weather\",\n \"input_schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"location\": {\n \"type\": \"string\",\n \"description\": \"The city and state, e.g. San Francisco, CA\"\n },\n \"format\": {\n \"type\": \"string\",\n \"enum\": [\"celsius\", \"fahrenheit\"],\n \"description\": \"The temperature unit to use. Infer this from the users location.\"\n }\n },\n \"required\": [\"location\", \"format\"]\n }\n }\n]", + "tool_choice": "auto", + "parallel_tool_calls": true, + "max_tokens_to_sample": 1024, + "custom_prompt_wrapper": "", + "stop_sequences": "\"\n\nHuman:\"", + "top_k": -1, + "top_p": -1 + }, + "progress": { + "success": 100, + "error": 0 + } + }, + "uid": "cd5e5b7f-7cf6-4ab3-91c1-ce9ab6052d07" + }, + { + "text": "In June, Kenya typically experiences variable temperatures depending on the region. Due to its diverse geography, which includes coastal areas, highlands, and savannahs, temperatures can differ significantly:\n\n1. **Coastal areas (e.g., Mombasa):** The temperatures are warmer with highs around 28°C (82°F) and lows about 20°C (68°F).\n \n2. **Highlands (e.g., Nairobi):** Cooler temperatures prevail, with daytime highs averaging 21°C (70°F) and nights around 12°C (54°F).\n\n3. **Western and Lake Victoria region:** Warm and humid with temperatures similar to the coastal areas but can be a bit cooler especially during the night.\n\nJune marks the start of the cooler season in Kenya, making it a pleasant time to visit, especially for those who prefer milder weather away from the heat and humidity typical of tropical climates.", "prompt": "What is the typical temperature in Kenya in June?", "fill_history": { "prompt": "What is the typical temperature in Kenya in June?" }, "metavars": { "__pt": "{prompt}", - "LLM_0": "gpt3.5-turbo-0613" + "LLM_0": "gpt-4-turbo" }, "llm": { + "key": "1725e2ec-cd4b-4211-a73a-3d6ef4482451", + "name": "gpt-4-turbo", + "emoji": "🤖", + "model": "gpt-4-turbo", "base_model": "gpt-3.5-turbo", - "emoji": "🌦️", - "formData": { - "frequency_penalty": 0, - "function_call": "auto", - "functions": "[\n {\n \"name\": \"get_current_weather\",\n \"description\": \"Get the current weather\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"location\": {\n \"type\": \"string\",\n \"description\": \"The city and state, e.g. San Francisco, CA\"\n },\n \"format\": {\n \"type\": \"string\",\n \"enum\": [\"celsius\", \"fahrenheit\"],\n \"description\": \"The temperature unit to use. Infer this from the users location.\"\n }\n },\n \"required\": [\"location\", \"format\"]\n }\n }\n]", - "model": "gpt-3.5-turbo-0613", - "presence_penalty": 0, - "shortname": "gpt3.5-turbo-0613", - "stop": "", + "temp": 1, + "settings": { "system_msg": "You are a helpful assistant.", "temperature": 1, - "top_p": 1 - }, - "key": "af7031af-96ae-44b8-bcc4-3538f3174b73", - "model": "gpt-3.5-turbo-0613", - "name": "gpt3.5-turbo-0613", - "settings": { - "frequency_penalty": 0, - "function_call": "auto", - "functions": [ + "response_format": { + "type": "text" + }, + "tools": [ { - "description": "Get the current weather", - "name": "get_current_weather", - "parameters": { - "properties": { - "format": { - "description": "The temperature unit to use. Infer this from the users location.", - "enum": [ - "celsius", - "fahrenheit" - ], - "type": "string" + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA" + }, + "format": { + "type": "string", + "enum": [ + "celsius", + "fahrenheit" + ], + "description": "The temperature unit to use. Infer this from the users location." + } }, - "location": { - "description": "The city and state, e.g. San Francisco, CA", - "type": "string" - } - }, - "required": [ - "location", - "format" - ], - "type": "object" + "required": [ + "location", + "format" + ] + } } } ], + "tool_choice": "auto", + "parallel_tool_calls": true, + "top_p": 1, + "stop": [], "presence_penalty": 0, - "temperature": 1, - "top_p": 1 + "frequency_penalty": 0 + }, + "formData": { + "shortname": "gpt-4-turbo", + "model": "gpt-4-turbo", + "system_msg": "You are a helpful assistant.", + "temperature": 1, + "response_format": "text", + "tools": "[{ \"type\": \"function\",\n \"function\": {\n \"name\": \"get_current_weather\",\n \"description\": \"Get the current weather\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"location\": {\n \"type\": \"string\",\n \"description\": \"The city and state, e.g. San Francisco, CA\"\n },\n \"format\": {\n \"type\": \"string\",\n \"enum\": [\"celsius\", \"fahrenheit\"],\n \"description\": \"The temperature unit to use. Infer this from the users location.\"\n }\n },\n \"required\": [\"location\", \"format\"]\n }\n }\n}]", + "tool_choice": "auto", + "parallel_tool_calls": true, + "top_p": 1, + "stop": "", + "presence_penalty": 0, + "frequency_penalty": 0 }, - "temp": 1, "progress": { - "success": 0, + "success": 80, "error": 0 } }, - "batch_id": "2a8ff935-4c25-4222-8d37-22634b8c6208" + "uid": "b3e061b4-b81d-4904-97b3-5627d4e64824" }, { - "text": "The typical temperature in Kenya in June varies depending on the region, but generally, temperatures range from 19°C (66°F) to 30°C (86°F). In some parts of the country, particularly in areas of higher elevation, temperatures may be cooler, while in areas closer to the equator temperatures can be quite hot.", + "text": "In Kenya, June is during the dry season, with temperatures averaging around 18-23°C (64-73°F) in the central highlands and around 24-30°C (75-86°F) along the coast and in lower elevations. However, temperatures can vary depending on the specific region and altitude within the country.", "prompt": "What is the typical temperature in Kenya in June?", "fill_history": { "prompt": "What is the typical temperature in Kenya in June?" }, "metavars": { "__pt": "{prompt}", - "LLM_0": "old-gpt3.5" + "LLM_0": "no-tools-gpt3.5" }, "llm": { "base_model": "gpt-3.5-turbo", "emoji": "🙂", "formData": { - "frequency_penalty": 0, - "function_call": "", - "functions": "", + "shortname": "no-tools-gpt3.5", "model": "gpt-3.5-turbo", - "presence_penalty": 0, - "shortname": "old-gpt3.5", - "stop": "", "system_msg": "You are a helpful assistant.", "temperature": 1, - "top_p": 1 + "response_format": "text", + "tools": "", + "tool_choice": "", + "parallel_tool_calls": true, + "top_p": 1, + "stop": "", + "presence_penalty": 0, + "frequency_penalty": 0, + "function_call": "", + "functions": "" }, "key": "fead74ce-615a-4812-869d-9735d9d985de", "model": "gpt-3.5-turbo", - "name": "old-gpt3.5", + "name": "no-tools-gpt3.5", "settings": { - "frequency_penalty": 0, - "presence_penalty": 0, + "system_msg": "You are a helpful assistant.", "temperature": 1, - "top_p": 1 + "response_format": { + "type": "text" + }, + "tools": [], + "tool_choice": "", + "parallel_tool_calls": true, + "top_p": 1, + "stop": [], + "presence_penalty": 0, + "frequency_penalty": 0, + "function_call": "", + "functions": [] }, "temp": 1, "progress": { - "success": 0, + "success": 100, "error": 0 } }, - "batch_id": "a993b28a-df28-462e-9e2b-e7afd6742824" + "uid": "7f84b658-031a-4bc8-9e2f-37e1db9b21e0" }, { - "text": "[[FUNCTION]] get_current_weather\\\\{\n \"format\": \"celsius\",\n \"location\": \"Honolulu, HI\"\n\\\\}", - "prompt": "What is the average temperature in Hawaii in December?", + "text": "[[TOOLS]] \\{\"name\":\"get_current_weather\",\"input\":\\{\"format\":\"celsius\",\"location\":\"Nairobi, Kenya\"\\}\\}", + "prompt": "What is the typical temperature in Kenya in June?", "fill_history": { - "prompt": "What is the average temperature in Hawaii in December?" + "prompt": "What is the typical temperature in Kenya in June?" }, "metavars": { "__pt": "{prompt}", - "LLM_0": "gpt3.5-turbo-0613" + "LLM_0": "Claude" }, "llm": { - "base_model": "gpt-3.5-turbo", - "emoji": "🌦️", - "formData": { - "frequency_penalty": 0, - "function_call": "auto", - "functions": "[\n {\n \"name\": \"get_current_weather\",\n \"description\": \"Get the current weather\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"location\": {\n \"type\": \"string\",\n \"description\": \"The city and state, e.g. San Francisco, CA\"\n },\n \"format\": {\n \"type\": \"string\",\n \"enum\": [\"celsius\", \"fahrenheit\"],\n \"description\": \"The temperature unit to use. Infer this from the users location.\"\n }\n },\n \"required\": [\"location\", \"format\"]\n }\n }\n]", - "model": "gpt-3.5-turbo-0613", - "presence_penalty": 0, - "shortname": "gpt3.5-turbo-0613", - "stop": "", - "system_msg": "You are a helpful assistant.", - "temperature": 1, - "top_p": 1 - }, - "key": "af7031af-96ae-44b8-bcc4-3538f3174b73", - "model": "gpt-3.5-turbo-0613", - "name": "gpt3.5-turbo-0613", + "key": "4caee194-fa12-4ca8-bb9e-1c8d7d5f4a08", + "name": "Claude", + "emoji": "📚", + "model": "claude-3-5-sonnet-latest", + "base_model": "claude-v1", + "temp": 0.5, "settings": { - "frequency_penalty": 0, - "function_call": "auto", - "functions": [ + "system_msg": "", + "temperature": 1, + "tools": [ { - "description": "Get the current weather", "name": "get_current_weather", - "parameters": { + "description": "Get the current weather", + "input_schema": { + "type": "object", "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA" + }, "format": { - "description": "The temperature unit to use. Infer this from the users location.", + "type": "string", "enum": [ "celsius", "fahrenheit" ], - "type": "string" - }, - "location": { - "description": "The city and state, e.g. San Francisco, CA", - "type": "string" + "description": "The temperature unit to use. Infer this from the users location." } }, "required": [ "location", "format" - ], - "type": "object" + ] } } ], - "presence_penalty": 0, - "temperature": 1, - "top_p": 1 + "tool_choice": { + "type": "auto" + }, + "parallel_tool_calls": true, + "max_tokens_to_sample": 1024, + "custom_prompt_wrapper": "", + "stop_sequences": [ + "\n\nHuman:" + ], + "top_k": -1, + "top_p": -1 + }, + "formData": { + "shortname": "Claude", + "model": "claude-3-5-sonnet-latest", + "system_msg": "", + "temperature": 1, + "tools": "[{\n \"name\": \"get_current_weather\",\n \"description\": \"Get the current weather\",\n \"input_schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"location\": {\n \"type\": \"string\",\n \"description\": \"The city and state, e.g. San Francisco, CA\"\n },\n \"format\": {\n \"type\": \"string\",\n \"enum\": [\"celsius\", \"fahrenheit\"],\n \"description\": \"The temperature unit to use. Infer this from the users location.\"\n }\n },\n \"required\": [\"location\", \"format\"]\n }\n }\n]", + "tool_choice": "auto", + "parallel_tool_calls": true, + "max_tokens_to_sample": 1024, + "custom_prompt_wrapper": "", + "stop_sequences": "\"\n\nHuman:\"", + "top_k": -1, + "top_p": -1 }, - "temp": 1, "progress": { - "success": 0, + "success": 100, "error": 0 } }, - "batch_id": "4b818464-a297-429e-b081-19c61b12bae7" + "uid": "eac6af65-6e4c-40ac-a8ad-b07e13c36ac5" }, { - "text": "The average temperature in Hawaii in December is around 77°F (25°C) during the day and 66°F (19°C) at night.", + "text": "The average temperature in Hawaii in December typically ranges from about 68°F (20°C) to 81°F (27°C). This offers pleasant, warm weather which is why Hawaii is such a popular winter destination. Local variations across the different islands can, of course, result in slightly different temperatures due to geography and elevation.", "prompt": "What is the average temperature in Hawaii in December?", "fill_history": { "prompt": "What is the average temperature in Hawaii in December?" }, "metavars": { "__pt": "{prompt}", - "LLM_0": "old-gpt3.5" + "LLM_0": "gpt-4-turbo" + }, + "llm": { + "key": "1725e2ec-cd4b-4211-a73a-3d6ef4482451", + "name": "gpt-4-turbo", + "emoji": "🤖", + "model": "gpt-4-turbo", + "base_model": "gpt-3.5-turbo", + "temp": 1, + "settings": { + "system_msg": "You are a helpful assistant.", + "temperature": 1, + "response_format": { + "type": "text" + }, + "tools": [ + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA" + }, + "format": { + "type": "string", + "enum": [ + "celsius", + "fahrenheit" + ], + "description": "The temperature unit to use. Infer this from the users location." + } + }, + "required": [ + "location", + "format" + ] + } + } + } + ], + "tool_choice": "auto", + "parallel_tool_calls": true, + "top_p": 1, + "stop": [], + "presence_penalty": 0, + "frequency_penalty": 0 + }, + "formData": { + "shortname": "gpt-4-turbo", + "model": "gpt-4-turbo", + "system_msg": "You are a helpful assistant.", + "temperature": 1, + "response_format": "text", + "tools": "[{ \"type\": \"function\",\n \"function\": {\n \"name\": \"get_current_weather\",\n \"description\": \"Get the current weather\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"location\": {\n \"type\": \"string\",\n \"description\": \"The city and state, e.g. San Francisco, CA\"\n },\n \"format\": {\n \"type\": \"string\",\n \"enum\": [\"celsius\", \"fahrenheit\"],\n \"description\": \"The temperature unit to use. Infer this from the users location.\"\n }\n },\n \"required\": [\"location\", \"format\"]\n }\n }\n}]", + "tool_choice": "auto", + "parallel_tool_calls": true, + "top_p": 1, + "stop": "", + "presence_penalty": 0, + "frequency_penalty": 0 + }, + "progress": { + "success": 80, + "error": 0 + } + }, + "uid": "de3fd081-62b5-4943-bd4d-36987a43004d" + }, + { + "text": "In December, the average temperature in Hawaii typically ranges from 75°F to 80°F (24°C to 27°C). However, the exact temperature can vary depending on the specific island and location within Hawaii.", + "prompt": "What is the average temperature in Hawaii in December?", + "fill_history": { + "prompt": "What is the average temperature in Hawaii in December?" + }, + "metavars": { + "__pt": "{prompt}", + "LLM_0": "no-tools-gpt3.5" }, "llm": { "base_model": "gpt-3.5-turbo", "emoji": "🙂", "formData": { - "frequency_penalty": 0, - "function_call": "", - "functions": "", + "shortname": "no-tools-gpt3.5", "model": "gpt-3.5-turbo", - "presence_penalty": 0, - "shortname": "old-gpt3.5", - "stop": "", "system_msg": "You are a helpful assistant.", "temperature": 1, - "top_p": 1 + "response_format": "text", + "tools": "", + "tool_choice": "", + "parallel_tool_calls": true, + "top_p": 1, + "stop": "", + "presence_penalty": 0, + "frequency_penalty": 0, + "function_call": "", + "functions": "" }, "key": "fead74ce-615a-4812-869d-9735d9d985de", "model": "gpt-3.5-turbo", - "name": "old-gpt3.5", + "name": "no-tools-gpt3.5", "settings": { - "frequency_penalty": 0, - "presence_penalty": 0, + "system_msg": "You are a helpful assistant.", "temperature": 1, - "top_p": 1 + "response_format": { + "type": "text" + }, + "tools": [], + "tool_choice": "", + "parallel_tool_calls": true, + "top_p": 1, + "stop": [], + "presence_penalty": 0, + "frequency_penalty": 0, + "function_call": "", + "functions": [] }, "temp": 1, "progress": { - "success": 0, + "success": 100, "error": 0 } }, - "batch_id": "426b0d32-7afd-46a8-a9db-b0e893e7e3b7" + "uid": "f3aa7139-def5-4397-8b87-ee075ffe315f" + }, + { + "text": "[[TOOLS]] \\{\"name\":\"get_current_weather\",\"input\":\\{\"format\":\"fahrenheit\",\"location\":\"Honolulu, HI\"\\}\\}", + "prompt": "What is the average temperature in Hawaii in December?", + "fill_history": { + "prompt": "What is the average temperature in Hawaii in December?" + }, + "metavars": { + "__pt": "{prompt}", + "LLM_0": "Claude" + }, + "llm": { + "key": "4caee194-fa12-4ca8-bb9e-1c8d7d5f4a08", + "name": "Claude", + "emoji": "📚", + "model": "claude-3-5-sonnet-latest", + "base_model": "claude-v1", + "temp": 0.5, + "settings": { + "system_msg": "", + "temperature": 1, + "tools": [ + { + "name": "get_current_weather", + "description": "Get the current weather", + "input_schema": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA" + }, + "format": { + "type": "string", + "enum": [ + "celsius", + "fahrenheit" + ], + "description": "The temperature unit to use. Infer this from the users location." + } + }, + "required": [ + "location", + "format" + ] + } + } + ], + "tool_choice": { + "type": "auto" + }, + "parallel_tool_calls": true, + "max_tokens_to_sample": 1024, + "custom_prompt_wrapper": "", + "stop_sequences": [ + "\n\nHuman:" + ], + "top_k": -1, + "top_p": -1 + }, + "formData": { + "shortname": "Claude", + "model": "claude-3-5-sonnet-latest", + "system_msg": "", + "temperature": 1, + "tools": "[{\n \"name\": \"get_current_weather\",\n \"description\": \"Get the current weather\",\n \"input_schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"location\": {\n \"type\": \"string\",\n \"description\": \"The city and state, e.g. San Francisco, CA\"\n },\n \"format\": {\n \"type\": \"string\",\n \"enum\": [\"celsius\", \"fahrenheit\"],\n \"description\": \"The temperature unit to use. Infer this from the users location.\"\n }\n },\n \"required\": [\"location\", \"format\"]\n }\n }\n]", + "tool_choice": "auto", + "parallel_tool_calls": true, + "max_tokens_to_sample": 1024, + "custom_prompt_wrapper": "", + "stop_sequences": "\"\n\nHuman:\"", + "top_k": -1, + "top_p": -1 + }, + "progress": { + "success": 100, + "error": 0 + } + }, + "uid": "58de1cb7-3514-4ecc-ae42-6e0900b91ff5" } ], "language": "javascript", @@ -1336,11 +2462,11 @@ "id": "evalNode-1688060142717", "position": { "x": 832, - "y": 144 + "y": 112 }, "positionAbsolute": { "x": 832, - "y": 144 + "y": 112 }, "selected": true, "type": "evaluator" @@ -1427,894 +2553,1461 @@ } ], "viewport": { - "x": -15, - "y": 81, + "x": -25, + "y": 24, "zoom": 1 } }, "cache": { - "prompt-1686689549227_6.json": { - "What causes thunderstorms?": { - "info": { - "prompt": "What causes thunderstorms?" - }, - "llm": "gpt3.5-turbo-0613", - "metavars": { - "LLM_0": "gpt3.5-turbo-0613" - }, - "prompt": "What causes thunderstorms?", - "query": { - "frequency_penalty": 0, - "function_call": "auto", - "functions": [ - { - "description": "Get the current weather", - "name": "get_current_weather", - "parameters": { - "properties": { - "format": { - "description": "The temperature unit to use. Infer this from the users location.", - "enum": [ - "celsius", - "fahrenheit" - ], - "type": "string" - }, - "location": { - "description": "The city and state, e.g. San Francisco, CA", - "type": "string" - } - }, - "required": [ - "location", - "format" - ], - "type": "object" - } - } - ], - "messages": [ - { - "content": "You are a helpful assistant.", - "role": "system" + "prompt-1686689549227_5.json": { + "Who directed the movie Twister?": [ + { + "prompt": "Who directed the movie Twister?", + "query": { + "model": "gpt-4-turbo", + "n": 1, + "temperature": 1, + "response_format": { + "type": "text" }, - { - "content": "What causes thunderstorms?", - "role": "user" - } - ], - "model": "gpt-3.5-turbo-0613", - "n": 1, - "presence_penalty": 0, - "temperature": 1, - "top_p": 1 - }, - "raw_response": { - "choices": [ - { - "finish_reason": "stop", - "index": 0, - "message": { - "content": "Thunderstorms are caused by the rapid upward movement of warm, moist air, which then cools and condenses, forming cumulonimbus clouds. These clouds contain large amounts of water vapor, ice, and electrically charged particles. As the air continues to rise, the water droplets and ice particles within the clouds collide, leading to the development of strong updrafts and downdrafts. The movement of these particles within the cloud generates electrical charges, resulting in the formation of lightning. The rapid expansion and contraction of the air around the lightning bolt creates a shock wave that we hear as thunder.", - "role": "assistant" - } - } - ], - "created": 1688060198, - "id": "chatcmpl-7WprKCpTroyKv5DTgRFskAovs7pbO", - "model": "gpt-3.5-turbo-0613", - "object": "chat.completion", - "usage": { - "completion_tokens": 123, - "prompt_tokens": 95, - "total_tokens": 218 - } - }, - "responses": [ - "Thunderstorms are caused by the rapid upward movement of warm, moist air, which then cools and condenses, forming cumulonimbus clouds. These clouds contain large amounts of water vapor, ice, and electrically charged particles. As the air continues to rise, the water droplets and ice particles within the clouds collide, leading to the development of strong updrafts and downdrafts. The movement of these particles within the cloud generates electrical charges, resulting in the formation of lightning. The rapid expansion and contraction of the air around the lightning bolt creates a shock wave that we hear as thunder." - ], - "uid": "ba600f1c-ea3c-4c74-b5e5-bf03739d6d1c" - }, - "What is the average temperature in Hawaii in December?": { - "info": { - "prompt": "What is the average temperature in Hawaii in December?" - }, - "llm": "gpt3.5-turbo-0613", - "metavars": { - "LLM_0": "gpt3.5-turbo-0613" - }, - "prompt": "What is the average temperature in Hawaii in December?", - "query": { - "frequency_penalty": 0, - "function_call": "auto", - "functions": [ - { - "description": "Get the current weather", - "name": "get_current_weather", - "parameters": { - "properties": { - "format": { - "description": "The temperature unit to use. Infer this from the users location.", - "enum": [ - "celsius", - "fahrenheit" - ], - "type": "string" - }, - "location": { - "description": "The city and state, e.g. San Francisco, CA", - "type": "string" + "tools": [ + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA" + }, + "format": { + "type": "string", + "enum": [ + "celsius", + "fahrenheit" + ], + "description": "The temperature unit to use. Infer this from the users location." + } + }, + "required": [ + "location", + "format" + ] } - }, - "required": [ - "location", - "format" - ], - "type": "object" + } } - } + ], + "tool_choice": "auto", + "parallel_tool_calls": true, + "top_p": 1, + "presence_penalty": 0, + "frequency_penalty": 0, + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant." + }, + { + "role": "user", + "content": "Who directed the movie Twister?" + } + ] + }, + "uid": "423a06d4-1111-48a8-9897-5348ea362ebc", + "responses": [ + "The movie \"Twister\" was directed by Jan de Bont." ], - "messages": [ - { - "content": "You are a helpful assistant.", - "role": "system" + "raw_response": { + "id": "chatcmpl-AfCpAdlkheDjPdCxfgjJblcMb6Wtd", + "object": "chat.completion", + "created": 1734384108, + "model": "gpt-4-turbo-2024-04-09", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "The movie \"Twister\" was directed by Jan de Bont.", + "refusal": null + }, + "logprobs": null, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 97, + "completion_tokens": 15, + "total_tokens": 112, + "prompt_tokens_details": { + "cached_tokens": 0, + "audio_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0 + } }, - { - "content": "What is the average temperature in Hawaii in December?", - "role": "user" - } - ], - "model": "gpt-3.5-turbo-0613", - "n": 1, - "presence_penalty": 0, - "temperature": 1, - "top_p": 1 - }, - "raw_response": { - "choices": [ - { - "finish_reason": "function_call", - "index": 0, - "message": { - "content": null, - "function_call": { - "arguments": "{\n \"format\": \"celsius\",\n \"location\": \"Honolulu, HI\"\n}", - "name": "get_current_weather" - }, - "role": "assistant" - } - } - ], - "created": 1688060205, - "id": "chatcmpl-7WprR8fKdPbiiKfpGNlzxp5rmGDDg", - "model": "gpt-3.5-turbo-0613", - "object": "chat.completion", - "usage": { - "completion_tokens": 27, - "prompt_tokens": 100, - "total_tokens": 127 + "system_fingerprint": "fp_1e9a2d1e89" + }, + "llm": "gpt-4-turbo", + "vars": { + "prompt": "Who directed the movie Twister?" + }, + "metavars": { + "__pt": "{prompt}", + "LLM_0": "gpt-4-turbo" } - }, - "responses": [ - "[[FUNCTION]] get_current_weather{\n \"format\": \"celsius\",\n \"location\": \"Honolulu, HI\"\n}" - ], - "uid": "4b818464-a297-429e-b081-19c61b12bae7" - }, - "What is the typical temperature in Kenya in June?": { - "info": { - "prompt": "What is the typical temperature in Kenya in June?" - }, - "llm": "gpt3.5-turbo-0613", - "metavars": { - "LLM_0": "gpt3.5-turbo-0613" - }, - "prompt": "What is the typical temperature in Kenya in June?", - "query": { - "frequency_penalty": 0, - "function_call": "auto", - "functions": [ - { - "description": "Get the current weather", - "name": "get_current_weather", - "parameters": { - "properties": { - "format": { - "description": "The temperature unit to use. Infer this from the users location.", - "enum": [ - "celsius", - "fahrenheit" - ], - "type": "string" - }, - "location": { - "description": "The city and state, e.g. San Francisco, CA", - "type": "string" + } + ], + "What's the weather like in Boston?": [ + { + "prompt": "What's the weather like in Boston?", + "query": { + "model": "gpt-4-turbo", + "n": 1, + "temperature": 1, + "response_format": { + "type": "text" + }, + "tools": [ + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA" + }, + "format": { + "type": "string", + "enum": [ + "celsius", + "fahrenheit" + ], + "description": "The temperature unit to use. Infer this from the users location." + } + }, + "required": [ + "location", + "format" + ] } - }, - "required": [ - "location", - "format" - ], - "type": "object" + } } - } + ], + "tool_choice": "auto", + "parallel_tool_calls": true, + "top_p": 1, + "presence_penalty": 0, + "frequency_penalty": 0, + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant." + }, + { + "role": "user", + "content": "What's the weather like in Boston?" + } + ] + }, + "uid": "2e2d2d3f-d162-4f6a-af6a-23f2628748e5", + "responses": [ + "[[TOOLS]] get_current_weather {\"location\":\"Boston, MA\",\"format\":\"fahrenheit\"}" ], - "messages": [ - { - "content": "You are a helpful assistant.", - "role": "system" + "raw_response": { + "id": "chatcmpl-AfCpBMxF3317N1PrY2eCMAaJFAVjz", + "object": "chat.completion", + "created": 1734384109, + "model": "gpt-4-turbo-2024-04-09", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": null, + "tool_calls": [ + { + "id": "call_yrH7RiZrhGHnrKbtwvX75xVe", + "type": "function", + "function": { + "name": "get_current_weather", + "arguments": "{\"location\":\"Boston, MA\",\"format\":\"fahrenheit\"}" + } + } + ], + "refusal": null + }, + "logprobs": null, + "finish_reason": "tool_calls" + } + ], + "usage": { + "prompt_tokens": 98, + "completion_tokens": 22, + "total_tokens": 120, + "prompt_tokens_details": { + "cached_tokens": 0, + "audio_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0 + } }, - { - "content": "What is the typical temperature in Kenya in June?", - "role": "user" - } - ], - "model": "gpt-3.5-turbo-0613", - "n": 1, - "presence_penalty": 0, - "temperature": 1, - "top_p": 1 - }, - "raw_response": { - "choices": [ - { - "finish_reason": "function_call", - "index": 0, - "message": { - "content": null, - "function_call": { - "arguments": "{\n \"format\": \"celsius\",\n \"location\": \"Kenya\"\n}", - "name": "get_current_weather" - }, - "role": "assistant" - } - } - ], - "created": 1688060203, - "id": "chatcmpl-7WprP14IkWO9tugoobEIllKmtPfvf", - "model": "gpt-3.5-turbo-0613", - "object": "chat.completion", - "usage": { - "completion_tokens": 25, - "prompt_tokens": 100, - "total_tokens": 125 + "system_fingerprint": "fp_1e9a2d1e89" + }, + "llm": "gpt-4-turbo", + "vars": { + "prompt": "What's the weather like in Boston?" + }, + "metavars": { + "__pt": "{prompt}", + "LLM_0": "gpt-4-turbo" } - }, - "responses": [ - "[[FUNCTION]] get_current_weather{\n \"format\": \"celsius\",\n \"location\": \"Kenya\"\n}" - ], - "uid": "2a8ff935-4c25-4222-8d37-22634b8c6208" - }, - "What's the weather like in Boston?": { - "info": { - "prompt": "What's the weather like in Boston?" - }, - "llm": "gpt3.5-turbo-0613", - "metavars": { - "LLM_0": "gpt3.5-turbo-0613" - }, - "prompt": "What's the weather like in Boston?", - "query": { - "frequency_penalty": 0, - "function_call": "auto", - "functions": [ - { - "description": "Get the current weather", - "name": "get_current_weather", - "parameters": { - "properties": { - "format": { - "description": "The temperature unit to use. Infer this from the users location.", - "enum": [ - "celsius", - "fahrenheit" - ], - "type": "string" - }, - "location": { - "description": "The city and state, e.g. San Francisco, CA", - "type": "string" + } + ], + "What is the average temperature in Hawaii in December?": [ + { + "prompt": "What is the average temperature in Hawaii in December?", + "query": { + "model": "gpt-4-turbo", + "n": 1, + "temperature": 1, + "response_format": { + "type": "text" + }, + "tools": [ + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA" + }, + "format": { + "type": "string", + "enum": [ + "celsius", + "fahrenheit" + ], + "description": "The temperature unit to use. Infer this from the users location." + } + }, + "required": [ + "location", + "format" + ] } - }, - "required": [ - "location", - "format" - ], - "type": "object" + } } - } + ], + "tool_choice": "auto", + "parallel_tool_calls": true, + "top_p": 1, + "presence_penalty": 0, + "frequency_penalty": 0, + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant." + }, + { + "role": "user", + "content": "What is the average temperature in Hawaii in December?" + } + ] + }, + "uid": "de3fd081-62b5-4943-bd4d-36987a43004d", + "responses": [ + "The average temperature in Hawaii in December typically ranges from about 68°F (20°C) to 81°F (27°C). This offers pleasant, warm weather which is why Hawaii is such a popular winter destination. Local variations across the different islands can, of course, result in slightly different temperatures due to geography and elevation." ], - "messages": [ - { - "content": "You are a helpful assistant.", - "role": "system" + "raw_response": { + "id": "chatcmpl-AfCpBjs2kSjT3igD7iKeJWN1alH8g", + "object": "chat.completion", + "created": 1734384109, + "model": "gpt-4-turbo-2024-04-09", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "The average temperature in Hawaii in December typically ranges from about 68°F (20°C) to 81°F (27°C). This offers pleasant, warm weather which is why Hawaii is such a popular winter destination. Local variations across the different islands can, of course, result in slightly different temperatures due to geography and elevation.", + "refusal": null + }, + "logprobs": null, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 100, + "completion_tokens": 66, + "total_tokens": 166, + "prompt_tokens_details": { + "cached_tokens": 0, + "audio_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0 + } }, - { - "content": "What's the weather like in Boston?", - "role": "user" - } - ], - "model": "gpt-3.5-turbo-0613", - "n": 1, - "presence_penalty": 0, - "temperature": 1, - "top_p": 1 - }, - "raw_response": { - "choices": [ - { - "finish_reason": "function_call", - "index": 0, - "message": { - "content": null, - "function_call": { - "arguments": "{\n \"format\": \"celsius\",\n \"location\": \"Boston, MA\"\n}", - "name": "get_current_weather" - }, - "role": "assistant" - } - } - ], - "created": 1688060195, - "id": "chatcmpl-7WprHWqiuPcjBcxrmxg45tolyLbHK", - "model": "gpt-3.5-turbo-0613", - "object": "chat.completion", - "usage": { - "completion_tokens": 26, - "prompt_tokens": 98, - "total_tokens": 124 + "system_fingerprint": "fp_1e9a2d1e89" + }, + "llm": "gpt-4-turbo", + "vars": { + "prompt": "What is the average temperature in Hawaii in December?" + }, + "metavars": { + "__pt": "{prompt}", + "LLM_0": "gpt-4-turbo" } - }, - "responses": [ - "[[FUNCTION]] get_current_weather{\n \"format\": \"celsius\",\n \"location\": \"Boston, MA\"\n}" - ], - "uid": "cfa8f743-32c7-4a24-8141-9e1fa264af49" - }, - "Who directed the movie Twister?": { - "info": { - "prompt": "Who directed the movie Twister?" - }, - "llm": "gpt3.5-turbo-0613", - "metavars": { - "LLM_0": "gpt3.5-turbo-0613" - }, - "prompt": "Who directed the movie Twister?", - "query": { - "frequency_penalty": 0, - "function_call": "auto", - "functions": [ - { - "description": "Get the current weather", - "name": "get_current_weather", - "parameters": { - "properties": { - "format": { - "description": "The temperature unit to use. Infer this from the users location.", - "enum": [ - "celsius", - "fahrenheit" - ], - "type": "string" - }, - "location": { - "description": "The city and state, e.g. San Francisco, CA", - "type": "string" + } + ], + "What is the typical temperature in Kenya in June?": [ + { + "prompt": "What is the typical temperature in Kenya in June?", + "query": { + "model": "gpt-4-turbo", + "n": 1, + "temperature": 1, + "response_format": { + "type": "text" + }, + "tools": [ + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA" + }, + "format": { + "type": "string", + "enum": [ + "celsius", + "fahrenheit" + ], + "description": "The temperature unit to use. Infer this from the users location." + } + }, + "required": [ + "location", + "format" + ] } + } + } + ], + "tool_choice": "auto", + "parallel_tool_calls": true, + "top_p": 1, + "presence_penalty": 0, + "frequency_penalty": 0, + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant." + }, + { + "role": "user", + "content": "What is the typical temperature in Kenya in June?" + } + ] + }, + "uid": "b3e061b4-b81d-4904-97b3-5627d4e64824", + "responses": [ + "In June, Kenya typically experiences variable temperatures depending on the region. Due to its diverse geography, which includes coastal areas, highlands, and savannahs, temperatures can differ significantly:\n\n1. **Coastal areas (e.g., Mombasa):** The temperatures are warmer with highs around 28°C (82°F) and lows about 20°C (68°F).\n \n2. **Highlands (e.g., Nairobi):** Cooler temperatures prevail, with daytime highs averaging 21°C (70°F) and nights around 12°C (54°F).\n\n3. **Western and Lake Victoria region:** Warm and humid with temperatures similar to the coastal areas but can be a bit cooler especially during the night.\n\nJune marks the start of the cooler season in Kenya, making it a pleasant time to visit, especially for those who prefer milder weather away from the heat and humidity typical of tropical climates." + ], + "raw_response": { + "id": "chatcmpl-AfCpAeIFNmzszs2hdQeZn2xdc3DGt", + "object": "chat.completion", + "created": 1734384108, + "model": "gpt-4-turbo-2024-04-09", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "In June, Kenya typically experiences variable temperatures depending on the region. Due to its diverse geography, which includes coastal areas, highlands, and savannahs, temperatures can differ significantly:\n\n1. **Coastal areas (e.g., Mombasa):** The temperatures are warmer with highs around 28°C (82°F) and lows about 20°C (68°F).\n \n2. **Highlands (e.g., Nairobi):** Cooler temperatures prevail, with daytime highs averaging 21°C (70°F) and nights around 12°C (54°F).\n\n3. **Western and Lake Victoria region:** Warm and humid with temperatures similar to the coastal areas but can be a bit cooler especially during the night.\n\nJune marks the start of the cooler season in Kenya, making it a pleasant time to visit, especially for those who prefer milder weather away from the heat and humidity typical of tropical climates.", + "refusal": null }, - "required": [ - "location", - "format" - ], - "type": "object" + "logprobs": null, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 100, + "completion_tokens": 184, + "total_tokens": 284, + "prompt_tokens_details": { + "cached_tokens": 0, + "audio_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0 } - } - ], - "messages": [ - { - "content": "You are a helpful assistant.", - "role": "system" }, - { - "content": "Who directed the movie Twister?", - "role": "user" - } - ], - "model": "gpt-3.5-turbo-0613", - "n": 1, - "presence_penalty": 0, - "temperature": 1, - "top_p": 1 - }, - "raw_response": { - "choices": [ - { - "finish_reason": "stop", - "index": 0, - "message": { - "content": "The movie \"Twister\" was directed by Jan de Bont.", - "role": "assistant" - } - } - ], - "created": 1688060197, - "id": "chatcmpl-7WprJS1TxKFYodJryajN6WIP4NA7s", - "model": "gpt-3.5-turbo-0613", - "object": "chat.completion", - "usage": { - "completion_tokens": 15, - "prompt_tokens": 97, - "total_tokens": 112 + "system_fingerprint": "fp_1e9a2d1e89" + }, + "llm": "gpt-4-turbo", + "vars": { + "prompt": "What is the typical temperature in Kenya in June?" + }, + "metavars": { + "__pt": "{prompt}", + "LLM_0": "gpt-4-turbo" } - }, - "responses": [ - "The movie \"Twister\" was directed by Jan de Bont." - ], - "uid": "ebcba322-b11c-4680-8170-98b0f3bebbe5" - } + } + ], + "What causes thunderstorms?": [ + { + "prompt": "What causes thunderstorms?", + "query": { + "model": "gpt-4-turbo", + "n": 1, + "temperature": 1, + "response_format": { + "type": "text" + }, + "tools": [ + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA" + }, + "format": { + "type": "string", + "enum": [ + "celsius", + "fahrenheit" + ], + "description": "The temperature unit to use. Infer this from the users location." + } + }, + "required": [ + "location", + "format" + ] + } + } + } + ], + "tool_choice": "auto", + "parallel_tool_calls": true, + "top_p": 1, + "presence_penalty": 0, + "frequency_penalty": 0, + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant." + }, + { + "role": "user", + "content": "What causes thunderstorms?" + } + ] + }, + "uid": "0068affe-862a-4864-b4ef-376913a78d4a", + "responses": [ + "Thunderstorms are caused by the combination of moisture, rapidly rising warm air, and a lifting mechanism. Here is a breakdown of the factors and processes involved:\n\n1. **Moisture**: Moisture is necessary to form clouds and rain. It comes primarily from bodies of water like oceans and lakes. Warmer air can hold more moisture than cooler air.\n\n2. **Rapidly Rising Warm Air**: When the surface air is warmed by the sun, it becomes lighter and rises quickly. As the air rises, it cools, and the moisture it carries condenses into a cumulus cloud. If the rising continues and conditions are favorable, these clouds can grow into cumulonimbus clouds, which are the towering clouds typically associated with thunderstorms.\n\n3. **Lifting Mechanism**: A lifting mechanism is needed to help or force the warm air to rise. This can be caused by a variety of geographical and meteorological factors:\n - Frontal systems (where cold and warm air masses collide)\n - Sea breezes\n - Mountains that force the air to rise (orographic lift)\n - Variations in temperature (thermal differences) on the surface caused by features such as urban settings, forests, and crops.\n\nOnce these elements are in place, a thunderstorm can develop. Within the storm, electrical charges become separated, with positive charges often accumulating near the top of the storm and negative charges near the bottom. This charge separation can eventually lead to lightning, which is the sudden neutralization of these charges. The rapid expansion of air heated by the lightning causes thunder.\n\nThroughout a thunderstorm, various types of precipitation (rain, hail) and wind phenomena (gusts, downdrafts) can occur, depending on the intensity and maturity of the storm." + ], + "raw_response": { + "id": "chatcmpl-AfCpAg1SfblY1MioKNZCeSNztUzXT", + "object": "chat.completion", + "created": 1734384108, + "model": "gpt-4-turbo-2024-04-09", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "Thunderstorms are caused by the combination of moisture, rapidly rising warm air, and a lifting mechanism. Here is a breakdown of the factors and processes involved:\n\n1. **Moisture**: Moisture is necessary to form clouds and rain. It comes primarily from bodies of water like oceans and lakes. Warmer air can hold more moisture than cooler air.\n\n2. **Rapidly Rising Warm Air**: When the surface air is warmed by the sun, it becomes lighter and rises quickly. As the air rises, it cools, and the moisture it carries condenses into a cumulus cloud. If the rising continues and conditions are favorable, these clouds can grow into cumulonimbus clouds, which are the towering clouds typically associated with thunderstorms.\n\n3. **Lifting Mechanism**: A lifting mechanism is needed to help or force the warm air to rise. This can be caused by a variety of geographical and meteorological factors:\n - Frontal systems (where cold and warm air masses collide)\n - Sea breezes\n - Mountains that force the air to rise (orographic lift)\n - Variations in temperature (thermal differences) on the surface caused by features such as urban settings, forests, and crops.\n\nOnce these elements are in place, a thunderstorm can develop. Within the storm, electrical charges become separated, with positive charges often accumulating near the top of the storm and negative charges near the bottom. This charge separation can eventually lead to lightning, which is the sudden neutralization of these charges. The rapid expansion of air heated by the lightning causes thunder.\n\nThroughout a thunderstorm, various types of precipitation (rain, hail) and wind phenomena (gusts, downdrafts) can occur, depending on the intensity and maturity of the storm.", + "refusal": null + }, + "logprobs": null, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 95, + "completion_tokens": 360, + "total_tokens": 455, + "prompt_tokens_details": { + "cached_tokens": 0, + "audio_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0 + } + }, + "system_fingerprint": "fp_1e9a2d1e89" + }, + "llm": "gpt-4-turbo", + "vars": { + "prompt": "What causes thunderstorms?" + }, + "metavars": { + "__pt": "{prompt}", + "LLM_0": "gpt-4-turbo" + } + } + ] + }, + "prompt-1686689549227_1.json": { + "Who directed the movie Twister?": [ + { + "prompt": "Who directed the movie Twister?", + "query": { + "model": "gpt-3.5-turbo", + "n": 1, + "temperature": 1, + "response_format": { + "type": "text" + }, + "top_p": 1, + "presence_penalty": 0, + "frequency_penalty": 0, + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant." + }, + { + "role": "user", + "content": "Who directed the movie Twister?" + } + ] + }, + "uid": "53b45f6f-12fe-4d2a-998b-4fc199c28c1d", + "responses": [ + "The movie Twister was directed by Jan de Bont. It was released in 1996 and stars Helen Hunt and Bill Paxton." + ], + "raw_response": { + "id": "chatcmpl-AfCbGELWVdAqgyPl8okECZ5mXVBKN", + "object": "chat.completion", + "created": 1734383246, + "model": "gpt-3.5-turbo-0125", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "The movie Twister was directed by Jan de Bont. It was released in 1996 and stars Helen Hunt and Bill Paxton.", + "refusal": null + }, + "logprobs": null, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 24, + "completion_tokens": 28, + "total_tokens": 52, + "prompt_tokens_details": { + "cached_tokens": 0, + "audio_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0 + } + }, + "system_fingerprint": null + }, + "llm": "old-gpt3.5", + "vars": { + "prompt": "Who directed the movie Twister?" + }, + "metavars": { + "__pt": "{prompt}", + "LLM_0": "old-gpt3.5" + } + } + ], + "What is the average temperature in Hawaii in December?": [ + { + "prompt": "What is the average temperature in Hawaii in December?", + "query": { + "model": "gpt-3.5-turbo", + "n": 1, + "temperature": 1, + "response_format": { + "type": "text" + }, + "top_p": 1, + "presence_penalty": 0, + "frequency_penalty": 0, + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant." + }, + { + "role": "user", + "content": "What is the average temperature in Hawaii in December?" + } + ] + }, + "uid": "f3aa7139-def5-4397-8b87-ee075ffe315f", + "responses": [ + "In December, the average temperature in Hawaii typically ranges from 75°F to 80°F (24°C to 27°C). However, the exact temperature can vary depending on the specific island and location within Hawaii." + ], + "raw_response": { + "id": "chatcmpl-AfCbGggwgVkoEhDXJzphmaTcopah0", + "object": "chat.completion", + "created": 1734383246, + "model": "gpt-3.5-turbo-0125", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "In December, the average temperature in Hawaii typically ranges from 75°F to 80°F (24°C to 27°C). However, the exact temperature can vary depending on the specific island and location within Hawaii.", + "refusal": null + }, + "logprobs": null, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 27, + "completion_tokens": 43, + "total_tokens": 70, + "prompt_tokens_details": { + "cached_tokens": 0, + "audio_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0 + } + }, + "system_fingerprint": null + }, + "llm": "old-gpt3.5", + "vars": { + "prompt": "What is the average temperature in Hawaii in December?" + }, + "metavars": { + "__pt": "{prompt}", + "LLM_0": "old-gpt3.5" + } + } + ], + "What's the weather like in Boston?": [ + { + "prompt": "What's the weather like in Boston?", + "query": { + "model": "gpt-3.5-turbo", + "n": 1, + "temperature": 1, + "response_format": { + "type": "text" + }, + "top_p": 1, + "presence_penalty": 0, + "frequency_penalty": 0, + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant." + }, + { + "role": "user", + "content": "What's the weather like in Boston?" + } + ] + }, + "uid": "64f90d18-939b-4e59-867e-5c433edd5103", + "responses": [ + "I'm sorry, but I don't have real-time data access. You can check the weather in Boston by using a weather website or app, or by asking a voice assistant like Siri or Google Assistant for the current weather in Boston." + ], + "raw_response": { + "id": "chatcmpl-AfCbGiqPSdLy7XFXPHC23kyCd2CMK", + "object": "chat.completion", + "created": 1734383246, + "model": "gpt-3.5-turbo-0125", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "I'm sorry, but I don't have real-time data access. You can check the weather in Boston by using a weather website or app, or by asking a voice assistant like Siri or Google Assistant for the current weather in Boston.", + "refusal": null + }, + "logprobs": null, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 25, + "completion_tokens": 47, + "total_tokens": 72, + "prompt_tokens_details": { + "cached_tokens": 0, + "audio_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0 + } + }, + "system_fingerprint": null + }, + "llm": "old-gpt3.5", + "vars": { + "prompt": "What's the weather like in Boston?" + }, + "metavars": { + "__pt": "{prompt}", + "LLM_0": "old-gpt3.5" + } + } + ], + "What is the typical temperature in Kenya in June?": [ + { + "prompt": "What is the typical temperature in Kenya in June?", + "query": { + "model": "gpt-3.5-turbo", + "n": 1, + "temperature": 1, + "response_format": { + "type": "text" + }, + "top_p": 1, + "presence_penalty": 0, + "frequency_penalty": 0, + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant." + }, + { + "role": "user", + "content": "What is the typical temperature in Kenya in June?" + } + ] + }, + "uid": "7f84b658-031a-4bc8-9e2f-37e1db9b21e0", + "responses": [ + "In Kenya, June is during the dry season, with temperatures averaging around 18-23°C (64-73°F) in the central highlands and around 24-30°C (75-86°F) along the coast and in lower elevations. However, temperatures can vary depending on the specific region and altitude within the country." + ], + "raw_response": { + "id": "chatcmpl-AfCbGNqTbBjuj4UnFYLPENnKeuVya", + "object": "chat.completion", + "created": 1734383246, + "model": "gpt-3.5-turbo-0125", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "In Kenya, June is during the dry season, with temperatures averaging around 18-23°C (64-73°F) in the central highlands and around 24-30°C (75-86°F) along the coast and in lower elevations. However, temperatures can vary depending on the specific region and altitude within the country.", + "refusal": null + }, + "logprobs": null, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 27, + "completion_tokens": 68, + "total_tokens": 95, + "prompt_tokens_details": { + "cached_tokens": 0, + "audio_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0 + } + }, + "system_fingerprint": null + }, + "llm": "old-gpt3.5", + "vars": { + "prompt": "What is the typical temperature in Kenya in June?" + }, + "metavars": { + "__pt": "{prompt}", + "LLM_0": "old-gpt3.5" + } + } + ], + "What causes thunderstorms?": [ + { + "prompt": "What causes thunderstorms?", + "query": { + "model": "gpt-3.5-turbo", + "n": 1, + "temperature": 1, + "response_format": { + "type": "text" + }, + "top_p": 1, + "presence_penalty": 0, + "frequency_penalty": 0, + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant." + }, + { + "role": "user", + "content": "What causes thunderstorms?" + } + ] + }, + "uid": "6056ad0a-a1c6-4c21-9788-f1c9143f6f85", + "responses": [ + "Thunderstorms are caused by the rapid upward movement of warm, moist air that cools and condenses into clouds. This process leads to the formation of cumulonimbus clouds, which can grow to great heights and create unstable atmospheric conditions. As the clouds continue to grow, water droplets and ice particles collide and create an electrical charge. The difference in charge between the clouds and the ground or between different parts of the cloud leads to the discharge of lightning, which is then followed by the sound of thunder. This entire process of rapid uplift, condensation, and electrical discharge is what we experience as a thunderstorm." + ], + "raw_response": { + "id": "chatcmpl-AfCbGTKfcTt0vPyWbhboe2rXAhmDr", + "object": "chat.completion", + "created": 1734383246, + "model": "gpt-3.5-turbo-0125", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "Thunderstorms are caused by the rapid upward movement of warm, moist air that cools and condenses into clouds. This process leads to the formation of cumulonimbus clouds, which can grow to great heights and create unstable atmospheric conditions. As the clouds continue to grow, water droplets and ice particles collide and create an electrical charge. The difference in charge between the clouds and the ground or between different parts of the cloud leads to the discharge of lightning, which is then followed by the sound of thunder. This entire process of rapid uplift, condensation, and electrical discharge is what we experience as a thunderstorm.", + "refusal": null + }, + "logprobs": null, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 22, + "completion_tokens": 124, + "total_tokens": 146, + "prompt_tokens_details": { + "cached_tokens": 0, + "audio_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0 + } + }, + "system_fingerprint": null + }, + "llm": "old-gpt3.5", + "vars": { + "prompt": "What causes thunderstorms?" + }, + "metavars": { + "__pt": "{prompt}", + "LLM_0": "old-gpt3.5" + } + } + ] }, "prompt-1686689549227_4.json": { - "What causes thunderstorms?": { - "info": { - "prompt": "What causes thunderstorms?" - }, - "llm": "gpt-3.5-turbo", - "metavars": {}, - "prompt": "What causes thunderstorms?", - "query": { - "frequency_penalty": 0, - "messages": [ - { - "content": "You are a helpful assistant.", - "role": "system" - }, - { - "content": "What causes thunderstorms?", - "role": "user" - } - ], - "model": "gpt-3.5-turbo", - "n": 1, - "presence_penalty": 0, - "temperature": 1, - "top_p": 1 - }, - "raw_response": { - "choices": [ - { - "finish_reason": "stop", - "index": 0, - "message": { - "content": "Thunderstorms are caused by the rapid upward movement of warm, moist air, which creates unstable atmospheric conditions. As the warm air rises, it cools and forms cumulonimbus clouds, which are the towering dark clouds that typically come to mind when thinking of a thunderstorm. Within these clouds, a variety of atmospheric conditions exist that can cause the development of lightning, thunder, heavy rain, hail, and tornadoes. The exact cause of a thunderstorm can vary depending on the location, time of year, and other environmental factors.", - "role": "assistant" - } - } - ], - "created": 1686690642, - "id": "chatcmpl-7R5ZiB1FBQAfXNSqYqhCKPqaNOeyE", - "model": "gpt-3.5-turbo-0301", - "object": "chat.completion", - "usage": { - "completion_tokens": 108, - "prompt_tokens": 24, - "total_tokens": 132 - } - }, - "responses": [ - "Thunderstorms are caused by the rapid upward movement of warm, moist air, which creates unstable atmospheric conditions. As the warm air rises, it cools and forms cumulonimbus clouds, which are the towering dark clouds that typically come to mind when thinking of a thunderstorm. Within these clouds, a variety of atmospheric conditions exist that can cause the development of lightning, thunder, heavy rain, hail, and tornadoes. The exact cause of a thunderstorm can vary depending on the location, time of year, and other environmental factors." - ], - "uid": "8c2e8fb8-76dc-455d-8ba7-252f99594b36" - }, - "What is the average temperature in Hawaii in December?": { - "info": { - "prompt": "What is the average temperature in Hawaii in December?" - }, - "llm": "gpt-3.5-turbo", - "metavars": {}, - "prompt": "What is the average temperature in Hawaii in December?", - "query": { - "frequency_penalty": 0, - "messages": [ - { - "content": "You are a helpful assistant.", - "role": "system" - }, - { - "content": "What is the average temperature in Hawaii in December?", - "role": "user" - } - ], - "model": "gpt-3.5-turbo", - "n": 1, - "presence_penalty": 0, - "temperature": 1, - "top_p": 1 - }, - "raw_response": { - "choices": [ - { - "finish_reason": "stop", - "index": 0, - "message": { - "content": "The average temperature in Hawaii in December is around 77°F (25°C) during the day and 66°F (19°C) at night.", - "role": "assistant" - } - } - ], - "created": 1686690979, - "id": "chatcmpl-7R5f9izQoSeVb7zt9MmdQtBwBv5NJ", - "model": "gpt-3.5-turbo-0301", - "object": "chat.completion", - "usage": { - "completion_tokens": 30, - "prompt_tokens": 29, - "total_tokens": 59 - } - }, - "responses": [ - "The average temperature in Hawaii in December is around 77°F (25°C) during the day and 66°F (19°C) at night." - ], - "uid": "426b0d32-7afd-46a8-a9db-b0e893e7e3b7" - }, - "What is the typical temperature in Hawaii in December?": { - "info": { - "prompt": "What is the typical temperature in Hawaii in December?" - }, - "llm": "gpt-3.5-turbo", - "metavars": {}, - "prompt": "What is the typical temperature in Hawaii in December?", - "query": { - "frequency_penalty": 0, - "messages": [ - { - "content": "You are a helpful assistant.", - "role": "system" - }, - { - "content": "What is the typical temperature in Hawaii in December?", - "role": "user" - } - ], - "model": "gpt-3.5-turbo", - "n": 1, - "presence_penalty": 0, - "temperature": 1, - "top_p": 1 - }, - "raw_response": { - "choices": [ - { - "finish_reason": "stop", - "index": 0, - "message": { - "content": "In Hawaii, the typical temperature during December is around 75°F (24°C) during the day and around 65°F (18°C) at night. However, the temperature can vary depending on the island and geographical location within the island.", - "role": "assistant" - } - } - ], - "created": 1686690881, - "id": "chatcmpl-7R5dZWHOPwJY7XQLu1CNJnSjX0pae", - "model": "gpt-3.5-turbo-0301", - "object": "chat.completion", - "usage": { - "completion_tokens": 49, - "prompt_tokens": 29, - "total_tokens": 78 - } - }, - "responses": [ - "In Hawaii, the typical temperature during December is around 75°F (24°C) during the day and around 65°F (18°C) at night. However, the temperature can vary depending on the island and geographical location within the island." - ], - "uid": "be76c4fc-5e04-475f-ace3-b1a9e6046610" - }, - "What is the typical temperature in Kenya in June?": { - "info": { - "prompt": "What is the typical temperature in Kenya in June?" - }, - "llm": "gpt-3.5-turbo", - "metavars": {}, - "prompt": "What is the typical temperature in Kenya in June?", - "query": { - "frequency_penalty": 0, - "messages": [ - { - "content": "You are a helpful assistant.", - "role": "system" - }, - { - "content": "What is the typical temperature in Kenya in June?", - "role": "user" - } - ], - "model": "gpt-3.5-turbo", - "n": 1, - "presence_penalty": 0, - "temperature": 1, - "top_p": 1 - }, - "raw_response": { - "choices": [ - { - "finish_reason": "stop", - "index": 0, - "message": { - "content": "The typical temperature in Kenya in June varies depending on the region, but generally, temperatures range from 19°C (66°F) to 30°C (86°F). In some parts of the country, particularly in areas of higher elevation, temperatures may be cooler, while in areas closer to the equator temperatures can be quite hot.", - "role": "assistant" - } - } - ], - "created": 1686690642, - "id": "chatcmpl-7R5ZiubwJ5ynhW8eL5ujFkFWMTU8d", - "model": "gpt-3.5-turbo-0301", - "object": "chat.completion", - "usage": { - "completion_tokens": 67, - "prompt_tokens": 29, - "total_tokens": 96 - } - }, - "responses": [ - "The typical temperature in Kenya in June varies depending on the region, but generally, temperatures range from 19°C (66°F) to 30°C (86°F). In some parts of the country, particularly in areas of higher elevation, temperatures may be cooler, while in areas closer to the equator temperatures can be quite hot." - ], - "uid": "a993b28a-df28-462e-9e2b-e7afd6742824" - }, - "What's the current weather in Boston?": { - "info": {}, - "llm": "gpt-3.5-turbo", - "metavars": {}, - "prompt": "What's the current weather in Boston?", - "query": { - "frequency_penalty": 0, - "messages": [ - { - "content": "You are a helpful assistant.", - "role": "system" - }, - { - "content": "What's the current weather in Boston?", - "role": "user" - } - ], - "model": "gpt-3.5-turbo", - "n": 1, - "presence_penalty": 0, - "temperature": 1, - "top_p": 1 - }, - "raw_response": { - "choices": [ - { - "finish_reason": "stop", - "index": 0, - "message": { - "content": "I'm sorry, as an AI language model, I don't have access to current, up-to-date information about the weather. However, you can check the current weather in Boston by doing a quick online search or by downloading a weather app that can give you updates on the weather in real-time.", - "role": "assistant" - } - } - ], - "created": 1686690387, - "id": "chatcmpl-7R5VbszTnpyqH0geylhPvnajW9bwi", - "model": "gpt-3.5-turbo-0301", - "object": "chat.completion", - "usage": { - "completion_tokens": 60, - "prompt_tokens": 27, - "total_tokens": 87 - } - }, - "responses": [ - "I'm sorry, as an AI language model, I don't have access to current, up-to-date information about the weather. However, you can check the current weather in Boston by doing a quick online search or by downloading a weather app that can give you updates on the weather in real-time." - ], - "uid": "d2578c6d-90a9-4208-9dbb-b1f5afb7d512" - }, - "What's the weather like in Boston?": { - "info": { - "prompt": "What's the weather like in Boston?" - }, - "llm": "gpt-3.5-turbo", - "metavars": {}, - "prompt": "What's the weather like in Boston?", - "query": { - "frequency_penalty": 0, - "messages": [ - { - "content": "You are a helpful assistant.", - "role": "system" - }, - { - "content": "What's the weather like in Boston?", - "role": "user" - } - ], - "model": "gpt-3.5-turbo", - "n": 1, - "presence_penalty": 0, - "temperature": 1, - "top_p": 1 - }, - "raw_response": { - "choices": [ - { - "finish_reason": "stop", - "index": 0, - "message": { - "content": "I'm sorry, I cannot provide real-time information as I do not have access to live data. However, you can check reliable weather websites or applications for up-to-date information about the weather in Boston.", - "role": "assistant" - } - } - ], - "created": 1686690642, - "id": "chatcmpl-7R5ZiwQcOHitrDS6OqbnzZtv7fOB8", - "model": "gpt-3.5-turbo-0301", - "object": "chat.completion", - "usage": { - "completion_tokens": 41, - "prompt_tokens": 27, - "total_tokens": 68 - } - }, - "responses": [ - "I'm sorry, I cannot provide real-time information as I do not have access to live data. However, you can check reliable weather websites or applications for up-to-date information about the weather in Boston." - ], - "uid": "e342daaf-acde-4988-a759-c1c753efd670" - }, - "Who directed the movie Tornado?": { - "info": { - "prompt": "Who directed the movie Tornado?" - }, - "llm": "gpt-3.5-turbo", - "metavars": {}, - "prompt": "Who directed the movie Tornado?", - "query": { - "frequency_penalty": 0, - "messages": [ - { - "content": "You are a helpful assistant.", - "role": "system" - }, - { - "content": "Who directed the movie Tornado?", - "role": "user" - } - ], - "model": "gpt-3.5-turbo", - "n": 1, - "presence_penalty": 0, - "temperature": 1, - "top_p": 1 - }, - "raw_response": { - "choices": [ - { - "finish_reason": "stop", - "index": 0, - "message": { - "content": "The 1996 movie \"Tornado\" was directed by Noel Nosseck.", - "role": "assistant" - } - } - ], - "created": 1686690642, - "id": "chatcmpl-7R5ZiVUVFwj5bfqk4B6nV6V70CR5Q", - "model": "gpt-3.5-turbo-0301", - "object": "chat.completion", - "usage": { - "completion_tokens": 17, - "prompt_tokens": 26, - "total_tokens": 43 - } - }, - "responses": [ - "The 1996 movie \"Tornado\" was directed by Noel Nosseck." - ], - "uid": "2f75b175-08fa-4d81-93ca-0716d72e9e4d" - }, - "Who directed the movie Twister?": { - "info": { - "prompt": "Who directed the movie Twister?" - }, - "llm": "old-gpt3.5", - "metavars": { - "LLM_0": "old-gpt3.5" - }, - "prompt": "Who directed the movie Twister?", - "query": { - "frequency_penalty": 0, - "messages": [ - { - "content": "You are a helpful assistant.", - "role": "system" - }, - { - "content": "Who directed the movie Twister?", - "role": "user" - } - ], - "model": "gpt-3.5-turbo", - "n": 1, - "presence_penalty": 0, - "temperature": 1, - "top_p": 1 - }, - "raw_response": { - "choices": [ - { - "finish_reason": "stop", - "index": 0, - "message": { - "content": "The movie \"Twister\" was directed by Jan de Bont.", - "role": "assistant" - } - } - ], - "created": 1688060195, - "id": "chatcmpl-7WprHzPMNAUbxvRIe6T6HYW36X331", - "model": "gpt-3.5-turbo-0613", - "object": "chat.completion", - "usage": { - "completion_tokens": 14, - "prompt_tokens": 24, - "total_tokens": 38 - } - }, - "responses": [ - "The movie \"Twister\" was directed by Jan de Bont." - ], - "uid": "1e884f42-d130-4a33-9039-22490772902d" - } - }, - "prompt-1686689549227.json": { - "cache_files": { - "prompt-1686689549227_6.json": { - "base_model": "gpt-3.5-turbo", - "emoji": "🌦️", - "formData": { - "frequency_penalty": 0, - "function_call": "auto", - "functions": "[\n {\n \"name\": \"get_current_weather\",\n \"description\": \"Get the current weather\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"location\": {\n \"type\": \"string\",\n \"description\": \"The city and state, e.g. San Francisco, CA\"\n },\n \"format\": {\n \"type\": \"string\",\n \"enum\": [\"celsius\", \"fahrenheit\"],\n \"description\": \"The temperature unit to use. Infer this from the users location.\"\n }\n },\n \"required\": [\"location\", \"format\"]\n }\n }\n]", - "model": "gpt-3.5-turbo-0613", - "presence_penalty": 0, - "shortname": "gpt3.5-turbo-0613", - "stop": "", - "system_msg": "You are a helpful assistant.", + "What is the typical temperature in Kenya in June?": [ + { + "prompt": "What is the typical temperature in Kenya in June?", + "query": { + "model": "claude-3-5-sonnet-latest", + "stop_sequences": [ + "\n\nHuman:" + ], "temperature": 1, - "top_p": 1 - }, - "key": "af7031af-96ae-44b8-bcc4-3538f3174b73", - "model": "gpt-3.5-turbo-0613", - "name": "gpt3.5-turbo-0613", - "settings": { - "frequency_penalty": 0, - "function_call": "auto", - "functions": [ + "tools": [ { - "description": "Get the current weather", "name": "get_current_weather", - "parameters": { + "description": "Get the current weather", + "input_schema": { + "type": "object", "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA" + }, "format": { - "description": "The temperature unit to use. Infer this from the users location.", + "type": "string", "enum": [ "celsius", "fahrenheit" ], - "type": "string" - }, - "location": { - "description": "The city and state, e.g. San Francisco, CA", - "type": "string" + "description": "The temperature unit to use. Infer this from the users location." } }, "required": [ "location", "format" - ], - "type": "object" + ] } } ], - "presence_penalty": 0, + "top_k": -1, + "top_p": -1, + "tool_choice": { + "type": "any", + "disable_parallel_tool_use": false + }, + "max_tokens": 1024, + "messages": [ + { + "role": "user", + "content": "What is the typical temperature in Kenya in June?" + } + ] + }, + "uid": "eac6af65-6e4c-40ac-a8ad-b07e13c36ac5", + "responses": [ + "[[TOOLS]] {\"name\":\"get_current_weather\",\"input\":{\"format\":\"celsius\",\"location\":\"Nairobi, Kenya\"}}" + ], + "raw_response": [ + { + "content": [ + { + "id": "toolu_01TFFmPjxfi4MyX7Q1S7eWR2", + "input": { + "format": "celsius", + "location": "Nairobi, Kenya" + }, + "name": "get_current_weather", + "type": "tool_use" + } + ], + "id": "msg_01TVPDZ4Puht2ceZG4G5neSH", + "model": "claude-3-5-sonnet-20241022", + "role": "assistant", + "stop_reason": "tool_use", + "stop_sequence": null, + "type": "message", + "usage": { + "input_tokens": 445, + "output_tokens": 62 + } + } + ], + "llm": "Claude", + "vars": { + "prompt": "What is the typical temperature in Kenya in June?" + }, + "metavars": { + "__pt": "{prompt}", + "LLM_0": "Claude" + } + } + ], + "What's the weather like in Boston?": [ + { + "prompt": "What's the weather like in Boston?", + "query": { + "model": "claude-3-5-sonnet-latest", + "stop_sequences": [ + "\n\nHuman:" + ], "temperature": 1, - "top_p": 1 + "tools": [ + { + "name": "get_current_weather", + "description": "Get the current weather", + "input_schema": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA" + }, + "format": { + "type": "string", + "enum": [ + "celsius", + "fahrenheit" + ], + "description": "The temperature unit to use. Infer this from the users location." + } + }, + "required": [ + "location", + "format" + ] + } + } + ], + "top_k": -1, + "top_p": -1, + "tool_choice": { + "type": "any", + "disable_parallel_tool_use": false + }, + "max_tokens": 1024, + "messages": [ + { + "role": "user", + "content": "What's the weather like in Boston?" + } + ] + }, + "uid": "eb2ba7da-fc44-4b3e-9cf9-6ee684e84ac6", + "responses": [ + "[[TOOLS]] {\"name\":\"get_current_weather\",\"input\":{\"format\":\"fahrenheit\",\"location\":\"Boston, MA\"}}" + ], + "raw_response": [ + { + "content": [ + { + "id": "toolu_01WLeUK3MNaj2BqGTUTXdR7g", + "input": { + "format": "fahrenheit", + "location": "Boston, MA" + }, + "name": "get_current_weather", + "type": "tool_use" + } + ], + "id": "msg_019v6jdbPmjLdkaAueukvgU3", + "model": "claude-3-5-sonnet-20241022", + "role": "assistant", + "stop_reason": "tool_use", + "stop_sequence": null, + "type": "message", + "usage": { + "input_tokens": 443, + "output_tokens": 60 + } + } + ], + "llm": "Claude", + "vars": { + "prompt": "What's the weather like in Boston?" + }, + "metavars": { + "__pt": "{prompt}", + "LLM_0": "Claude" + } + } + ], + "Who directed the movie Twister?": [ + { + "prompt": "Who directed the movie Twister?", + "query": { + "model": "claude-3-5-sonnet-latest", + "stop_sequences": [ + "\n\nHuman:" + ], + "temperature": 1, + "tools": [ + { + "name": "get_current_weather", + "description": "Get the current weather", + "input_schema": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA" + }, + "format": { + "type": "string", + "enum": [ + "celsius", + "fahrenheit" + ], + "description": "The temperature unit to use. Infer this from the users location." + } + }, + "required": [ + "location", + "format" + ] + } + } + ], + "top_k": -1, + "top_p": -1, + "tool_choice": { + "type": "any", + "disable_parallel_tool_use": false + }, + "max_tokens": 1024, + "messages": [ + { + "role": "user", + "content": "Who directed the movie Twister?" + } + ] + }, + "uid": "19109992-fdef-479d-92fd-7a48c0b4b527", + "responses": [ + "[[TOOLS]] {\"name\":\"get_current_weather\",\"input\":{\"format\":\"celsius\",\"location\":\"\"}}" + ], + "raw_response": [ + { + "content": [ + { + "id": "toolu_01JW1eWcqWc5CcDfmG765gj1", + "input": { + "format": "celsius", + "location": "" + }, + "name": "get_current_weather", + "type": "tool_use" + } + ], + "id": "msg_01HXFMbjQTA3VPhh5t3baUuK", + "model": "claude-3-5-sonnet-20241022", + "role": "assistant", + "stop_reason": "tool_use", + "stop_sequence": null, + "type": "message", + "usage": { + "input_tokens": 443, + "output_tokens": 58 + } + } + ], + "llm": "Claude", + "vars": { + "prompt": "Who directed the movie Twister?" + }, + "metavars": { + "__pt": "{prompt}", + "LLM_0": "Claude" + } + } + ], + "What is the average temperature in Hawaii in December?": [ + { + "prompt": "What is the average temperature in Hawaii in December?", + "query": { + "model": "claude-3-5-sonnet-latest", + "stop_sequences": [ + "\n\nHuman:" + ], + "temperature": 1, + "tools": [ + { + "name": "get_current_weather", + "description": "Get the current weather", + "input_schema": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA" + }, + "format": { + "type": "string", + "enum": [ + "celsius", + "fahrenheit" + ], + "description": "The temperature unit to use. Infer this from the users location." + } + }, + "required": [ + "location", + "format" + ] + } + } + ], + "top_k": -1, + "top_p": -1, + "tool_choice": { + "type": "any", + "disable_parallel_tool_use": false + }, + "max_tokens": 1024, + "messages": [ + { + "role": "user", + "content": "What is the average temperature in Hawaii in December?" + } + ] + }, + "uid": "58de1cb7-3514-4ecc-ae42-6e0900b91ff5", + "responses": [ + "[[TOOLS]] {\"name\":\"get_current_weather\",\"input\":{\"format\":\"fahrenheit\",\"location\":\"Honolulu, HI\"}}" + ], + "raw_response": [ + { + "content": [ + { + "id": "toolu_01TRx8R2YjRVrobxwmDr4mqR", + "input": { + "format": "fahrenheit", + "location": "Honolulu, HI" + }, + "name": "get_current_weather", + "type": "tool_use" + } + ], + "id": "msg_018PFzotfDuy1VH9KXabt1BB", + "model": "claude-3-5-sonnet-20241022", + "role": "assistant", + "stop_reason": "tool_use", + "stop_sequence": null, + "type": "message", + "usage": { + "input_tokens": 445, + "output_tokens": 64 + } + } + ], + "llm": "Claude", + "vars": { + "prompt": "What is the average temperature in Hawaii in December?" + }, + "metavars": { + "__pt": "{prompt}", + "LLM_0": "Claude" + } + } + ], + "What causes thunderstorms?": [ + { + "prompt": "What causes thunderstorms?", + "query": { + "model": "claude-3-5-sonnet-latest", + "stop_sequences": [ + "\n\nHuman:" + ], + "temperature": 1, + "tools": [ + { + "name": "get_current_weather", + "description": "Get the current weather", + "input_schema": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA" + }, + "format": { + "type": "string", + "enum": [ + "celsius", + "fahrenheit" + ], + "description": "The temperature unit to use. Infer this from the users location." + } + }, + "required": [ + "location", + "format" + ] + } + } + ], + "top_k": -1, + "top_p": -1, + "tool_choice": { + "type": "any", + "disable_parallel_tool_use": false + }, + "max_tokens": 1024, + "messages": [ + { + "role": "user", + "content": "What causes thunderstorms?" + } + ] + }, + "uid": "cd5e5b7f-7cf6-4ab3-91c1-ce9ab6052d07", + "responses": [ + "[[TOOLS]] {\"name\":\"get_current_weather\",\"input\":{\"format\":\"celsius\",\"location\":\"\"}}" + ], + "raw_response": [ + { + "content": [ + { + "id": "toolu_01QXfzP6KDoM4vVCNbzW9Wc3", + "input": { + "format": "celsius", + "location": "" + }, + "name": "get_current_weather", + "type": "tool_use" + } + ], + "id": "msg_01TBX4MH3W35uRwA8B6tSr5q", + "model": "claude-3-5-sonnet-20241022", + "role": "assistant", + "stop_reason": "tool_use", + "stop_sequence": null, + "type": "message", + "usage": { + "input_tokens": 441, + "output_tokens": 58 + } + } + ], + "llm": "Claude", + "vars": { + "prompt": "What causes thunderstorms?" + }, + "metavars": { + "__pt": "{prompt}", + "LLM_0": "Claude" + } + } + ] + }, + "prompt-1686689549227.json": { + "cache_files": { + "prompt-1686689549227_5.json": { + "key": "1725e2ec-cd4b-4211-a73a-3d6ef4482451", + "name": "gpt-4-turbo", + "emoji": "🤖", + "model": "gpt-4-turbo", + "base_model": "gpt-3.5-turbo", + "temp": 1, + "settings": { + "system_msg": "You are a helpful assistant.", + "temperature": 1, + "response_format": { + "type": "text" + }, + "tools": [ + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA" + }, + "format": { + "type": "string", + "enum": [ + "celsius", + "fahrenheit" + ], + "description": "The temperature unit to use. Infer this from the users location." + } + }, + "required": [ + "location", + "format" + ] + } + } + } + ], + "tool_choice": "auto", + "parallel_tool_calls": true, + "top_p": 1, + "stop": [], + "presence_penalty": 0, + "frequency_penalty": 0 + }, + "formData": { + "shortname": "gpt-4-turbo", + "model": "gpt-4-turbo", + "system_msg": "You are a helpful assistant.", + "temperature": 1, + "response_format": "text", + "tools": "[{ \"type\": \"function\",\n \"function\": {\n \"name\": \"get_current_weather\",\n \"description\": \"Get the current weather\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"location\": {\n \"type\": \"string\",\n \"description\": \"The city and state, e.g. San Francisco, CA\"\n },\n \"format\": {\n \"type\": \"string\",\n \"enum\": [\"celsius\", \"fahrenheit\"],\n \"description\": \"The temperature unit to use. Infer this from the users location.\"\n }\n },\n \"required\": [\"location\", \"format\"]\n }\n }\n}]", + "tool_choice": "auto", + "parallel_tool_calls": true, + "top_p": 1, + "stop": "", + "presence_penalty": 0, + "frequency_penalty": 0 + }, + "progress": { + "success": 0, + "error": 0 + } + }, + "prompt-1686689549227_1.json": { + "base_model": "gpt-3.5-turbo", + "emoji": "🙂", + "formData": { + "shortname": "no-tools-gpt3.5", + "model": "gpt-3.5-turbo", + "system_msg": "You are a helpful assistant.", + "temperature": 1, + "response_format": "text", + "tools": "", + "tool_choice": "", + "parallel_tool_calls": true, + "top_p": 1, + "stop": "", + "presence_penalty": 0, + "frequency_penalty": 0, + "function_call": "", + "functions": "" + }, + "key": "fead74ce-615a-4812-869d-9735d9d985de", + "model": "gpt-3.5-turbo", + "name": "no-tools-gpt3.5", + "settings": { + "system_msg": "You are a helpful assistant.", + "temperature": 1, + "response_format": { + "type": "text" + }, + "tools": [], + "tool_choice": "", + "parallel_tool_calls": true, + "top_p": 1, + "stop": [], + "presence_penalty": 0, + "frequency_penalty": 0, + "function_call": "", + "functions": [] }, "temp": 1, "progress": { @@ -2323,30 +4016,68 @@ } }, "prompt-1686689549227_4.json": { - "base_model": "gpt-3.5-turbo", - "emoji": "🙂", - "formData": { - "frequency_penalty": 0, - "function_call": "", - "functions": "", - "model": "gpt-3.5-turbo", - "presence_penalty": 0, - "shortname": "old-gpt3.5", - "stop": "", - "system_msg": "You are a helpful assistant.", - "temperature": 1, - "top_p": 1 - }, - "key": "fead74ce-615a-4812-869d-9735d9d985de", - "model": "gpt-3.5-turbo", - "name": "old-gpt3.5", + "key": "4caee194-fa12-4ca8-bb9e-1c8d7d5f4a08", + "name": "Claude", + "emoji": "📚", + "model": "claude-3-5-sonnet-latest", + "base_model": "claude-v1", + "temp": 0.5, "settings": { - "frequency_penalty": 0, - "presence_penalty": 0, + "system_msg": "", "temperature": 1, - "top_p": 1 + "tools": [ + { + "name": "get_current_weather", + "description": "Get the current weather", + "input_schema": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA" + }, + "format": { + "type": "string", + "enum": [ + "celsius", + "fahrenheit" + ], + "description": "The temperature unit to use. Infer this from the users location." + } + }, + "required": [ + "location", + "format" + ] + } + } + ], + "tool_choice": { + "type": "auto" + }, + "parallel_tool_calls": true, + "max_tokens_to_sample": 1024, + "custom_prompt_wrapper": "", + "stop_sequences": [ + "\n\nHuman:" + ], + "top_k": -1, + "top_p": -1 + }, + "formData": { + "shortname": "Claude", + "model": "claude-3-5-sonnet-latest", + "system_msg": "", + "temperature": 1, + "tools": "[{\n \"name\": \"get_current_weather\",\n \"description\": \"Get the current weather\",\n \"input_schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"location\": {\n \"type\": \"string\",\n \"description\": \"The city and state, e.g. San Francisco, CA\"\n },\n \"format\": {\n \"type\": \"string\",\n \"enum\": [\"celsius\", \"fahrenheit\"],\n \"description\": \"The temperature unit to use. Infer this from the users location.\"\n }\n },\n \"required\": [\"location\", \"format\"]\n }\n }\n]", + "tool_choice": "auto", + "parallel_tool_calls": true, + "max_tokens_to_sample": 1024, + "custom_prompt_wrapper": "", + "stop_sequences": "\"\n\nHuman:\"", + "top_k": -1, + "top_p": -1 }, - "temp": 1, "progress": { "success": 0, "error": 0 @@ -2360,19 +4091,14 @@ }, "metavars": { "__pt": "{prompt}", - "LLM_0": "gpt3.5-turbo-0613" + "LLM_0": "gpt-4-turbo" }, - "llm": "gpt3.5-turbo-0613", + "llm": "gpt-4-turbo", "prompt": "What's the weather like in Boston?", "responses": [ - "[[FUNCTION]] get_current_weather{\n \"format\": \"celsius\",\n \"location\": \"Boston, MA\"\n}" + "[[TOOLS]] get_current_weather {\"location\":\"Boston, MA\",\"format\":\"fahrenheit\"}" ], - "tokens": { - "completion_tokens": 26, - "prompt_tokens": 98, - "total_tokens": 124 - }, - "uid": "cfa8f743-32c7-4a24-8141-9e1fa264af49" + "uid": "2e2d2d3f-d162-4f6a-af6a-23f2628748e5" }, { "vars": { @@ -2380,19 +4106,29 @@ }, "metavars": { "__pt": "{prompt}", - "LLM_0": "old-gpt3.5" + "LLM_0": "no-tools-gpt3.5" }, - "llm": "old-gpt3.5", + "llm": "no-tools-gpt3.5", "prompt": "What's the weather like in Boston?", "responses": [ - "I'm sorry, I cannot provide real-time information as I do not have access to live data. However, you can check reliable weather websites or applications for up-to-date information about the weather in Boston." + "I'm sorry, but I don't have real-time data access. You can check the weather in Boston by using a weather website or app, or by asking a voice assistant like Siri or Google Assistant for the current weather in Boston." ], - "tokens": { - "completion_tokens": 41, - "prompt_tokens": 27, - "total_tokens": 68 + "uid": "64f90d18-939b-4e59-867e-5c433edd5103" + }, + { + "vars": { + "prompt": "What's the weather like in Boston?" }, - "uid": "e342daaf-acde-4988-a759-c1c753efd670" + "metavars": { + "__pt": "{prompt}", + "LLM_0": "Claude" + }, + "llm": "Claude", + "prompt": "What's the weather like in Boston?", + "responses": [ + "[[TOOLS]] {\"name\":\"get_current_weather\",\"input\":{\"format\":\"fahrenheit\",\"location\":\"Boston, MA\"}}" + ], + "uid": "eb2ba7da-fc44-4b3e-9cf9-6ee684e84ac6" }, { "vars": { @@ -2400,19 +4136,14 @@ }, "metavars": { "__pt": "{prompt}", - "LLM_0": "gpt3.5-turbo-0613" + "LLM_0": "gpt-4-turbo" }, - "llm": "gpt3.5-turbo-0613", + "llm": "gpt-4-turbo", "prompt": "Who directed the movie Twister?", "responses": [ "The movie \"Twister\" was directed by Jan de Bont." ], - "tokens": { - "completion_tokens": 15, - "prompt_tokens": 97, - "total_tokens": 112 - }, - "uid": "ebcba322-b11c-4680-8170-98b0f3bebbe5" + "uid": "423a06d4-1111-48a8-9897-5348ea362ebc" }, { "vars": { @@ -2420,19 +4151,29 @@ }, "metavars": { "__pt": "{prompt}", - "LLM_0": "old-gpt3.5" + "LLM_0": "no-tools-gpt3.5" }, - "llm": "old-gpt3.5", + "llm": "no-tools-gpt3.5", "prompt": "Who directed the movie Twister?", "responses": [ - "The movie \"Twister\" was directed by Jan de Bont." + "The movie Twister was directed by Jan de Bont. It was released in 1996 and stars Helen Hunt and Bill Paxton." ], - "tokens": { - "completion_tokens": 14, - "prompt_tokens": 24, - "total_tokens": 38 + "uid": "53b45f6f-12fe-4d2a-998b-4fc199c28c1d" + }, + { + "vars": { + "prompt": "Who directed the movie Twister?" }, - "uid": "1e884f42-d130-4a33-9039-22490772902d" + "metavars": { + "__pt": "{prompt}", + "LLM_0": "Claude" + }, + "llm": "Claude", + "prompt": "Who directed the movie Twister?", + "responses": [ + "[[TOOLS]] {\"name\":\"get_current_weather\",\"input\":{\"format\":\"celsius\",\"location\":\"\"}}" + ], + "uid": "19109992-fdef-479d-92fd-7a48c0b4b527" }, { "vars": { @@ -2440,19 +4181,14 @@ }, "metavars": { "__pt": "{prompt}", - "LLM_0": "gpt3.5-turbo-0613" + "LLM_0": "gpt-4-turbo" }, - "llm": "gpt3.5-turbo-0613", + "llm": "gpt-4-turbo", "prompt": "What causes thunderstorms?", "responses": [ - "Thunderstorms are caused by the rapid upward movement of warm, moist air, which then cools and condenses, forming cumulonimbus clouds. These clouds contain large amounts of water vapor, ice, and electrically charged particles. As the air continues to rise, the water droplets and ice particles within the clouds collide, leading to the development of strong updrafts and downdrafts. The movement of these particles within the cloud generates electrical charges, resulting in the formation of lightning. The rapid expansion and contraction of the air around the lightning bolt creates a shock wave that we hear as thunder." + "Thunderstorms are caused by the combination of moisture, rapidly rising warm air, and a lifting mechanism. Here is a breakdown of the factors and processes involved:\n\n1. **Moisture**: Moisture is necessary to form clouds and rain. It comes primarily from bodies of water like oceans and lakes. Warmer air can hold more moisture than cooler air.\n\n2. **Rapidly Rising Warm Air**: When the surface air is warmed by the sun, it becomes lighter and rises quickly. As the air rises, it cools, and the moisture it carries condenses into a cumulus cloud. If the rising continues and conditions are favorable, these clouds can grow into cumulonimbus clouds, which are the towering clouds typically associated with thunderstorms.\n\n3. **Lifting Mechanism**: A lifting mechanism is needed to help or force the warm air to rise. This can be caused by a variety of geographical and meteorological factors:\n - Frontal systems (where cold and warm air masses collide)\n - Sea breezes\n - Mountains that force the air to rise (orographic lift)\n - Variations in temperature (thermal differences) on the surface caused by features such as urban settings, forests, and crops.\n\nOnce these elements are in place, a thunderstorm can develop. Within the storm, electrical charges become separated, with positive charges often accumulating near the top of the storm and negative charges near the bottom. This charge separation can eventually lead to lightning, which is the sudden neutralization of these charges. The rapid expansion of air heated by the lightning causes thunder.\n\nThroughout a thunderstorm, various types of precipitation (rain, hail) and wind phenomena (gusts, downdrafts) can occur, depending on the intensity and maturity of the storm." ], - "tokens": { - "completion_tokens": 123, - "prompt_tokens": 95, - "total_tokens": 218 - }, - "uid": "ba600f1c-ea3c-4c74-b5e5-bf03739d6d1c" + "uid": "0068affe-862a-4864-b4ef-376913a78d4a" }, { "vars": { @@ -2460,19 +4196,29 @@ }, "metavars": { "__pt": "{prompt}", - "LLM_0": "old-gpt3.5" + "LLM_0": "no-tools-gpt3.5" }, - "llm": "old-gpt3.5", + "llm": "no-tools-gpt3.5", "prompt": "What causes thunderstorms?", "responses": [ - "Thunderstorms are caused by the rapid upward movement of warm, moist air, which creates unstable atmospheric conditions. As the warm air rises, it cools and forms cumulonimbus clouds, which are the towering dark clouds that typically come to mind when thinking of a thunderstorm. Within these clouds, a variety of atmospheric conditions exist that can cause the development of lightning, thunder, heavy rain, hail, and tornadoes. The exact cause of a thunderstorm can vary depending on the location, time of year, and other environmental factors." + "Thunderstorms are caused by the rapid upward movement of warm, moist air that cools and condenses into clouds. This process leads to the formation of cumulonimbus clouds, which can grow to great heights and create unstable atmospheric conditions. As the clouds continue to grow, water droplets and ice particles collide and create an electrical charge. The difference in charge between the clouds and the ground or between different parts of the cloud leads to the discharge of lightning, which is then followed by the sound of thunder. This entire process of rapid uplift, condensation, and electrical discharge is what we experience as a thunderstorm." ], - "tokens": { - "completion_tokens": 108, - "prompt_tokens": 24, - "total_tokens": 132 + "uid": "6056ad0a-a1c6-4c21-9788-f1c9143f6f85" + }, + { + "vars": { + "prompt": "What causes thunderstorms?" }, - "uid": "8c2e8fb8-76dc-455d-8ba7-252f99594b36" + "metavars": { + "__pt": "{prompt}", + "LLM_0": "Claude" + }, + "llm": "Claude", + "prompt": "What causes thunderstorms?", + "responses": [ + "[[TOOLS]] {\"name\":\"get_current_weather\",\"input\":{\"format\":\"celsius\",\"location\":\"\"}}" + ], + "uid": "cd5e5b7f-7cf6-4ab3-91c1-ce9ab6052d07" }, { "vars": { @@ -2480,19 +4226,14 @@ }, "metavars": { "__pt": "{prompt}", - "LLM_0": "gpt3.5-turbo-0613" + "LLM_0": "gpt-4-turbo" }, - "llm": "gpt3.5-turbo-0613", + "llm": "gpt-4-turbo", "prompt": "What is the typical temperature in Kenya in June?", "responses": [ - "[[FUNCTION]] get_current_weather{\n \"format\": \"celsius\",\n \"location\": \"Kenya\"\n}" + "In June, Kenya typically experiences variable temperatures depending on the region. Due to its diverse geography, which includes coastal areas, highlands, and savannahs, temperatures can differ significantly:\n\n1. **Coastal areas (e.g., Mombasa):** The temperatures are warmer with highs around 28°C (82°F) and lows about 20°C (68°F).\n \n2. **Highlands (e.g., Nairobi):** Cooler temperatures prevail, with daytime highs averaging 21°C (70°F) and nights around 12°C (54°F).\n\n3. **Western and Lake Victoria region:** Warm and humid with temperatures similar to the coastal areas but can be a bit cooler especially during the night.\n\nJune marks the start of the cooler season in Kenya, making it a pleasant time to visit, especially for those who prefer milder weather away from the heat and humidity typical of tropical climates." ], - "tokens": { - "completion_tokens": 25, - "prompt_tokens": 100, - "total_tokens": 125 - }, - "uid": "2a8ff935-4c25-4222-8d37-22634b8c6208" + "uid": "b3e061b4-b81d-4904-97b3-5627d4e64824" }, { "vars": { @@ -2500,19 +4241,29 @@ }, "metavars": { "__pt": "{prompt}", - "LLM_0": "old-gpt3.5" + "LLM_0": "no-tools-gpt3.5" }, - "llm": "old-gpt3.5", + "llm": "no-tools-gpt3.5", "prompt": "What is the typical temperature in Kenya in June?", "responses": [ - "The typical temperature in Kenya in June varies depending on the region, but generally, temperatures range from 19°C (66°F) to 30°C (86°F). In some parts of the country, particularly in areas of higher elevation, temperatures may be cooler, while in areas closer to the equator temperatures can be quite hot." + "In Kenya, June is during the dry season, with temperatures averaging around 18-23°C (64-73°F) in the central highlands and around 24-30°C (75-86°F) along the coast and in lower elevations. However, temperatures can vary depending on the specific region and altitude within the country." ], - "tokens": { - "completion_tokens": 67, - "prompt_tokens": 29, - "total_tokens": 96 + "uid": "7f84b658-031a-4bc8-9e2f-37e1db9b21e0" + }, + { + "vars": { + "prompt": "What is the typical temperature in Kenya in June?" }, - "uid": "a993b28a-df28-462e-9e2b-e7afd6742824" + "metavars": { + "__pt": "{prompt}", + "LLM_0": "Claude" + }, + "llm": "Claude", + "prompt": "What is the typical temperature in Kenya in June?", + "responses": [ + "[[TOOLS]] {\"name\":\"get_current_weather\",\"input\":{\"format\":\"celsius\",\"location\":\"Nairobi, Kenya\"}}" + ], + "uid": "eac6af65-6e4c-40ac-a8ad-b07e13c36ac5" }, { "vars": { @@ -2520,19 +4271,14 @@ }, "metavars": { "__pt": "{prompt}", - "LLM_0": "gpt3.5-turbo-0613" + "LLM_0": "gpt-4-turbo" }, - "llm": "gpt3.5-turbo-0613", + "llm": "gpt-4-turbo", "prompt": "What is the average temperature in Hawaii in December?", "responses": [ - "[[FUNCTION]] get_current_weather{\n \"format\": \"celsius\",\n \"location\": \"Honolulu, HI\"\n}" + "The average temperature in Hawaii in December typically ranges from about 68°F (20°C) to 81°F (27°C). This offers pleasant, warm weather which is why Hawaii is such a popular winter destination. Local variations across the different islands can, of course, result in slightly different temperatures due to geography and elevation." ], - "tokens": { - "completion_tokens": 27, - "prompt_tokens": 100, - "total_tokens": 127 - }, - "uid": "4b818464-a297-429e-b081-19c61b12bae7" + "uid": "de3fd081-62b5-4943-bd4d-36987a43004d" }, { "vars": { @@ -2540,19 +4286,29 @@ }, "metavars": { "__pt": "{prompt}", - "LLM_0": "old-gpt3.5" + "LLM_0": "no-tools-gpt3.5" }, - "llm": "old-gpt3.5", + "llm": "no-tools-gpt3.5", "prompt": "What is the average temperature in Hawaii in December?", "responses": [ - "The average temperature in Hawaii in December is around 77°F (25°C) during the day and 66°F (19°C) at night." + "In December, the average temperature in Hawaii typically ranges from 75°F to 80°F (24°C to 27°C). However, the exact temperature can vary depending on the specific island and location within Hawaii." ], - "tokens": { - "completion_tokens": 30, - "prompt_tokens": 29, - "total_tokens": 59 + "uid": "f3aa7139-def5-4397-8b87-ee075ffe315f" + }, + { + "vars": { + "prompt": "What is the average temperature in Hawaii in December?" }, - "uid": "426b0d32-7afd-46a8-a9db-b0e893e7e3b7" + "metavars": { + "__pt": "{prompt}", + "LLM_0": "Claude" + }, + "llm": "Claude", + "prompt": "What is the average temperature in Hawaii in December?", + "responses": [ + "[[TOOLS]] {\"name\":\"get_current_weather\",\"input\":{\"format\":\"fahrenheit\",\"location\":\"Honolulu, HI\"}}" + ], + "uid": "58de1cb7-3514-4ecc-ae42-6e0900b91ff5" } ] }, @@ -2566,70 +4322,81 @@ }, "metavars": { "__pt": "{prompt}", - "LLM_0": "gpt3.5-turbo-0613" + "LLM_0": "gpt-4-turbo" }, - "uid": "cfa8f743-32c7-4a24-8141-9e1fa264af49", + "uid": "2e2d2d3f-d162-4f6a-af6a-23f2628748e5", "llm": { + "key": "1725e2ec-cd4b-4211-a73a-3d6ef4482451", + "name": "gpt-4-turbo", + "emoji": "🤖", + "model": "gpt-4-turbo", "base_model": "gpt-3.5-turbo", - "emoji": "🌦️", - "formData": { - "frequency_penalty": 0, - "function_call": "auto", - "functions": "[\n {\n \"name\": \"get_current_weather\",\n \"description\": \"Get the current weather\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"location\": {\n \"type\": \"string\",\n \"description\": \"The city and state, e.g. San Francisco, CA\"\n },\n \"format\": {\n \"type\": \"string\",\n \"enum\": [\"celsius\", \"fahrenheit\"],\n \"description\": \"The temperature unit to use. Infer this from the users location.\"\n }\n },\n \"required\": [\"location\", \"format\"]\n }\n }\n]", - "model": "gpt-3.5-turbo-0613", - "presence_penalty": 0, - "shortname": "gpt3.5-turbo-0613", - "stop": "", + "temp": 1, + "settings": { "system_msg": "You are a helpful assistant.", "temperature": 1, - "top_p": 1 - }, - "key": "af7031af-96ae-44b8-bcc4-3538f3174b73", - "model": "gpt-3.5-turbo-0613", - "name": "gpt3.5-turbo-0613", - "settings": { - "frequency_penalty": 0, - "function_call": "auto", - "functions": [ + "response_format": { + "type": "text" + }, + "tools": [ { - "description": "Get the current weather", - "name": "get_current_weather", - "parameters": { - "properties": { - "format": { - "description": "The temperature unit to use. Infer this from the users location.", - "enum": [ - "celsius", - "fahrenheit" - ], - "type": "string" + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA" + }, + "format": { + "type": "string", + "enum": [ + "celsius", + "fahrenheit" + ], + "description": "The temperature unit to use. Infer this from the users location." + } }, - "location": { - "description": "The city and state, e.g. San Francisco, CA", - "type": "string" - } - }, - "required": [ - "location", - "format" - ], - "type": "object" + "required": [ + "location", + "format" + ] + } } } ], + "tool_choice": "auto", + "parallel_tool_calls": true, + "top_p": 1, + "stop": [], "presence_penalty": 0, - "temperature": 1, - "top_p": 1 + "frequency_penalty": 0 + }, + "formData": { + "shortname": "gpt-4-turbo", + "model": "gpt-4-turbo", + "system_msg": "You are a helpful assistant.", + "temperature": 1, + "response_format": "text", + "tools": "[{ \"type\": \"function\",\n \"function\": {\n \"name\": \"get_current_weather\",\n \"description\": \"Get the current weather\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"location\": {\n \"type\": \"string\",\n \"description\": \"The city and state, e.g. San Francisco, CA\"\n },\n \"format\": {\n \"type\": \"string\",\n \"enum\": [\"celsius\", \"fahrenheit\"],\n \"description\": \"The temperature unit to use. Infer this from the users location.\"\n }\n },\n \"required\": [\"location\", \"format\"]\n }\n }\n}]", + "tool_choice": "auto", + "parallel_tool_calls": true, + "top_p": 1, + "stop": "", + "presence_penalty": 0, + "frequency_penalty": 0 }, - "temp": 1, "progress": { - "success": 0, + "success": 80, "error": 0 } }, "prompt": "What's the weather like in Boston?", "responses": [ - "[[FUNCTION]] get_current_weather\\{\n \"format\": \"celsius\",\n \"location\": \"Boston, MA\"\n\\}" + "[[TOOLS]] get_current_weather {\"location\":\"Boston, MA\",\"format\":\"fahrenheit\"}" ], "tokens": {}, "eval_res": { @@ -2645,42 +4412,56 @@ }, "metavars": { "__pt": "{prompt}", - "LLM_0": "old-gpt3.5" + "LLM_0": "no-tools-gpt3.5" }, - "uid": "e342daaf-acde-4988-a759-c1c753efd670", + "uid": "64f90d18-939b-4e59-867e-5c433edd5103", "llm": { "base_model": "gpt-3.5-turbo", "emoji": "🙂", "formData": { - "frequency_penalty": 0, - "function_call": "", - "functions": "", + "shortname": "no-tools-gpt3.5", "model": "gpt-3.5-turbo", - "presence_penalty": 0, - "shortname": "old-gpt3.5", - "stop": "", "system_msg": "You are a helpful assistant.", "temperature": 1, - "top_p": 1 + "response_format": "text", + "tools": "", + "tool_choice": "", + "parallel_tool_calls": true, + "top_p": 1, + "stop": "", + "presence_penalty": 0, + "frequency_penalty": 0, + "function_call": "", + "functions": "" }, "key": "fead74ce-615a-4812-869d-9735d9d985de", "model": "gpt-3.5-turbo", - "name": "old-gpt3.5", + "name": "no-tools-gpt3.5", "settings": { - "frequency_penalty": 0, - "presence_penalty": 0, + "system_msg": "You are a helpful assistant.", "temperature": 1, - "top_p": 1 + "response_format": { + "type": "text" + }, + "tools": [], + "tool_choice": "", + "parallel_tool_calls": true, + "top_p": 1, + "stop": [], + "presence_penalty": 0, + "frequency_penalty": 0, + "function_call": "", + "functions": [] }, "temp": 1, "progress": { - "success": 0, + "success": 100, "error": 0 } }, "prompt": "What's the weather like in Boston?", "responses": [ - "I'm sorry, I cannot provide real-time information as I do not have access to live data. However, you can check reliable weather websites or applications for up-to-date information about the weather in Boston." + "I'm sorry, but I don't have real-time data access. You can check the weather in Boston by using a weather website or app, or by asking a voice assistant like Siri or Google Assistant for the current weather in Boston." ], "tokens": {}, "eval_res": { @@ -2692,68 +4473,168 @@ }, { "vars": { - "prompt": "Who directed the movie Twister?" + "prompt": "What's the weather like in Boston?" }, "metavars": { "__pt": "{prompt}", - "LLM_0": "gpt3.5-turbo-0613" + "LLM_0": "Claude" }, - "uid": "ebcba322-b11c-4680-8170-98b0f3bebbe5", + "uid": "eb2ba7da-fc44-4b3e-9cf9-6ee684e84ac6", "llm": { - "base_model": "gpt-3.5-turbo", - "emoji": "🌦️", - "formData": { - "frequency_penalty": 0, - "function_call": "auto", - "functions": "[\n {\n \"name\": \"get_current_weather\",\n \"description\": \"Get the current weather\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"location\": {\n \"type\": \"string\",\n \"description\": \"The city and state, e.g. San Francisco, CA\"\n },\n \"format\": {\n \"type\": \"string\",\n \"enum\": [\"celsius\", \"fahrenheit\"],\n \"description\": \"The temperature unit to use. Infer this from the users location.\"\n }\n },\n \"required\": [\"location\", \"format\"]\n }\n }\n]", - "model": "gpt-3.5-turbo-0613", - "presence_penalty": 0, - "shortname": "gpt3.5-turbo-0613", - "stop": "", - "system_msg": "You are a helpful assistant.", - "temperature": 1, - "top_p": 1 - }, - "key": "af7031af-96ae-44b8-bcc4-3538f3174b73", - "model": "gpt-3.5-turbo-0613", - "name": "gpt3.5-turbo-0613", + "key": "4caee194-fa12-4ca8-bb9e-1c8d7d5f4a08", + "name": "Claude", + "emoji": "📚", + "model": "claude-3-5-sonnet-latest", + "base_model": "claude-v1", + "temp": 0.5, "settings": { - "frequency_penalty": 0, - "function_call": "auto", - "functions": [ + "system_msg": "", + "temperature": 1, + "tools": [ { - "description": "Get the current weather", "name": "get_current_weather", - "parameters": { + "description": "Get the current weather", + "input_schema": { + "type": "object", "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA" + }, "format": { - "description": "The temperature unit to use. Infer this from the users location.", + "type": "string", "enum": [ "celsius", "fahrenheit" ], - "type": "string" - }, - "location": { - "description": "The city and state, e.g. San Francisco, CA", - "type": "string" + "description": "The temperature unit to use. Infer this from the users location." } }, "required": [ "location", "format" - ], - "type": "object" + ] } } ], - "presence_penalty": 0, - "temperature": 1, - "top_p": 1 + "tool_choice": { + "type": "auto" + }, + "parallel_tool_calls": true, + "max_tokens_to_sample": 1024, + "custom_prompt_wrapper": "", + "stop_sequences": [ + "\n\nHuman:" + ], + "top_k": -1, + "top_p": -1 + }, + "formData": { + "shortname": "Claude", + "model": "claude-3-5-sonnet-latest", + "system_msg": "", + "temperature": 1, + "tools": "[{\n \"name\": \"get_current_weather\",\n \"description\": \"Get the current weather\",\n \"input_schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"location\": {\n \"type\": \"string\",\n \"description\": \"The city and state, e.g. San Francisco, CA\"\n },\n \"format\": {\n \"type\": \"string\",\n \"enum\": [\"celsius\", \"fahrenheit\"],\n \"description\": \"The temperature unit to use. Infer this from the users location.\"\n }\n },\n \"required\": [\"location\", \"format\"]\n }\n }\n]", + "tool_choice": "auto", + "parallel_tool_calls": true, + "max_tokens_to_sample": 1024, + "custom_prompt_wrapper": "", + "stop_sequences": "\"\n\nHuman:\"", + "top_k": -1, + "top_p": -1 }, - "temp": 1, "progress": { - "success": 0, + "success": 100, + "error": 0 + } + }, + "prompt": "What's the weather like in Boston?", + "responses": [ + "[[TOOLS]] {\"name\":\"get_current_weather\",\"input\":{\"format\":\"fahrenheit\",\"location\":\"Boston, MA\"}}" + ], + "tokens": {}, + "eval_res": { + "items": [ + true + ], + "dtype": "Categorical" + } + }, + { + "vars": { + "prompt": "Who directed the movie Twister?" + }, + "metavars": { + "__pt": "{prompt}", + "LLM_0": "gpt-4-turbo" + }, + "uid": "423a06d4-1111-48a8-9897-5348ea362ebc", + "llm": { + "key": "1725e2ec-cd4b-4211-a73a-3d6ef4482451", + "name": "gpt-4-turbo", + "emoji": "🤖", + "model": "gpt-4-turbo", + "base_model": "gpt-3.5-turbo", + "temp": 1, + "settings": { + "system_msg": "You are a helpful assistant.", + "temperature": 1, + "response_format": { + "type": "text" + }, + "tools": [ + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA" + }, + "format": { + "type": "string", + "enum": [ + "celsius", + "fahrenheit" + ], + "description": "The temperature unit to use. Infer this from the users location." + } + }, + "required": [ + "location", + "format" + ] + } + } + } + ], + "tool_choice": "auto", + "parallel_tool_calls": true, + "top_p": 1, + "stop": [], + "presence_penalty": 0, + "frequency_penalty": 0 + }, + "formData": { + "shortname": "gpt-4-turbo", + "model": "gpt-4-turbo", + "system_msg": "You are a helpful assistant.", + "temperature": 1, + "response_format": "text", + "tools": "[{ \"type\": \"function\",\n \"function\": {\n \"name\": \"get_current_weather\",\n \"description\": \"Get the current weather\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"location\": {\n \"type\": \"string\",\n \"description\": \"The city and state, e.g. San Francisco, CA\"\n },\n \"format\": {\n \"type\": \"string\",\n \"enum\": [\"celsius\", \"fahrenheit\"],\n \"description\": \"The temperature unit to use. Infer this from the users location.\"\n }\n },\n \"required\": [\"location\", \"format\"]\n }\n }\n}]", + "tool_choice": "auto", + "parallel_tool_calls": true, + "top_p": 1, + "stop": "", + "presence_penalty": 0, + "frequency_penalty": 0 + }, + "progress": { + "success": 80, "error": 0 } }, @@ -2775,42 +4656,235 @@ }, "metavars": { "__pt": "{prompt}", - "LLM_0": "old-gpt3.5" + "LLM_0": "no-tools-gpt3.5" }, - "uid": "1e884f42-d130-4a33-9039-22490772902d", + "uid": "53b45f6f-12fe-4d2a-998b-4fc199c28c1d", "llm": { "base_model": "gpt-3.5-turbo", "emoji": "🙂", "formData": { - "frequency_penalty": 0, - "function_call": "", - "functions": "", + "shortname": "no-tools-gpt3.5", "model": "gpt-3.5-turbo", - "presence_penalty": 0, - "shortname": "old-gpt3.5", - "stop": "", "system_msg": "You are a helpful assistant.", "temperature": 1, - "top_p": 1 + "response_format": "text", + "tools": "", + "tool_choice": "", + "parallel_tool_calls": true, + "top_p": 1, + "stop": "", + "presence_penalty": 0, + "frequency_penalty": 0, + "function_call": "", + "functions": "" }, "key": "fead74ce-615a-4812-869d-9735d9d985de", "model": "gpt-3.5-turbo", - "name": "old-gpt3.5", + "name": "no-tools-gpt3.5", "settings": { - "frequency_penalty": 0, - "presence_penalty": 0, + "system_msg": "You are a helpful assistant.", "temperature": 1, - "top_p": 1 + "response_format": { + "type": "text" + }, + "tools": [], + "tool_choice": "", + "parallel_tool_calls": true, + "top_p": 1, + "stop": [], + "presence_penalty": 0, + "frequency_penalty": 0, + "function_call": "", + "functions": [] }, "temp": 1, "progress": { - "success": 0, + "success": 100, "error": 0 } }, "prompt": "Who directed the movie Twister?", "responses": [ - "The movie \"Twister\" was directed by Jan de Bont." + "The movie Twister was directed by Jan de Bont. It was released in 1996 and stars Helen Hunt and Bill Paxton." + ], + "tokens": {}, + "eval_res": { + "items": [ + false + ], + "dtype": "Categorical" + } + }, + { + "vars": { + "prompt": "Who directed the movie Twister?" + }, + "metavars": { + "__pt": "{prompt}", + "LLM_0": "Claude" + }, + "uid": "19109992-fdef-479d-92fd-7a48c0b4b527", + "llm": { + "key": "4caee194-fa12-4ca8-bb9e-1c8d7d5f4a08", + "name": "Claude", + "emoji": "📚", + "model": "claude-3-5-sonnet-latest", + "base_model": "claude-v1", + "temp": 0.5, + "settings": { + "system_msg": "", + "temperature": 1, + "tools": [ + { + "name": "get_current_weather", + "description": "Get the current weather", + "input_schema": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA" + }, + "format": { + "type": "string", + "enum": [ + "celsius", + "fahrenheit" + ], + "description": "The temperature unit to use. Infer this from the users location." + } + }, + "required": [ + "location", + "format" + ] + } + } + ], + "tool_choice": { + "type": "auto" + }, + "parallel_tool_calls": true, + "max_tokens_to_sample": 1024, + "custom_prompt_wrapper": "", + "stop_sequences": [ + "\n\nHuman:" + ], + "top_k": -1, + "top_p": -1 + }, + "formData": { + "shortname": "Claude", + "model": "claude-3-5-sonnet-latest", + "system_msg": "", + "temperature": 1, + "tools": "[{\n \"name\": \"get_current_weather\",\n \"description\": \"Get the current weather\",\n \"input_schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"location\": {\n \"type\": \"string\",\n \"description\": \"The city and state, e.g. San Francisco, CA\"\n },\n \"format\": {\n \"type\": \"string\",\n \"enum\": [\"celsius\", \"fahrenheit\"],\n \"description\": \"The temperature unit to use. Infer this from the users location.\"\n }\n },\n \"required\": [\"location\", \"format\"]\n }\n }\n]", + "tool_choice": "auto", + "parallel_tool_calls": true, + "max_tokens_to_sample": 1024, + "custom_prompt_wrapper": "", + "stop_sequences": "\"\n\nHuman:\"", + "top_k": -1, + "top_p": -1 + }, + "progress": { + "success": 100, + "error": 0 + } + }, + "prompt": "Who directed the movie Twister?", + "responses": [ + "[[TOOLS]] {\"name\":\"get_current_weather\",\"input\":{\"format\":\"celsius\",\"location\":\"\"}}" + ], + "tokens": {}, + "eval_res": { + "items": [ + true + ], + "dtype": "Categorical" + } + }, + { + "vars": { + "prompt": "What causes thunderstorms?" + }, + "metavars": { + "__pt": "{prompt}", + "LLM_0": "gpt-4-turbo" + }, + "uid": "0068affe-862a-4864-b4ef-376913a78d4a", + "llm": { + "key": "1725e2ec-cd4b-4211-a73a-3d6ef4482451", + "name": "gpt-4-turbo", + "emoji": "🤖", + "model": "gpt-4-turbo", + "base_model": "gpt-3.5-turbo", + "temp": 1, + "settings": { + "system_msg": "You are a helpful assistant.", + "temperature": 1, + "response_format": { + "type": "text" + }, + "tools": [ + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA" + }, + "format": { + "type": "string", + "enum": [ + "celsius", + "fahrenheit" + ], + "description": "The temperature unit to use. Infer this from the users location." + } + }, + "required": [ + "location", + "format" + ] + } + } + } + ], + "tool_choice": "auto", + "parallel_tool_calls": true, + "top_p": 1, + "stop": [], + "presence_penalty": 0, + "frequency_penalty": 0 + }, + "formData": { + "shortname": "gpt-4-turbo", + "model": "gpt-4-turbo", + "system_msg": "You are a helpful assistant.", + "temperature": 1, + "response_format": "text", + "tools": "[{ \"type\": \"function\",\n \"function\": {\n \"name\": \"get_current_weather\",\n \"description\": \"Get the current weather\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"location\": {\n \"type\": \"string\",\n \"description\": \"The city and state, e.g. San Francisco, CA\"\n },\n \"format\": {\n \"type\": \"string\",\n \"enum\": [\"celsius\", \"fahrenheit\"],\n \"description\": \"The temperature unit to use. Infer this from the users location.\"\n }\n },\n \"required\": [\"location\", \"format\"]\n }\n }\n}]", + "tool_choice": "auto", + "parallel_tool_calls": true, + "top_p": 1, + "stop": "", + "presence_penalty": 0, + "frequency_penalty": 0 + }, + "progress": { + "success": 80, + "error": 0 + } + }, + "prompt": "What causes thunderstorms?", + "responses": [ + "Thunderstorms are caused by the combination of moisture, rapidly rising warm air, and a lifting mechanism. Here is a breakdown of the factors and processes involved:\n\n1. **Moisture**: Moisture is necessary to form clouds and rain. It comes primarily from bodies of water like oceans and lakes. Warmer air can hold more moisture than cooler air.\n\n2. **Rapidly Rising Warm Air**: When the surface air is warmed by the sun, it becomes lighter and rises quickly. As the air rises, it cools, and the moisture it carries condenses into a cumulus cloud. If the rising continues and conditions are favorable, these clouds can grow into cumulonimbus clouds, which are the towering clouds typically associated with thunderstorms.\n\n3. **Lifting Mechanism**: A lifting mechanism is needed to help or force the warm air to rise. This can be caused by a variety of geographical and meteorological factors:\n - Frontal systems (where cold and warm air masses collide)\n - Sea breezes\n - Mountains that force the air to rise (orographic lift)\n - Variations in temperature (thermal differences) on the surface caused by features such as urban settings, forests, and crops.\n\nOnce these elements are in place, a thunderstorm can develop. Within the storm, electrical charges become separated, with positive charges often accumulating near the top of the storm and negative charges near the bottom. This charge separation can eventually lead to lightning, which is the sudden neutralization of these charges. The rapid expansion of air heated by the lightning causes thunder.\n\nThroughout a thunderstorm, various types of precipitation (rain, hail) and wind phenomena (gusts, downdrafts) can occur, depending on the intensity and maturity of the storm." ], "tokens": {}, "eval_res": { @@ -2826,70 +4900,56 @@ }, "metavars": { "__pt": "{prompt}", - "LLM_0": "gpt3.5-turbo-0613" + "LLM_0": "no-tools-gpt3.5" }, - "uid": "ba600f1c-ea3c-4c74-b5e5-bf03739d6d1c", + "uid": "6056ad0a-a1c6-4c21-9788-f1c9143f6f85", "llm": { "base_model": "gpt-3.5-turbo", - "emoji": "🌦️", + "emoji": "🙂", "formData": { - "frequency_penalty": 0, - "function_call": "auto", - "functions": "[\n {\n \"name\": \"get_current_weather\",\n \"description\": \"Get the current weather\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"location\": {\n \"type\": \"string\",\n \"description\": \"The city and state, e.g. San Francisco, CA\"\n },\n \"format\": {\n \"type\": \"string\",\n \"enum\": [\"celsius\", \"fahrenheit\"],\n \"description\": \"The temperature unit to use. Infer this from the users location.\"\n }\n },\n \"required\": [\"location\", \"format\"]\n }\n }\n]", - "model": "gpt-3.5-turbo-0613", - "presence_penalty": 0, - "shortname": "gpt3.5-turbo-0613", - "stop": "", + "shortname": "no-tools-gpt3.5", + "model": "gpt-3.5-turbo", "system_msg": "You are a helpful assistant.", "temperature": 1, - "top_p": 1 - }, - "key": "af7031af-96ae-44b8-bcc4-3538f3174b73", - "model": "gpt-3.5-turbo-0613", - "name": "gpt3.5-turbo-0613", - "settings": { - "frequency_penalty": 0, - "function_call": "auto", - "functions": [ - { - "description": "Get the current weather", - "name": "get_current_weather", - "parameters": { - "properties": { - "format": { - "description": "The temperature unit to use. Infer this from the users location.", - "enum": [ - "celsius", - "fahrenheit" - ], - "type": "string" - }, - "location": { - "description": "The city and state, e.g. San Francisco, CA", - "type": "string" - } - }, - "required": [ - "location", - "format" - ], - "type": "object" - } - } - ], + "response_format": "text", + "tools": "", + "tool_choice": "", + "parallel_tool_calls": true, + "top_p": 1, + "stop": "", "presence_penalty": 0, + "frequency_penalty": 0, + "function_call": "", + "functions": "" + }, + "key": "fead74ce-615a-4812-869d-9735d9d985de", + "model": "gpt-3.5-turbo", + "name": "no-tools-gpt3.5", + "settings": { + "system_msg": "You are a helpful assistant.", "temperature": 1, - "top_p": 1 + "response_format": { + "type": "text" + }, + "tools": [], + "tool_choice": "", + "parallel_tool_calls": true, + "top_p": 1, + "stop": [], + "presence_penalty": 0, + "frequency_penalty": 0, + "function_call": "", + "functions": [] }, "temp": 1, "progress": { - "success": 0, + "success": 100, "error": 0 } }, "prompt": "What causes thunderstorms?", "responses": [ - "Thunderstorms are caused by the rapid upward movement of warm, moist air, which then cools and condenses, forming cumulonimbus clouds. These clouds contain large amounts of water vapor, ice, and electrically charged particles. As the air continues to rise, the water droplets and ice particles within the clouds collide, leading to the development of strong updrafts and downdrafts. The movement of these particles within the cloud generates electrical charges, resulting in the formation of lightning. The rapid expansion and contraction of the air around the lightning bolt creates a shock wave that we hear as thunder." + "Thunderstorms are caused by the rapid upward movement of warm, moist air that cools and condenses into clouds. This process leads to the formation of cumulonimbus clouds, which can grow to great heights and create unstable atmospheric conditions. As the clouds continue to grow, water droplets and ice particles collide and create an electrical charge. The difference in charge between the clouds and the ground or between different parts of the cloud leads to the discharge of lightning, which is then followed by the sound of thunder. This entire process of rapid uplift, condensation, and electrical discharge is what we experience as a thunderstorm." ], "tokens": {}, "eval_res": { @@ -2905,121 +4965,80 @@ }, "metavars": { "__pt": "{prompt}", - "LLM_0": "old-gpt3.5" + "LLM_0": "Claude" }, - "uid": "8c2e8fb8-76dc-455d-8ba7-252f99594b36", + "uid": "cd5e5b7f-7cf6-4ab3-91c1-ce9ab6052d07", "llm": { - "base_model": "gpt-3.5-turbo", - "emoji": "🙂", - "formData": { - "frequency_penalty": 0, - "function_call": "", - "functions": "", - "model": "gpt-3.5-turbo", - "presence_penalty": 0, - "shortname": "old-gpt3.5", - "stop": "", - "system_msg": "You are a helpful assistant.", - "temperature": 1, - "top_p": 1 - }, - "key": "fead74ce-615a-4812-869d-9735d9d985de", - "model": "gpt-3.5-turbo", - "name": "old-gpt3.5", + "key": "4caee194-fa12-4ca8-bb9e-1c8d7d5f4a08", + "name": "Claude", + "emoji": "📚", + "model": "claude-3-5-sonnet-latest", + "base_model": "claude-v1", + "temp": 0.5, "settings": { - "frequency_penalty": 0, - "presence_penalty": 0, + "system_msg": "", "temperature": 1, - "top_p": 1 + "tools": [ + { + "name": "get_current_weather", + "description": "Get the current weather", + "input_schema": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA" + }, + "format": { + "type": "string", + "enum": [ + "celsius", + "fahrenheit" + ], + "description": "The temperature unit to use. Infer this from the users location." + } + }, + "required": [ + "location", + "format" + ] + } + } + ], + "tool_choice": { + "type": "auto" + }, + "parallel_tool_calls": true, + "max_tokens_to_sample": 1024, + "custom_prompt_wrapper": "", + "stop_sequences": [ + "\n\nHuman:" + ], + "top_k": -1, + "top_p": -1 + }, + "formData": { + "shortname": "Claude", + "model": "claude-3-5-sonnet-latest", + "system_msg": "", + "temperature": 1, + "tools": "[{\n \"name\": \"get_current_weather\",\n \"description\": \"Get the current weather\",\n \"input_schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"location\": {\n \"type\": \"string\",\n \"description\": \"The city and state, e.g. San Francisco, CA\"\n },\n \"format\": {\n \"type\": \"string\",\n \"enum\": [\"celsius\", \"fahrenheit\"],\n \"description\": \"The temperature unit to use. Infer this from the users location.\"\n }\n },\n \"required\": [\"location\", \"format\"]\n }\n }\n]", + "tool_choice": "auto", + "parallel_tool_calls": true, + "max_tokens_to_sample": 1024, + "custom_prompt_wrapper": "", + "stop_sequences": "\"\n\nHuman:\"", + "top_k": -1, + "top_p": -1 }, - "temp": 1, "progress": { - "success": 0, + "success": 100, "error": 0 } }, "prompt": "What causes thunderstorms?", "responses": [ - "Thunderstorms are caused by the rapid upward movement of warm, moist air, which creates unstable atmospheric conditions. As the warm air rises, it cools and forms cumulonimbus clouds, which are the towering dark clouds that typically come to mind when thinking of a thunderstorm. Within these clouds, a variety of atmospheric conditions exist that can cause the development of lightning, thunder, heavy rain, hail, and tornadoes. The exact cause of a thunderstorm can vary depending on the location, time of year, and other environmental factors." - ], - "tokens": {}, - "eval_res": { - "items": [ - false - ], - "dtype": "Categorical" - } - }, - { - "vars": { - "prompt": "What is the typical temperature in Kenya in June?" - }, - "metavars": { - "__pt": "{prompt}", - "LLM_0": "gpt3.5-turbo-0613" - }, - "uid": "2a8ff935-4c25-4222-8d37-22634b8c6208", - "llm": { - "base_model": "gpt-3.5-turbo", - "emoji": "🌦️", - "formData": { - "frequency_penalty": 0, - "function_call": "auto", - "functions": "[\n {\n \"name\": \"get_current_weather\",\n \"description\": \"Get the current weather\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"location\": {\n \"type\": \"string\",\n \"description\": \"The city and state, e.g. San Francisco, CA\"\n },\n \"format\": {\n \"type\": \"string\",\n \"enum\": [\"celsius\", \"fahrenheit\"],\n \"description\": \"The temperature unit to use. Infer this from the users location.\"\n }\n },\n \"required\": [\"location\", \"format\"]\n }\n }\n]", - "model": "gpt-3.5-turbo-0613", - "presence_penalty": 0, - "shortname": "gpt3.5-turbo-0613", - "stop": "", - "system_msg": "You are a helpful assistant.", - "temperature": 1, - "top_p": 1 - }, - "key": "af7031af-96ae-44b8-bcc4-3538f3174b73", - "model": "gpt-3.5-turbo-0613", - "name": "gpt3.5-turbo-0613", - "settings": { - "frequency_penalty": 0, - "function_call": "auto", - "functions": [ - { - "description": "Get the current weather", - "name": "get_current_weather", - "parameters": { - "properties": { - "format": { - "description": "The temperature unit to use. Infer this from the users location.", - "enum": [ - "celsius", - "fahrenheit" - ], - "type": "string" - }, - "location": { - "description": "The city and state, e.g. San Francisco, CA", - "type": "string" - } - }, - "required": [ - "location", - "format" - ], - "type": "object" - } - } - ], - "presence_penalty": 0, - "temperature": 1, - "top_p": 1 - }, - "temp": 1, - "progress": { - "success": 0, - "error": 0 - } - }, - "prompt": "What is the typical temperature in Kenya in June?", - "responses": [ - "[[FUNCTION]] get_current_weather\\{\n \"format\": \"celsius\",\n \"location\": \"Kenya\"\n\\}" + "[[TOOLS]] {\"name\":\"get_current_weather\",\"input\":{\"format\":\"celsius\",\"location\":\"\"}}" ], "tokens": {}, "eval_res": { @@ -3035,42 +5054,81 @@ }, "metavars": { "__pt": "{prompt}", - "LLM_0": "old-gpt3.5" + "LLM_0": "gpt-4-turbo" }, - "uid": "a993b28a-df28-462e-9e2b-e7afd6742824", + "uid": "b3e061b4-b81d-4904-97b3-5627d4e64824", "llm": { + "key": "1725e2ec-cd4b-4211-a73a-3d6ef4482451", + "name": "gpt-4-turbo", + "emoji": "🤖", + "model": "gpt-4-turbo", "base_model": "gpt-3.5-turbo", - "emoji": "🙂", - "formData": { - "frequency_penalty": 0, - "function_call": "", - "functions": "", - "model": "gpt-3.5-turbo", - "presence_penalty": 0, - "shortname": "old-gpt3.5", - "stop": "", + "temp": 1, + "settings": { "system_msg": "You are a helpful assistant.", "temperature": 1, - "top_p": 1 - }, - "key": "fead74ce-615a-4812-869d-9735d9d985de", - "model": "gpt-3.5-turbo", - "name": "old-gpt3.5", - "settings": { - "frequency_penalty": 0, + "response_format": { + "type": "text" + }, + "tools": [ + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA" + }, + "format": { + "type": "string", + "enum": [ + "celsius", + "fahrenheit" + ], + "description": "The temperature unit to use. Infer this from the users location." + } + }, + "required": [ + "location", + "format" + ] + } + } + } + ], + "tool_choice": "auto", + "parallel_tool_calls": true, + "top_p": 1, + "stop": [], "presence_penalty": 0, - "temperature": 1, - "top_p": 1 + "frequency_penalty": 0 + }, + "formData": { + "shortname": "gpt-4-turbo", + "model": "gpt-4-turbo", + "system_msg": "You are a helpful assistant.", + "temperature": 1, + "response_format": "text", + "tools": "[{ \"type\": \"function\",\n \"function\": {\n \"name\": \"get_current_weather\",\n \"description\": \"Get the current weather\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"location\": {\n \"type\": \"string\",\n \"description\": \"The city and state, e.g. San Francisco, CA\"\n },\n \"format\": {\n \"type\": \"string\",\n \"enum\": [\"celsius\", \"fahrenheit\"],\n \"description\": \"The temperature unit to use. Infer this from the users location.\"\n }\n },\n \"required\": [\"location\", \"format\"]\n }\n }\n}]", + "tool_choice": "auto", + "parallel_tool_calls": true, + "top_p": 1, + "stop": "", + "presence_penalty": 0, + "frequency_penalty": 0 }, - "temp": 1, "progress": { - "success": 0, + "success": 80, "error": 0 } }, "prompt": "What is the typical temperature in Kenya in June?", "responses": [ - "The typical temperature in Kenya in June varies depending on the region, but generally, temperatures range from 19°C (66°F) to 30°C (86°F). In some parts of the country, particularly in areas of higher elevation, temperatures may be cooler, while in areas closer to the equator temperatures can be quite hot." + "In June, Kenya typically experiences variable temperatures depending on the region. Due to its diverse geography, which includes coastal areas, highlands, and savannahs, temperatures can differ significantly:\n\n1. **Coastal areas (e.g., Mombasa):** The temperatures are warmer with highs around 28°C (82°F) and lows about 20°C (68°F).\n \n2. **Highlands (e.g., Nairobi):** Cooler temperatures prevail, with daytime highs averaging 21°C (70°F) and nights around 12°C (54°F).\n\n3. **Western and Lake Victoria region:** Warm and humid with temperatures similar to the coastal areas but can be a bit cooler especially during the night.\n\nJune marks the start of the cooler season in Kenya, making it a pleasant time to visit, especially for those who prefer milder weather away from the heat and humidity typical of tropical climates." ], "tokens": {}, "eval_res": { @@ -3082,74 +5140,149 @@ }, { "vars": { - "prompt": "What is the average temperature in Hawaii in December?" + "prompt": "What is the typical temperature in Kenya in June?" }, "metavars": { "__pt": "{prompt}", - "LLM_0": "gpt3.5-turbo-0613" + "LLM_0": "no-tools-gpt3.5" }, - "uid": "4b818464-a297-429e-b081-19c61b12bae7", + "uid": "7f84b658-031a-4bc8-9e2f-37e1db9b21e0", "llm": { "base_model": "gpt-3.5-turbo", - "emoji": "🌦️", + "emoji": "🙂", "formData": { - "frequency_penalty": 0, - "function_call": "auto", - "functions": "[\n {\n \"name\": \"get_current_weather\",\n \"description\": \"Get the current weather\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"location\": {\n \"type\": \"string\",\n \"description\": \"The city and state, e.g. San Francisco, CA\"\n },\n \"format\": {\n \"type\": \"string\",\n \"enum\": [\"celsius\", \"fahrenheit\"],\n \"description\": \"The temperature unit to use. Infer this from the users location.\"\n }\n },\n \"required\": [\"location\", \"format\"]\n }\n }\n]", - "model": "gpt-3.5-turbo-0613", - "presence_penalty": 0, - "shortname": "gpt3.5-turbo-0613", - "stop": "", + "shortname": "no-tools-gpt3.5", + "model": "gpt-3.5-turbo", "system_msg": "You are a helpful assistant.", "temperature": 1, - "top_p": 1 - }, - "key": "af7031af-96ae-44b8-bcc4-3538f3174b73", - "model": "gpt-3.5-turbo-0613", - "name": "gpt3.5-turbo-0613", - "settings": { + "response_format": "text", + "tools": "", + "tool_choice": "", + "parallel_tool_calls": true, + "top_p": 1, + "stop": "", + "presence_penalty": 0, "frequency_penalty": 0, - "function_call": "auto", - "functions": [ + "function_call": "", + "functions": "" + }, + "key": "fead74ce-615a-4812-869d-9735d9d985de", + "model": "gpt-3.5-turbo", + "name": "no-tools-gpt3.5", + "settings": { + "system_msg": "You are a helpful assistant.", + "temperature": 1, + "response_format": { + "type": "text" + }, + "tools": [], + "tool_choice": "", + "parallel_tool_calls": true, + "top_p": 1, + "stop": [], + "presence_penalty": 0, + "frequency_penalty": 0, + "function_call": "", + "functions": [] + }, + "temp": 1, + "progress": { + "success": 100, + "error": 0 + } + }, + "prompt": "What is the typical temperature in Kenya in June?", + "responses": [ + "In Kenya, June is during the dry season, with temperatures averaging around 18-23°C (64-73°F) in the central highlands and around 24-30°C (75-86°F) along the coast and in lower elevations. However, temperatures can vary depending on the specific region and altitude within the country." + ], + "tokens": {}, + "eval_res": { + "items": [ + false + ], + "dtype": "Categorical" + } + }, + { + "vars": { + "prompt": "What is the typical temperature in Kenya in June?" + }, + "metavars": { + "__pt": "{prompt}", + "LLM_0": "Claude" + }, + "uid": "eac6af65-6e4c-40ac-a8ad-b07e13c36ac5", + "llm": { + "key": "4caee194-fa12-4ca8-bb9e-1c8d7d5f4a08", + "name": "Claude", + "emoji": "📚", + "model": "claude-3-5-sonnet-latest", + "base_model": "claude-v1", + "temp": 0.5, + "settings": { + "system_msg": "", + "temperature": 1, + "tools": [ { - "description": "Get the current weather", "name": "get_current_weather", - "parameters": { + "description": "Get the current weather", + "input_schema": { + "type": "object", "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA" + }, "format": { - "description": "The temperature unit to use. Infer this from the users location.", + "type": "string", "enum": [ "celsius", "fahrenheit" ], - "type": "string" - }, - "location": { - "description": "The city and state, e.g. San Francisco, CA", - "type": "string" + "description": "The temperature unit to use. Infer this from the users location." } }, "required": [ "location", "format" - ], - "type": "object" + ] } } ], - "presence_penalty": 0, - "temperature": 1, - "top_p": 1 + "tool_choice": { + "type": "auto" + }, + "parallel_tool_calls": true, + "max_tokens_to_sample": 1024, + "custom_prompt_wrapper": "", + "stop_sequences": [ + "\n\nHuman:" + ], + "top_k": -1, + "top_p": -1 + }, + "formData": { + "shortname": "Claude", + "model": "claude-3-5-sonnet-latest", + "system_msg": "", + "temperature": 1, + "tools": "[{\n \"name\": \"get_current_weather\",\n \"description\": \"Get the current weather\",\n \"input_schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"location\": {\n \"type\": \"string\",\n \"description\": \"The city and state, e.g. San Francisco, CA\"\n },\n \"format\": {\n \"type\": \"string\",\n \"enum\": [\"celsius\", \"fahrenheit\"],\n \"description\": \"The temperature unit to use. Infer this from the users location.\"\n }\n },\n \"required\": [\"location\", \"format\"]\n }\n }\n]", + "tool_choice": "auto", + "parallel_tool_calls": true, + "max_tokens_to_sample": 1024, + "custom_prompt_wrapper": "", + "stop_sequences": "\"\n\nHuman:\"", + "top_k": -1, + "top_p": -1 }, - "temp": 1, "progress": { - "success": 0, + "success": 100, "error": 0 } }, - "prompt": "What is the average temperature in Hawaii in December?", + "prompt": "What is the typical temperature in Kenya in June?", "responses": [ - "[[FUNCTION]] get_current_weather\\{\n \"format\": \"celsius\",\n \"location\": \"Honolulu, HI\"\n\\}" + "[[TOOLS]] {\"name\":\"get_current_weather\",\"input\":{\"format\":\"celsius\",\"location\":\"Nairobi, Kenya\"}}" ], "tokens": {}, "eval_res": { @@ -3165,42 +5298,81 @@ }, "metavars": { "__pt": "{prompt}", - "LLM_0": "old-gpt3.5" + "LLM_0": "gpt-4-turbo" }, - "uid": "426b0d32-7afd-46a8-a9db-b0e893e7e3b7", + "uid": "de3fd081-62b5-4943-bd4d-36987a43004d", "llm": { + "key": "1725e2ec-cd4b-4211-a73a-3d6ef4482451", + "name": "gpt-4-turbo", + "emoji": "🤖", + "model": "gpt-4-turbo", "base_model": "gpt-3.5-turbo", - "emoji": "🙂", - "formData": { - "frequency_penalty": 0, - "function_call": "", - "functions": "", - "model": "gpt-3.5-turbo", - "presence_penalty": 0, - "shortname": "old-gpt3.5", - "stop": "", + "temp": 1, + "settings": { "system_msg": "You are a helpful assistant.", "temperature": 1, - "top_p": 1 - }, - "key": "fead74ce-615a-4812-869d-9735d9d985de", - "model": "gpt-3.5-turbo", - "name": "old-gpt3.5", - "settings": { - "frequency_penalty": 0, + "response_format": { + "type": "text" + }, + "tools": [ + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA" + }, + "format": { + "type": "string", + "enum": [ + "celsius", + "fahrenheit" + ], + "description": "The temperature unit to use. Infer this from the users location." + } + }, + "required": [ + "location", + "format" + ] + } + } + } + ], + "tool_choice": "auto", + "parallel_tool_calls": true, + "top_p": 1, + "stop": [], "presence_penalty": 0, - "temperature": 1, - "top_p": 1 + "frequency_penalty": 0 + }, + "formData": { + "shortname": "gpt-4-turbo", + "model": "gpt-4-turbo", + "system_msg": "You are a helpful assistant.", + "temperature": 1, + "response_format": "text", + "tools": "[{ \"type\": \"function\",\n \"function\": {\n \"name\": \"get_current_weather\",\n \"description\": \"Get the current weather\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"location\": {\n \"type\": \"string\",\n \"description\": \"The city and state, e.g. San Francisco, CA\"\n },\n \"format\": {\n \"type\": \"string\",\n \"enum\": [\"celsius\", \"fahrenheit\"],\n \"description\": \"The temperature unit to use. Infer this from the users location.\"\n }\n },\n \"required\": [\"location\", \"format\"]\n }\n }\n}]", + "tool_choice": "auto", + "parallel_tool_calls": true, + "top_p": 1, + "stop": "", + "presence_penalty": 0, + "frequency_penalty": 0 }, - "temp": 1, "progress": { - "success": 0, + "success": 80, "error": 0 } }, "prompt": "What is the average temperature in Hawaii in December?", "responses": [ - "The average temperature in Hawaii in December is around 77°F (25°C) during the day and 66°F (19°C) at night." + "The average temperature in Hawaii in December typically ranges from about 68°F (20°C) to 81°F (27°C). This offers pleasant, warm weather which is why Hawaii is such a popular winter destination. Local variations across the different islands can, of course, result in slightly different temperatures due to geography and elevation." ], "tokens": {}, "eval_res": { @@ -3209,6 +5381,160 @@ ], "dtype": "Categorical" } + }, + { + "vars": { + "prompt": "What is the average temperature in Hawaii in December?" + }, + "metavars": { + "__pt": "{prompt}", + "LLM_0": "no-tools-gpt3.5" + }, + "uid": "f3aa7139-def5-4397-8b87-ee075ffe315f", + "llm": { + "base_model": "gpt-3.5-turbo", + "emoji": "🙂", + "formData": { + "shortname": "no-tools-gpt3.5", + "model": "gpt-3.5-turbo", + "system_msg": "You are a helpful assistant.", + "temperature": 1, + "response_format": "text", + "tools": "", + "tool_choice": "", + "parallel_tool_calls": true, + "top_p": 1, + "stop": "", + "presence_penalty": 0, + "frequency_penalty": 0, + "function_call": "", + "functions": "" + }, + "key": "fead74ce-615a-4812-869d-9735d9d985de", + "model": "gpt-3.5-turbo", + "name": "no-tools-gpt3.5", + "settings": { + "system_msg": "You are a helpful assistant.", + "temperature": 1, + "response_format": { + "type": "text" + }, + "tools": [], + "tool_choice": "", + "parallel_tool_calls": true, + "top_p": 1, + "stop": [], + "presence_penalty": 0, + "frequency_penalty": 0, + "function_call": "", + "functions": [] + }, + "temp": 1, + "progress": { + "success": 100, + "error": 0 + } + }, + "prompt": "What is the average temperature in Hawaii in December?", + "responses": [ + "In December, the average temperature in Hawaii typically ranges from 75°F to 80°F (24°C to 27°C). However, the exact temperature can vary depending on the specific island and location within Hawaii." + ], + "tokens": {}, + "eval_res": { + "items": [ + false + ], + "dtype": "Categorical" + } + }, + { + "vars": { + "prompt": "What is the average temperature in Hawaii in December?" + }, + "metavars": { + "__pt": "{prompt}", + "LLM_0": "Claude" + }, + "uid": "58de1cb7-3514-4ecc-ae42-6e0900b91ff5", + "llm": { + "key": "4caee194-fa12-4ca8-bb9e-1c8d7d5f4a08", + "name": "Claude", + "emoji": "📚", + "model": "claude-3-5-sonnet-latest", + "base_model": "claude-v1", + "temp": 0.5, + "settings": { + "system_msg": "", + "temperature": 1, + "tools": [ + { + "name": "get_current_weather", + "description": "Get the current weather", + "input_schema": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA" + }, + "format": { + "type": "string", + "enum": [ + "celsius", + "fahrenheit" + ], + "description": "The temperature unit to use. Infer this from the users location." + } + }, + "required": [ + "location", + "format" + ] + } + } + ], + "tool_choice": { + "type": "auto" + }, + "parallel_tool_calls": true, + "max_tokens_to_sample": 1024, + "custom_prompt_wrapper": "", + "stop_sequences": [ + "\n\nHuman:" + ], + "top_k": -1, + "top_p": -1 + }, + "formData": { + "shortname": "Claude", + "model": "claude-3-5-sonnet-latest", + "system_msg": "", + "temperature": 1, + "tools": "[{\n \"name\": \"get_current_weather\",\n \"description\": \"Get the current weather\",\n \"input_schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"location\": {\n \"type\": \"string\",\n \"description\": \"The city and state, e.g. San Francisco, CA\"\n },\n \"format\": {\n \"type\": \"string\",\n \"enum\": [\"celsius\", \"fahrenheit\"],\n \"description\": \"The temperature unit to use. Infer this from the users location.\"\n }\n },\n \"required\": [\"location\", \"format\"]\n }\n }\n]", + "tool_choice": "auto", + "parallel_tool_calls": true, + "max_tokens_to_sample": 1024, + "custom_prompt_wrapper": "", + "stop_sequences": "\"\n\nHuman:\"", + "top_k": -1, + "top_p": -1 + }, + "progress": { + "success": 100, + "error": 0 + } + }, + "prompt": "What is the average temperature in Hawaii in December?", + "responses": [ + "[[TOOLS]] {\"name\":\"get_current_weather\",\"input\":{\"format\":\"fahrenheit\",\"location\":\"Honolulu, HI\"}}" + ], + "tokens": {}, + "eval_res": { + "items": [ + true + ], + "dtype": "Categorical" + } } ] } diff --git a/chainforge/react-server/src/backend/utils.ts b/chainforge/react-server/src/backend/utils.ts index 4109241..5cd5f40 100644 --- a/chainforge/react-server/src/backend/utils.ts +++ b/chainforge/react-server/src/backend/utils.ts @@ -1615,7 +1615,12 @@ function _extract_openai_chat_choice_content(choice: Dict): string { ("tool_calls" in choice.message && choice.message.tool_calls.length > 0) ) { const tools = choice.message.tool_calls; - return "[[TOOLS]] " + tools.toString(); + return ( + "[[TOOLS]] " + + tools + .map((t: Dict) => t.function.name + " " + t.function.arguments) + .join("\n\n") + ); } else { // Extract the content. Note that structured outputs in OpenAI's API as of late 2024 // can sometimes output a response to a "refusal" key, which is annoying. We check for that here: