From 0cf0944bf898861adb436c1a1be3d78f0d4dee71 Mon Sep 17 00:00:00 2001
From: Ian Arawjo <fatso784@gmail.com>
Date: Mon, 16 Dec 2024 16:00:17 -0500
Subject: [PATCH] Add num_ctx to Ollama API call

---
 chainforge/react-server/src/ModelSettingSchemas.tsx |  8 ++++++++
 chainforge/react-server/src/backend/utils.ts        | 12 ++++++++----
 2 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/chainforge/react-server/src/ModelSettingSchemas.tsx b/chainforge/react-server/src/ModelSettingSchemas.tsx
index 552347c..13cb040 100644
--- a/chainforge/react-server/src/ModelSettingSchemas.tsx
+++ b/chainforge/react-server/src/ModelSettingSchemas.tsx
@@ -1366,6 +1366,14 @@ const OllamaSettings: ModelSettingsDict = {
           "The JSON schema to use for structured outputs. Note that using structured outputs, you should still prompt the model to output JSON. (Supported by Ollama since Dec 6 2024. For more info, see https://ollama.com/blog/structured-outputs)",
         default: "",
       },
+      num_ctx: {
+        type: "integer",
+        title: "num_ctx",
+        description:
+          "How many tokens to allow in the *input* to the model. Defaults to Ollama's default setting of 2048. You must increase this if your context is lengthy.",
+        minimum: 1,
+        default: 2048,
+      },
       raw: {
         type: "boolean",
         title: "raw",
diff --git a/chainforge/react-server/src/backend/utils.ts b/chainforge/react-server/src/backend/utils.ts
index 4c8bada..4109241 100644
--- a/chainforge/react-server/src/backend/utils.ts
+++ b/chainforge/react-server/src/backend/utils.ts
@@ -1205,6 +1205,7 @@ export async function call_ollama_provider(
   const model_type: string = params?.model_type ?? "text";
   const system_msg: string = params?.system_msg ?? "";
   const chat_history: ChatHistory | undefined = params?.chat_history;
+  const format: Dict | string | undefined = params?.format;
 
   // Cleanup
   for (const name of [
@@ -1213,6 +1214,7 @@ export async function call_ollama_provider(
     "model_type",
     "system_msg",
     "chat_history",
+    "format",
   ])
     if (params && name in params) delete params[name];
 
@@ -1221,8 +1223,10 @@ export async function call_ollama_provider(
   const query: Dict = {
     model: ollama_model,
     stream: false,
-    temperature,
-    ...params, // 'the rest' of the settings, passed from the front-end settings
+    options: {
+      temperature,
+      ...params, // 'the rest' of the settings, passed from the front-end settings
+    },
   };
 
   // If the model type is explicitly or implicitly set to "chat", pass chat history instead:
@@ -1245,9 +1249,9 @@ export async function call_ollama_provider(
   );
 
   // If there are structured outputs specified, convert to an object:
-  if (typeof query.format === "string" && query.format.trim().length > 0) {
+  if (typeof format === "string" && format.trim().length > 0) {
     try {
-      query.format = JSON.parse(query.format);
+      query.format = JSON.parse(format);
     } catch (err) {
       throw Error(
         "Cannot parse structured output format into JSON: JSON schema is incorrectly structured.",