From d098d2679361bcf5b5701e151ddec935cb740afb Mon Sep 17 00:00:00 2001 From: Ian Arawjo Date: Sun, 10 Mar 2024 18:29:17 -0400 Subject: [PATCH] wip --- chainforge/react-server/package-lock.json | 10 + chainforge/react-server/package.json | 1 + chainforge/react-server/src/AiPopover.tsx | 24 +- chainforge/react-server/src/AlertModal.tsx | 54 ++- chainforge/react-server/src/App.tsx | 8 +- .../react-server/src/AreYouSureModal.tsx | 117 +++-- .../react-server/src/CodeEvaluatorNode.tsx | 407 ++++++++++-------- chainforge/react-server/src/InspectFooter.tsx | 4 +- .../src/LLMResponseInspectorModal.tsx | 65 +++ .../react-server/src/ModelSettingsModal.tsx | 29 +- .../react-server/src/NodeLabelComponent.tsx | 8 +- chainforge/react-server/src/PlotLegend.tsx | 8 +- chainforge/react-server/src/PromptNode.tsx | 8 +- .../react-server/src/RenameValueModal.tsx | 99 +++-- .../react-server/src/TabularDataNode.tsx | 8 +- .../react-server/src/backend/backend.ts | 22 +- chainforge/react-server/src/backend/errors.ts | 2 - chainforge/react-server/src/backend/typing.ts | 17 +- chainforge/react-server/src/backend/utils.ts | 12 +- .../{example_flows.js => example_flows.tsx} | 0 20 files changed, 524 insertions(+), 379 deletions(-) create mode 100644 chainforge/react-server/src/LLMResponseInspectorModal.tsx rename chainforge/react-server/src/{example_flows.js => example_flows.tsx} (100%) diff --git a/chainforge/react-server/package-lock.json b/chainforge/react-server/package-lock.json index cdc218b..4360d16 100644 --- a/chainforge/react-server/package-lock.json +++ b/chainforge/react-server/package-lock.json @@ -105,6 +105,7 @@ "devDependencies": { "@craco/craco": "^7.1.0", "@types/papaparse": "^5.3.14", + "@types/react-beautiful-dnd": "^13.1.8", "@types/react-edit-text": "^5.0.4", "eslint": "^8.56.0", "eslint-config-prettier": "^9.1.0", @@ -7001,6 +7002,15 @@ "csstype": "^3.0.2" } }, + "node_modules/@types/react-beautiful-dnd": { + "version": "13.1.8", + "resolved": "https://registry.npmjs.org/@types/react-beautiful-dnd/-/react-beautiful-dnd-13.1.8.tgz", + "integrity": "sha512-E3TyFsro9pQuK4r8S/OL6G99eq7p8v29sX0PM7oT8Z+PJfZvSQTx4zTQbUJ+QZXioAF0e7TGBEcA1XhYhCweyQ==", + "dev": true, + "dependencies": { + "@types/react": "*" + } + }, "node_modules/@types/react-dom": { "version": "18.2.7", "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-18.2.7.tgz", diff --git a/chainforge/react-server/package.json b/chainforge/react-server/package.json index 635bb8c..dd95f9e 100644 --- a/chainforge/react-server/package.json +++ b/chainforge/react-server/package.json @@ -131,6 +131,7 @@ "devDependencies": { "@craco/craco": "^7.1.0", "@types/papaparse": "^5.3.14", + "@types/react-beautiful-dnd": "^13.1.8", "@types/react-edit-text": "^5.0.4", "eslint": "^8.56.0", "eslint-config-prettier": "^9.1.0", diff --git a/chainforge/react-server/src/AiPopover.tsx b/chainforge/react-server/src/AiPopover.tsx index 49b2ce1..da7c9ee 100644 --- a/chainforge/react-server/src/AiPopover.tsx +++ b/chainforge/react-server/src/AiPopover.tsx @@ -18,7 +18,7 @@ import { getAIFeaturesModels, } from "./backend/ai"; import { IconSparkles, IconAlertCircle } from "@tabler/icons-react"; -import AlertModal, { AlertModalHandles } from "./AlertModal"; +import AlertModal, { AlertModalRef } from "./AlertModal"; import useStore from "./store"; import { INFO_CODEBLOCK_JS, @@ -30,7 +30,7 @@ import { queryLLM } from "./backend/backend"; import { splitText } from "./SplitNode"; import { escapeBraces } from "./backend/template"; import { cleanMetavarsFilterFunc } from "./backend/utils"; -import { Dict, TemplateVarInfo } from "./backend/typing"; +import { Dict, TemplateVarInfo, VarsContext } from "./backend/typing"; const zeroGap = { gap: "0rem" }; const popoverShadow = "rgb(38, 57, 77) 0px 10px 30px -14px"; @@ -94,10 +94,7 @@ ${specPrompt}`; // Builds part of a longer prompt to the LLM about the shape of Response objects // input into an evaluator (the names of template vars, and available metavars) -export const buildContextPromptForVarsMetavars = (context: { - vars: string[]; - metavars: string[]; -}) => { +export const buildContextPromptForVarsMetavars = (context: VarsContext) => { if (!context) return ""; const promptify_key_arr = (arr: string[]) => { @@ -106,10 +103,11 @@ export const buildContextPromptForVarsMetavars = (context: { }; let context_str = ""; - const metavars = context.metavars - ? context.metavars.filter(cleanMetavarsFilterFunc) - : []; - const has_vars = context.vars && context.vars.length > 0; + const metavars = + "metavars" in context + ? context.metavars.filter(cleanMetavarsFilterFunc) + : []; + const has_vars = "vars" in context && context.vars.length > 0; const has_metavars = metavars && metavars.length > 0; const has_context = has_vars || has_metavars; if (has_context) context_str = "\nThe ResponseInfo instances have "; @@ -257,7 +255,7 @@ export function AIGenReplaceItemsPopover({ const aiFeaturesProvider = useStore((state) => state.aiFeaturesProvider); // Alerts - const alertModal = useRef(null); + const alertModal = useRef(null); // Command Fill state const [commandFillNumber, setCommandFillNumber] = useState(3); @@ -485,7 +483,7 @@ export interface AIGenCodeEvaluatorPopoverProps { // Callback that takes a boolean that the popover will call to set whether the values are loading and are done loading onLoadingChange: (isLoading: boolean) => void; // The keys available in vars and metavar dicts, for added context to the LLM - context: { vars: string[]; metavars: string[] }; + context: VarsContext; // The code currently in the evaluator currentEvalCode: string; } @@ -510,7 +508,7 @@ export function AIGenCodeEvaluatorPopover({ const [awaitingResponse, setAwaitingResponse] = useState(false); // Alerts - const alertModal = useRef(null); + const alertModal = useRef(null); const [didEncounterError, setDidEncounterError] = useState(false); // Handle errors diff --git a/chainforge/react-server/src/AlertModal.tsx b/chainforge/react-server/src/AlertModal.tsx index aeb60a2..fe1fa3b 100644 --- a/chainforge/react-server/src/AlertModal.tsx +++ b/chainforge/react-server/src/AlertModal.tsx @@ -8,38 +8,36 @@ const ALERT_MODAL_STYLE = { root: { position: "relative", left: "-5%" }, } as Styles; -export interface AlertModalHandles { +export interface AlertModalRef { trigger: (msg?: string) => void; } -const AlertModal = forwardRef( - function AlertModal(props, ref) { - // Mantine modal popover for alerts - const [opened, { open, close }] = useDisclosure(false); - const [alertMsg, setAlertMsg] = useState(""); +const AlertModal = forwardRef(function AlertModal(props, ref) { + // Mantine modal popover for alerts + const [opened, { open, close }] = useDisclosure(false); + const [alertMsg, setAlertMsg] = useState(""); - // This gives the parent access to triggering the modal alert - const trigger = (msg?: string) => { - if (!msg) msg = "Unknown error."; - console.error(msg); - setAlertMsg(msg); - open(); - }; - useImperativeHandle(ref, () => ({ - trigger, - })); + // This gives the parent access to triggering the modal alert + const trigger = (msg?: string) => { + if (!msg) msg = "Unknown error."; + console.error(msg); + setAlertMsg(msg); + open(); + }; + useImperativeHandle(ref, () => ({ + trigger, + })); - return ( - -

{alertMsg}

- - ); - }, -); + return ( + +

{alertMsg}

+ + ); +}); export default AlertModal; diff --git a/chainforge/react-server/src/App.tsx b/chainforge/react-server/src/App.tsx index c08c5ed..d282c56 100644 --- a/chainforge/react-server/src/App.tsx +++ b/chainforge/react-server/src/App.tsx @@ -31,7 +31,7 @@ import CodeEvaluatorNode from "./CodeEvaluatorNode"; import VisNode from "./VisNode"; import InspectNode from "./InspectorNode"; import ScriptNode from "./ScriptNode"; -import AlertModal, { AlertModalHandles } from "./AlertModal"; +import AlertModal, { AlertModalRef } from "./AlertModal"; import ItemsNode from "./ItemsNode"; import TabularDataNode from "./TabularDataNode"; import JoinNode from "./JoinNode"; @@ -243,7 +243,7 @@ const App = () => { const { hideContextMenu } = useContextMenu(); // For displaying error messages to user - const alertModal = useRef(null); + const alertModal = useRef(null); // For displaying a pending 'loading' status const [isLoading, setIsLoading] = useState(true); @@ -455,8 +455,8 @@ const App = () => { // Save the current flow to localStorage for later recall. Useful to getting // back progress upon leaving the site / browser crash / system restart. const saveFlow = useCallback( - (rf_inst) => { - const rf = rf_inst || rfInstance; + (rf_inst: ReactFlowInstance) => { + const rf = rf_inst ?? rfInstance; if (!rf) return; // NOTE: This currently only saves the front-end state. Cache files diff --git a/chainforge/react-server/src/AreYouSureModal.tsx b/chainforge/react-server/src/AreYouSureModal.tsx index ebd4fa2..b0ac59c 100644 --- a/chainforge/react-server/src/AreYouSureModal.tsx +++ b/chainforge/react-server/src/AreYouSureModal.tsx @@ -8,73 +8,72 @@ export interface AreYouSureModalProps { onConfirm?: () => void; } -export interface AreYouSureModalHandles { +export interface AreYouSureModalRef { trigger: () => void; } /** Modal that lets user rename a single value, using a TextInput field. */ -const AreYouSureModal = forwardRef< - AreYouSureModalHandles, - AreYouSureModalProps ->(function AreYouSureModal({ title, message, onConfirm }, ref) { - const [opened, { open, close }] = useDisclosure(false); - const description = message || "Are you sure?"; +const AreYouSureModal = forwardRef( + function AreYouSureModal({ title, message, onConfirm }, ref) { + const [opened, { open, close }] = useDisclosure(false); + const description = message || "Are you sure?"; - // This gives the parent access to triggering the modal alert - const trigger = () => { - open(); - }; - useImperativeHandle(ref, () => ({ - trigger, - })); + // This gives the parent access to triggering the modal alert + const trigger = () => { + open(); + }; + useImperativeHandle(ref, () => ({ + trigger, + })); - const confirmAndClose = () => { - close(); - if (onConfirm) onConfirm(); - }; + const confirmAndClose = () => { + close(); + if (onConfirm) onConfirm(); + }; - return ( - - - {description} - - - - - - - ); -}); + + + + + ); + }, +); export default AreYouSureModal; diff --git a/chainforge/react-server/src/CodeEvaluatorNode.tsx b/chainforge/react-server/src/CodeEvaluatorNode.tsx index 37202c4..e895726 100644 --- a/chainforge/react-server/src/CodeEvaluatorNode.tsx +++ b/chainforge/react-server/src/CodeEvaluatorNode.tsx @@ -18,7 +18,6 @@ import { Switch, } from "@mantine/core"; import { Prism } from "@mantine/prism"; -import { Language } from "prism-react-renderer"; import { useDisclosure } from "@mantine/hooks"; import useStore from "./store"; import BaseNode from "./BaseNode"; @@ -29,7 +28,9 @@ import { IconInfoCircle, IconBox, } from "@tabler/icons-react"; -import LLMResponseInspectorModal from "./LLMResponseInspectorModal"; +import LLMResponseInspectorModal, { + LLMResponseInspectorModalRef, +} from "./LLMResponseInspectorModal"; // Ace code editor import AceEditor from "react-ace"; @@ -37,7 +38,6 @@ import "ace-builds/src-noconflict/mode-python"; import "ace-builds/src-noconflict/mode-javascript"; import "ace-builds/src-noconflict/theme-xcode"; import "ace-builds/src-noconflict/ext-language_tools"; -import fetch_from_backend from "./fetch_from_backend"; import { APP_IS_RUNNING_LOCALLY, getVarsAndMetavars, @@ -48,8 +48,17 @@ import InspectFooter from "./InspectFooter"; import { escapeBraces } from "./backend/template"; import LLMResponseInspectorDrawer from "./LLMResponseInspectorDrawer"; import { AIGenCodeEvaluatorPopover } from "./AiPopover"; -import { Dict, EvaluatedResponsesResults, StandardizedLLMResponse } from "./backend/typing"; +import { + Dict, + EvaluatedResponsesResults, + PythonInterpreter, + StandardizedLLMResponse, + TemplateVarInfo, + VarsContext, +} from "./backend/typing"; import { Status } from "./StatusIndicatorComponent"; +import { executejs, executepy, grabResponses } from "./backend/backend"; +import { AlertModalRef } from "./AlertModal"; // Whether we are running on localhost or not, and hence whether // we have access to the Flask backend for, e.g., Python code evaluation. @@ -162,18 +171,26 @@ function process(response) { return "NOT FOUND"; }`; -export interface CodeEvaluatorComponentHandles { - run: (inputs: Dict[], script_paths?: string[], runInSandbox?: boolean) => Promise<({ - code: string, responses?: StandardizedLLMResponse[], error?: string | Error, logs?: string[] })>, - serialize: () => ({code: string}), - setCodeText: (code: string) => void, +export interface CodeEvaluatorComponentRef { + run: ( + inputs: StandardizedLLMResponse[], + script_paths?: string[], + runInSandbox?: boolean, + ) => Promise<{ + code: string; + responses?: StandardizedLLMResponse[]; + error?: string | undefined; + logs?: string[]; + }>; + serialize: () => { code: string }; + setCodeText: (code: string) => void; } export interface CodeEvaluatorComponentProps { code: string; id: string; - type: 'evaluator' | 'processor'; - progLang: 'python' | 'javascript'; + type: "evaluator" | "processor"; + progLang: "python" | "javascript"; showUserInstruction: boolean; onCodeEdit?: (code: string) => void; onCodeChangedFromLastRun?: () => void; @@ -183,155 +200,159 @@ export interface CodeEvaluatorComponentProps { /** * Inner component for code evaluators/processors, storing the body of the UI (outside of the header and footers). */ -export const CodeEvaluatorComponent = forwardRef( - function CodeEvaluatorComponent( - { - code, - id, - type: node_type, - progLang, - showUserInstruction, - onCodeEdit, - onCodeChangedFromLastRun, - onCodeEqualToLastRun, - }, - ref, - ) { - // Code in the editor - const [codeText, setCodeText] = useState(code ?? ""); - const [codeTextOnLastRun, setCodeTextOnLastRun] = useState(false); - - // Controlled handle when user edits code - const handleCodeEdit = (code: string) => { - if (codeTextOnLastRun !== false) { - const code_changed = code !== codeTextOnLastRun; - if (code_changed && onCodeChangedFromLastRun) - onCodeChangedFromLastRun(); - else if (!code_changed && onCodeEqualToLastRun) onCodeEqualToLastRun(); - } - setCodeText(code); - if (onCodeEdit) onCodeEdit(code); - }; - - // Runs the code evaluator/processor over the inputs, returning the results as a Promise. - // Errors are raised as a rejected Promise. - const run = (inputs: Dict[], script_paths?: string[], runInSandbox?: boolean) => { - // Double-check that the code includes an 'evaluate' or 'process' function, whichever is needed: - const find_func_regex = - node_type === "evaluator" - ? progLang === "python" - ? /def\s+evaluate\s*(.*):/ - : /function\s+evaluate\s*(.*)/ - : progLang === "python" - ? /def\s+process\s*(.*):/ - : /function\s+process\s*(.*)/; - if (codeText.search(find_func_regex) === -1) { - const req_func_name = - node_type === "evaluator" ? "evaluate" : "process"; - const err_msg = `Could not find required function '${req_func_name}'. Make sure you have defined an '${req_func_name}' function.`; - return Promise.reject(new Error(err_msg)); // hard fail - } - - const codeTextOnRun = codeText + ""; - const execute_route = progLang === "python" ? "executepy" : "executejs"; - let executor = progLang === "python" ? "pyodide" : undefined; - - // Enable running Python in Flask backend (unsafe) if running locally and the user has turned off the sandbox: - if (progLang === "python" && IS_RUNNING_LOCALLY && !runInSandbox) - executor = "flask"; - - return fetch_from_backend(execute_route, { - id, - code: codeTextOnRun, - responses: inputs, - scope: "response", - process_type: node_type, - script_paths, - executor, - }).then(function (json) { - json = json as EvaluatedResponsesResults; - // Check if there's an error; if so, bubble it up to user and exit: - if (json.error) { - if (json.logs) json.logs.push(json.error); - } else { - setCodeTextOnLastRun(codeTextOnRun); - } - - return { - code, // string - responses: json?.responses, // array of ResponseInfo Objects - error: json?.error, // undefined or, if present, a string of the error message - logs: json?.logs, // an array of strings representing console.logs/prints made during execution - }; - }); - }; - - // Export the current internal state as JSON - const serialize = () => ({ code: codeText }); - - // Define functions accessible from the parent component - useImperativeHandle(ref, () => ({ - run, - serialize, - setCodeText, - })); - - // Helpful instruction for user - const code_instruct_header = useMemo(() => { - if (node_type === "evaluator") - return ( -
- Define an evaluate func to map over each response: -
- ); - else - return ( -
- Define a process func to map over each response: -
- ); - }, [node_type]); - - return ( -
- {showUserInstruction ? code_instruct_header : <>} -
- { - // Make Ace Editor div resizeable. - editorInstance.container.style.resize = "both"; - document.addEventListener("mouseup", () => - editorInstance.resize(), - ); - }} - /> -
-
- ); +export const CodeEvaluatorComponent = forwardRef< + CodeEvaluatorComponentRef, + CodeEvaluatorComponentProps +>(function CodeEvaluatorComponent( + { + code, + id, + type: node_type, + progLang, + showUserInstruction, + onCodeEdit, + onCodeChangedFromLastRun, + onCodeEqualToLastRun, }, -); + ref, +) { + // Code in the editor + const [codeText, setCodeText] = useState(code ?? ""); + const [codeTextOnLastRun, setCodeTextOnLastRun] = useState( + false, + ); + + // Controlled handle when user edits code + const handleCodeEdit = (code: string) => { + if (codeTextOnLastRun !== false) { + const code_changed = code !== codeTextOnLastRun; + if (code_changed && onCodeChangedFromLastRun) onCodeChangedFromLastRun(); + else if (!code_changed && onCodeEqualToLastRun) onCodeEqualToLastRun(); + } + setCodeText(code); + if (onCodeEdit) onCodeEdit(code); + }; + + // Runs the code evaluator/processor over the inputs, returning the results as a Promise. + // Errors are raised as a rejected Promise. + const run = ( + inputs: StandardizedLLMResponse[], + script_paths?: string[], + runInSandbox?: boolean, + ) => { + // Double-check that the code includes an 'evaluate' or 'process' function, whichever is needed: + const find_func_regex = + node_type === "evaluator" + ? progLang === "python" + ? /def\s+evaluate\s*(.*):/ + : /function\s+evaluate\s*(.*)/ + : progLang === "python" + ? /def\s+process\s*(.*):/ + : /function\s+process\s*(.*)/; + if (codeText.search(find_func_regex) === -1) { + const req_func_name = node_type === "evaluator" ? "evaluate" : "process"; + const err_msg = `Could not find required function '${req_func_name}'. Make sure you have defined an '${req_func_name}' function.`; + return Promise.reject(new Error(err_msg)); // hard fail + } + + const codeTextOnRun = codeText + ""; + const execute_route = progLang === "python" ? executepy : executejs; + let executor: PythonInterpreter | undefined = + progLang === "python" ? "pyodide" : undefined; + + // Enable running Python in Flask backend (unsafe) if running locally and the user has turned off the sandbox: + if (progLang === "python" && IS_RUNNING_LOCALLY && !runInSandbox) + executor = "flask"; + + return execute_route( + id, + codeTextOnRun, + inputs, + "response", + node_type, + script_paths, + executor, + ).then(function (json) { + json = json as EvaluatedResponsesResults; + // Check if there's an error; if so, bubble it up to user and exit: + if (json.error) { + if (json.logs) json.logs.push(json.error); + } else { + setCodeTextOnLastRun(codeTextOnRun); + } + + return { + code, // string + responses: json?.responses, // array of ResponseInfo Objects + error: json?.error, // undefined or, if present, a string of the error message + logs: json?.logs, // an array of strings representing console.logs/prints made during execution + }; + }); + }; + + // Export the current internal state as JSON + const serialize = () => ({ code: codeText }); + + // Define functions accessible from the parent component + useImperativeHandle(ref, () => ({ + run, + serialize, + setCodeText, + })); + + // Helpful instruction for user + const code_instruct_header = useMemo(() => { + if (node_type === "evaluator") + return ( +
+ Define an evaluate func to map over each response: +
+ ); + else + return ( +
+ Define a process func to map over each response: +
+ ); + }, [node_type]); + + return ( +
+ {showUserInstruction ? code_instruct_header : <>} +
+ { + // Make Ace Editor div resizeable. + editorInstance.container.style.resize = "both"; + document.addEventListener("mouseup", () => editorInstance.resize()); + }} + /> +
+
+ ); +}); export interface CodeEvaluatorNodeProps { data: { title: string; code: string; - language: 'python' | 'javascript'; + language: "python" | "javascript"; sandbox: boolean; refresh: boolean; }; - id: string; - type: 'evaluator' | 'processor'; + id: string; + type: "evaluator" | "processor"; } /** @@ -339,9 +360,13 @@ export interface CodeEvaluatorNodeProps { * It has two possible node_types: 'evaluator' and 'processor' mode. * Evaluators annotate responses with scores; processors transform response objects themselves. */ -const CodeEvaluatorNode: React.FC = ({ data, id, type: node_type }) => { +const CodeEvaluatorNode: React.FC = ({ + data, + id, + type: node_type, +}) => { // The inner component storing the code UI and providing an interface to run the code over inputs - const codeEvaluatorRef = useRef(null); + const codeEvaluatorRef = useRef(null); const currentCode = useMemo(() => data.code, [data.code]); // Whether to sandbox the code execution. Currently this option only displays for Python running locally, @@ -359,17 +384,17 @@ const CodeEvaluatorNode: React.FC = ({ data, id, type: n // For genAI features const flags = useStore((state) => state.flags); const [isEvalCodeGenerating, setIsEvalCodeGenerating] = useState(false); - const [lastContext, setLastContext] = useState({}); + const [lastContext, setLastContext] = useState({}); // For displaying error messages to user - const alertModal = useRef(null); + const alertModal = useRef(null); // For an info pop-up that explains the type of ResponseInfo const [infoModalOpened, { open: openInfoModal, close: closeInfoModal }] = useDisclosure(false); // For a way to inspect responses without having to attach a dedicated node - const inspectModal = useRef(null); + const inspectModal = useRef(null); // eslint-disable-next-line const [uninspectedResponses, setUninspectedResponses] = useState(false); const [showDrawer, setShowDrawer] = useState(false); @@ -380,19 +405,35 @@ const CodeEvaluatorNode: React.FC = ({ data, id, type: n const [progLang, setProgLang] = useState(data.language ?? "python"); const [lastRunLogs, setLastRunLogs] = useState(""); - const [lastResponses, setLastResponses] = useState([]); + const [lastResponses, setLastResponses] = useState( + [], + ); const [lastRunSuccess, setLastRunSuccess] = useState(true); + const handleError = useCallback( + (err: string | Error) => { + setStatus(Status.ERROR); + setLastRunSuccess(false); + if (typeof err !== "string") console.error(err); + alertModal.current?.trigger(typeof err === "string" ? err : err?.message); + }, + [alertModal], + ); + const pullInputs = useCallback(() => { // Pull input data - let pulled_inputs = pullInputData(["responseBatch"], id); + let pulled_inputs: Dict | StandardizedLLMResponse[] = pullInputData( + ["responseBatch"], + id, + ); if (!pulled_inputs || !pulled_inputs.responseBatch) { console.warn(`No inputs for code ${node_type} node.`); return null; } // Convert to standard response format (StandardLLMResponseFormat) - pulled_inputs = pulled_inputs.responseBatch.map(toStandardResponseFormat); - return pulled_inputs; + return pulled_inputs.responseBatch.map( + toStandardResponseFormat, + ) as StandardizedLLMResponse[]; }, [id, pullInputData]); // On initialization @@ -408,15 +449,13 @@ The Python interpeter in the browser is Pyodide. You may not be able to run some } // Attempt to grab cache'd responses - fetch_from_backend("grabResponses", { - responses: [id], - }).then(function (json) { - if (json.responses && json.responses.length > 0) { + grabResponses([id]) + .then(function (resps) { // Store responses and set status to green checkmark - setLastResponses(stripLLMDetailsFromResponses(json.responses)); + setLastResponses(stripLLMDetailsFromResponses(resps)); setStatus(Status.READY); - } - }); + }) + .catch(handleError); }, []); // On upstream changes @@ -449,20 +488,17 @@ The Python interpeter in the browser is Pyodide. You may not be able to run some setLastRunLogs(""); setLastResponses([]); - const rejected = (err) => { - setStatus(Status.ERROR); - setLastRunSuccess(false); - if (typeof err !== "string") console.error(err); - alertModal.current.trigger(typeof err === "string" ? err : err?.message); - }; - // Get all the Python script nodes, and get all the folder paths // NOTE: Python only! let script_paths: string[] = []; if (progLang === "python") { const script_nodes = nodes.filter((n) => n.type === "script"); script_paths = script_nodes - .map((n) => Object.values(n.data.scriptFiles as Dict).filter((f: string) => f !== "")) + .map((n) => + Object.values(n.data.scriptFiles as Dict).filter( + (f: string) => f !== "", + ), + ) .flat(); } @@ -473,10 +509,12 @@ The Python interpeter in the browser is Pyodide. You may not be able to run some if (json?.logs) setLastRunLogs(json.logs.join("\n > ")); // Check if there's an error; if so, bubble it up to user and exit: - if (!json || json.error) { - rejected(json?.error); - return; - } + if (!json || json.error || json.responses === undefined) + throw new Error( + typeof json.error === "string" + ? json.error + : "Unknown error when running code evaluator.", + ); console.log(json.responses); @@ -495,13 +533,13 @@ The Python interpeter in the browser is Pyodide. You may not be able to run some .map((resp_obj) => resp_obj.responses.map((r) => { // Carry over the response text, prompt, prompt fill history (vars), and llm data - const o = { + const o: TemplateVarInfo = { text: escapeBraces(r), prompt: resp_obj.prompt, fill_history: resp_obj.vars, metavars: resp_obj.metavars || {}, llm: resp_obj.llm, - batch_id: resp_obj.uid, + uid: resp_obj.uid, }; // Carry over any chat history @@ -513,8 +551,13 @@ The Python interpeter in the browser is Pyodide. You may not be able to run some ) .flat(), }); + + if (status !== Status.READY && !showDrawer) + setUninspectedResponses(true); + + setStatus(Status.READY); }) - .catch(rejected); + .catch(handleError); }; const hideStatusIndicator = () => { diff --git a/chainforge/react-server/src/InspectFooter.tsx b/chainforge/react-server/src/InspectFooter.tsx index d299659..ed0846e 100644 --- a/chainforge/react-server/src/InspectFooter.tsx +++ b/chainforge/react-server/src/InspectFooter.tsx @@ -9,10 +9,10 @@ import { export interface InspectFooterProps { label: React.ReactNode; onClick: () => void; - showNotificationDot: boolean; showDrawerButton: boolean; onDrawerClick: () => void; isDrawerOpen: boolean; + showNotificationDot?: boolean; } /** @@ -22,10 +22,10 @@ export interface InspectFooterProps { const InspectFooter: React.FC = ({ label, onClick, - showNotificationDot, showDrawerButton, onDrawerClick, isDrawerOpen, + showNotificationDot, }) => { const text = useMemo( () => diff --git a/chainforge/react-server/src/LLMResponseInspectorModal.tsx b/chainforge/react-server/src/LLMResponseInspectorModal.tsx new file mode 100644 index 0000000..c689277 --- /dev/null +++ b/chainforge/react-server/src/LLMResponseInspectorModal.tsx @@ -0,0 +1,65 @@ +/** + * A fullscreen version of the Inspect node that + * appears in a Mantine modal pop-up which takes up much of the screen. + */ +import React, { forwardRef, useImperativeHandle } from "react"; +import { Modal } from "@mantine/core"; +import { useDisclosure } from "@mantine/hooks"; +import LLMResponseInspector, { exportToExcel } from "./LLMResponseInspector"; +import { StandardizedLLMResponse } from "./backend/typing"; + +export interface LLMResponseInspectorModalRef { + trigger: () => void; +} + +export interface LLMResponseInspectorModalProps { + jsonResponses: StandardizedLLMResponse[]; +} + +const LLMResponseInspectorModal = forwardRef< + LLMResponseInspectorModalRef, + LLMResponseInspectorModalProps +>(function LLMResponseInspectorModal({ jsonResponses }, ref) { + const [opened, { open, close }] = useDisclosure(false); + + // This gives the parent access to triggering the modal + const trigger = () => { + open(); + }; + useImperativeHandle(ref, () => ({ + trigger, + })); + + return ( + + Response Inspector + + + } + styles={{ title: { justifyContent: "space-between", width: "100%" } }} + > +
+ +
+ + ); +}); + +export default LLMResponseInspectorModal; diff --git a/chainforge/react-server/src/ModelSettingsModal.tsx b/chainforge/react-server/src/ModelSettingsModal.tsx index aab6b32..ac11726 100644 --- a/chainforge/react-server/src/ModelSettingsModal.tsx +++ b/chainforge/react-server/src/ModelSettingsModal.tsx @@ -17,7 +17,13 @@ import { getDefaultModelFormData, postProcessFormData, } from "./ModelSettingSchemas"; -import { Dict, Func, JSONCompatible, LLMSpec, ModelSettingsDict } from "./backend/typing"; +import { + Dict, + Func, + JSONCompatible, + LLMSpec, + ModelSettingsDict, +} from "./backend/typing"; export interface ModelSettingsModalHandle { trigger: () => void; @@ -28,7 +34,10 @@ export interface ModelSettingsModalProps { } type FormData = LLMSpec["formData"]; -const ModelSettingsModal = forwardRef(function ModelSettingsModal({model, onSettingsSubmit}, ref) { +const ModelSettingsModal = forwardRef< + ModelSettingsModalHandle, + ModelSettingsModalProps +>(function ModelSettingsModal({ model, onSettingsSubmit }, ref) { const [opened, { open, close }] = useDisclosure(false); const [formData, setFormData] = useState(undefined); @@ -42,8 +51,12 @@ const ModelSettingsModal = forwardRef({}); const [baseModelName, setBaseModelName] = useState("(unknown)"); - const [initShortname, setInitShortname] = useState(undefined); - const [initModelName, setInitModelName] = useState(undefined); + const [initShortname, setInitShortname] = useState( + undefined, + ); + const [initModelName, setInitModelName] = useState( + undefined, + ); // Totally necessary emoji picker const [modelEmoji, setModelEmoji] = useState(""); @@ -98,7 +111,7 @@ const ModelSettingsModal = forwardRef { - if (fdata === undefined) return; + if (fdata === undefined) return; // For some reason react-json-form-schema returns 'undefined' on empty strings. // We need to (1) detect undefined values for keys in formData and (2) if they are of type string, replace with "", // if that property is marked with a special "allow_empty_str" property. @@ -117,11 +130,7 @@ const ModelSettingsModal = forwardRef; + alertModal?: React.Ref; customButtons?: React.ReactElement[]; handleRunClick?: () => void; handleStopClick?: (nodeId: string) => void; @@ -58,7 +58,7 @@ export const NodeLabel: React.FC = ({ const removeNode = useStore((state) => state.removeNode); // For 'delete node' confirmation popup - const deleteConfirmModal = useRef(null); + const deleteConfirmModal = useRef(null); const [deleteConfirmProps, setDeleteConfirmProps] = useState({ title: "Delete node", diff --git a/chainforge/react-server/src/PlotLegend.tsx b/chainforge/react-server/src/PlotLegend.tsx index ba5985a..531f6bc 100644 --- a/chainforge/react-server/src/PlotLegend.tsx +++ b/chainforge/react-server/src/PlotLegend.tsx @@ -2,7 +2,13 @@ import React from "react"; import { Dict } from "./backend/typing"; import { truncStr } from "./backend/utils"; -const PlotLegend = ({ labels, onClickLabel }: { labels: Dict, onClickLabel: (label: string) => void }) => { +const PlotLegend = ({ + labels, + onClickLabel, +}: { + labels: Dict; + onClickLabel: (label: string) => void; +}) => { return (
{Object.entries(labels).map(([label, color]) => ( diff --git a/chainforge/react-server/src/PromptNode.tsx b/chainforge/react-server/src/PromptNode.tsx index c5a50fd..86980a5 100644 --- a/chainforge/react-server/src/PromptNode.tsx +++ b/chainforge/react-server/src/PromptNode.tsx @@ -51,7 +51,7 @@ import { StandardizedLLMResponse, TemplateVarInfo, } from "./backend/typing"; -import { AlertModalHandles } from "./AlertModal"; +import { AlertModalRef } from "./AlertModal"; import { Status } from "./StatusIndicatorComponent"; const getUniqueLLMMetavarKey = (responses: StandardizedLLMResponse[]) => { @@ -199,7 +199,7 @@ const PromptNode = ({ data, id, type: node_type }) => { const [llmItemsCurrState, setLLMItemsCurrState] = useState([]); // For displaying error messages to user - const alertModal = useRef(null); + const alertModal = useRef(null); // For a way to inspect responses without having to attach a dedicated node const inspectModal = useRef(null); @@ -441,7 +441,7 @@ const PromptNode = ({ data, id, type: node_type }) => { fill_history: info.fill_history, metavars: info.metavars, llm: info?.llm?.name, - batch_id: uuid(), + uid: uuid(), }; }); @@ -873,7 +873,7 @@ Soft failing by replacing undefined with empty strings.`, llm: _llmItemsCurrState.find( (item) => item.name === resp_obj.llm, ), - batch_id: resp_obj.uid, + uid: resp_obj.uid, }; // Carry over any metavars diff --git a/chainforge/react-server/src/RenameValueModal.tsx b/chainforge/react-server/src/RenameValueModal.tsx index ab3e718..22d8ee6 100644 --- a/chainforge/react-server/src/RenameValueModal.tsx +++ b/chainforge/react-server/src/RenameValueModal.tsx @@ -10,63 +10,62 @@ export interface RenameValueModalProps { onSubmit?: (val: string) => void; } -export interface RenameValueModalHandles { +export interface RenameValueModalRef { trigger: (msg?: string) => void; } /** Modal that lets user rename a single value, using a TextInput field. */ -const RenameValueModal = forwardRef< - RenameValueModalHandles, - RenameValueModalProps ->(function RenameValueModal({ initialValue, title, label, onSubmit }, ref) { - const [opened, { open, close }] = useDisclosure(false); - const form = useForm({ - initialValues: { - value: initialValue, - }, - validate: { - value: (v) => - v.trim().length > 0 - ? null - : "Column names must have at least one character", - }, - }); +const RenameValueModal = forwardRef( + function RenameValueModal({ initialValue, title, label, onSubmit }, ref) { + const [opened, { open, close }] = useDisclosure(false); + const form = useForm({ + initialValues: { + value: initialValue, + }, + validate: { + value: (v) => + v.trim().length > 0 + ? null + : "Column names must have at least one character", + }, + }); - useEffect(() => { - form.setValues({ value: initialValue }); - }, [initialValue]); + useEffect(() => { + form.setValues({ value: initialValue }); + }, [initialValue]); - // This gives the parent access to triggering the modal alert - const trigger = () => { - open(); - }; - useImperativeHandle(ref, () => ({ - trigger, - })); + // This gives the parent access to triggering the modal alert + const trigger = () => { + open(); + }; + useImperativeHandle(ref, () => ({ + trigger, + })); - return ( - - -
{ - if (onSubmit) onSubmit(values.value); - close(); - })} - > - + return ( + + + { + if (onSubmit) onSubmit(values.value); + close(); + })} + > + - - - - - - - ); -}); + + + + +
+ + ); + }, +); export default RenameValueModal; diff --git a/chainforge/react-server/src/TabularDataNode.tsx b/chainforge/react-server/src/TabularDataNode.tsx index 9ef39ac..c1ea325 100644 --- a/chainforge/react-server/src/TabularDataNode.tsx +++ b/chainforge/react-server/src/TabularDataNode.tsx @@ -12,8 +12,8 @@ import { import TemplateHooks from "./TemplateHooksComponent"; import BaseNode from "./BaseNode"; import NodeLabel from "./NodeLabelComponent"; -import AlertModal, { AlertModalHandles } from "./AlertModal"; -import RenameValueModal, { RenameValueModalHandles } from "./RenameValueModal"; +import AlertModal, { AlertModalRef } from "./AlertModal"; +import RenameValueModal, { RenameValueModalRef } from "./RenameValueModal"; import useStore from "./store"; import { sampleRandomElements } from "./backend/utils"; import { Dict, TabularDataRowType, TabularDataColType } from "./backend/typing"; @@ -91,10 +91,10 @@ const TabularDataNode: React.FC = ({ data, id }) => { const [hooksY, setHooksY] = useState(120); // For displaying error messages to user - const alertModal = useRef(null); + const alertModal = useRef(null); // For renaming a column - const renameColumnModal = useRef(null); + const renameColumnModal = useRef(null); const [renameColumnInitialVal, setRenameColumnInitialVal] = useState< TabularDataColType | string >(""); diff --git a/chainforge/react-server/src/backend/backend.ts b/chainforge/react-server/src/backend/backend.ts index a6db2ce..046a696 100644 --- a/chainforge/react-server/src/backend/backend.ts +++ b/chainforge/react-server/src/backend/backend.ts @@ -187,7 +187,9 @@ function load_from_cache(storageKey: string): Dict { return StorageCache.get(storageKey) || {}; } -function load_cache_responses(storageKey: string): Array { +function load_cache_responses( + storageKey: string, +): Dict | StandardizedLLMResponse[] { const data = load_from_cache(storageKey); if (Array.isArray(data)) return data; else if (typeof data === "object" && data.responses_last_run !== undefined) { @@ -1228,9 +1230,9 @@ export async function evalWithLLM( return { error: `Did not find cache file for id ${cache_id}` }; // Load the raw responses from the cache + clone them all: - const resp_objs = load_cache_responses(fname).map((r) => - JSON.parse(JSON.stringify(r)), - ) as StandardizedLLMResponse[]; + const resp_objs = ( + load_cache_responses(fname) as StandardizedLLMResponse[] + ).map((r) => JSON.parse(JSON.stringify(r))) as StandardizedLLMResponse[]; if (resp_objs.length === 0) continue; // We need to keep track of the index of each response in the response object. @@ -1337,15 +1339,17 @@ export async function evalWithLLM( * @returns If success, a Dict with a single key, 'responses', with an array of StandardizedLLMResponse objects * If failure, a Dict with a single key, 'error', with the error message. */ -export async function grabResponses(responses: Array): Promise { +export async function grabResponses( + responses: string[], +): Promise { // Grab all responses with the given ID: - let grabbed_resps: Dict[] = []; + let grabbed_resps: StandardizedLLMResponse[] = []; for (const cache_id of responses) { const storageKey = `${cache_id}.json`; if (!StorageCache.has(storageKey)) - return { error: `Did not find cache data for id ${cache_id}` }; + throw new Error(`Did not find cache data for id ${cache_id}`); - let res: Dict | Array<{ [key: string]: Dict }> = + let res: StandardizedLLMResponse[] | Dict = load_cache_responses(storageKey); if (typeof res === "object" && !Array.isArray(res)) { // Convert to standard response format @@ -1356,7 +1360,7 @@ export async function grabResponses(responses: Array): Promise { grabbed_resps = grabbed_resps.concat(res); } - return { responses: grabbed_resps }; + return grabbed_resps; } /** diff --git a/chainforge/react-server/src/backend/errors.ts b/chainforge/react-server/src/backend/errors.ts index 488b296..ec45f5b 100644 --- a/chainforge/react-server/src/backend/errors.ts +++ b/chainforge/react-server/src/backend/errors.ts @@ -1,6 +1,4 @@ export class DuplicateVariableNameError extends Error { - variable: string; - constructor(variable: string) { super(); this.name = "DuplicateVariableNameError"; diff --git a/chainforge/react-server/src/backend/typing.ts b/chainforge/react-server/src/backend/typing.ts index b75f0b2..38d96ed 100644 --- a/chainforge/react-server/src/backend/typing.ts +++ b/chainforge/react-server/src/backend/typing.ts @@ -201,7 +201,11 @@ export type LLMResponsesByVarDict = Dict< (BaseLLMResponseObject | StandardizedLLMResponse)[] >; -export type EvaluatedResponsesResults = ({responses?: StandardizedLLMResponse[], logs?: string[], error?: string}); +export type EvaluatedResponsesResults = { + responses?: StandardizedLLMResponse[]; + logs?: string[]; + error?: string; +}; /** The outputs of prompt nodes, text fields or other data passed internally in the front-end and to the PromptTemplate backend. * Used to populate prompt templates and carry variables/metavariables along the chain. */ @@ -210,10 +214,19 @@ export interface TemplateVarInfo { fill_history: Dict; metavars?: Dict; associate_id?: string; + prompt?: string; + uid?: ResponseUID; llm?: string | LLMSpec; chat_history?: ChatHistory; } +export type VarsContext = + | { + vars: string[]; + metavars: string[]; + } + | object; + export type PromptVarType = string | TemplateVarInfo; export type PromptVarsDict = { [key: string]: PromptVarType[]; @@ -229,3 +242,5 @@ export type TabularDataColType = { key: string; header: string; }; + +export type PythonInterpreter = "flask" | "pyodide"; diff --git a/chainforge/react-server/src/backend/utils.ts b/chainforge/react-server/src/backend/utils.ts index 74d3758..63d61ab 100644 --- a/chainforge/react-server/src/backend/utils.ts +++ b/chainforge/react-server/src/backend/utils.ts @@ -17,9 +17,9 @@ import { GeminiChatContext, GeminiChatMessage, StandardizedLLMResponse, - BaseLLMResponseObject, LLMResponsesByVarDict, Func, + VarsContext, } from "./typing"; import { v4 as uuid } from "uuid"; import { StringTemplate } from "./template"; @@ -1682,8 +1682,8 @@ export const getLLMsInPulledInputData = (pulled_data: Dict) => { }; export const stripLLMDetailsFromResponses = ( - resps: (StandardizedLLMResponse | BaseLLMResponseObject)[], -) => + resps: StandardizedLLMResponse[], +): StandardizedLLMResponse[] => resps.map((r) => ({ ...r, llm: typeof r?.llm === "string" ? r?.llm : r?.llm?.name ?? "undefined", @@ -1834,11 +1834,11 @@ export function sampleRandomElements(arr: any[], num_sample: number): any[] { return Array.from(idxs).map((idx) => arr[idx]); } -export const getVarsAndMetavars = (input_data: Dict) => { +export const getVarsAndMetavars = (input_data: Dict): VarsContext => { // Find all vars and metavars in the input data (if any): // NOTE: The typing is purposefully general for some backwards compatibility concenrs. - const varnames = new Set(); - const metavars = new Set(); + const varnames = new Set(); + const metavars = new Set(); const add_from_resp_obj = (resp_obj: Dict) => { if (typeof resp_obj === "string") return; diff --git a/chainforge/react-server/src/example_flows.js b/chainforge/react-server/src/example_flows.tsx similarity index 100% rename from chainforge/react-server/src/example_flows.js rename to chainforge/react-server/src/example_flows.tsx