mirror of
https://github.com/ianarawjo/ChainForge.git
synced 2025-03-14 08:16:37 +00:00
Autosaving, on the way to better plots (#78)
* Flow autosaving every 60 seconds * Set viewport upon resetFlow * Added x-axis, y-axis etc header to Vis node. Ensured left padding sizes to shortnames. * When num gen per prompt = 1, now plots single bar chart w solid LLM color * Rebuilt react and update package version
This commit is contained in:
ianarawjo
GitHub
parent
4bd5c15e82
commit
be9c176f7c
@ -1,15 +1,15 @@
|
||||
{
|
||||
"files": {
|
||||
"main.css": "/static/css/main.e7a9f023.css",
|
||||
"main.js": "/static/js/main.c2936487.js",
|
||||
"main.css": "/static/css/main.98db168d.css",
|
||||
"main.js": "/static/js/main.2c2b1596.js",
|
||||
"static/js/787.4c72bb55.chunk.js": "/static/js/787.4c72bb55.chunk.js",
|
||||
"index.html": "/index.html",
|
||||
"main.e7a9f023.css.map": "/static/css/main.e7a9f023.css.map",
|
||||
"main.c2936487.js.map": "/static/js/main.c2936487.js.map",
|
||||
"main.98db168d.css.map": "/static/css/main.98db168d.css.map",
|
||||
"main.2c2b1596.js.map": "/static/js/main.2c2b1596.js.map",
|
||||
"787.4c72bb55.chunk.js.map": "/static/js/787.4c72bb55.chunk.js.map"
|
||||
},
|
||||
"entrypoints": [
|
||||
"static/css/main.e7a9f023.css",
|
||||
"static/js/main.c2936487.js"
|
||||
"static/css/main.98db168d.css",
|
||||
"static/js/main.2c2b1596.js"
|
||||
]
|
||||
}
|
||||
@ -1 +1 @@
|
||||
<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="/favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#000000"/><meta name="description" content="Web site created using create-react-app"/><link rel="apple-touch-icon" href="/logo192.png"/><link rel="manifest" href="/manifest.json"/><title>Chainforge</title><script defer="defer" src="/static/js/main.c2936487.js"></script><link href="/static/css/main.e7a9f023.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
|
||||
<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="/favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#000000"/><meta name="description" content="Web site created using create-react-app"/><link rel="apple-touch-icon" href="/logo192.png"/><link rel="manifest" href="/manifest.json"/><title>Chainforge</title><script defer="defer" src="/static/js/main.2c2b1596.js"></script><link href="/static/css/main.98db168d.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
114
chainforge/react-server/src/App.js
vendored
114
chainforge/react-server/src/App.js
vendored
@ -1,12 +1,13 @@
|
||||
// import logo from './logo.svg';
|
||||
// import './App.css';
|
||||
|
||||
import React, { useState, useCallback, useRef } from 'react';
|
||||
import React, { useState, useCallback, useRef, useEffect } from 'react';
|
||||
import ReactFlow, {
|
||||
Controls,
|
||||
Background,
|
||||
useReactFlow,
|
||||
useViewport
|
||||
useViewport,
|
||||
setViewport,
|
||||
} from 'react-flow-renderer';
|
||||
import { Button, Menu, LoadingOverlay } from '@mantine/core';
|
||||
import { IconSettings, IconTextPlus, IconTerminal, IconCsv, IconSettingsAutomation } from '@tabler/icons-react';
|
||||
@ -55,6 +56,7 @@ const nodeTypes = {
|
||||
|
||||
// const connectionLineStyle = { stroke: '#ddd' };
|
||||
const snapGrid = [16, 16];
|
||||
let saveIntervalInitialized = false;
|
||||
|
||||
const App = () => {
|
||||
|
||||
@ -63,6 +65,7 @@ const App = () => {
|
||||
|
||||
// For saving / loading
|
||||
const [rfInstance, setRfInstance] = useState(null);
|
||||
const [autosavingInterval, setAutosavingInterval] = useState(null);
|
||||
|
||||
// For modal popup to set global settings like API keys
|
||||
const settingsModal = useRef(null);
|
||||
@ -80,7 +83,7 @@ const App = () => {
|
||||
const alertModal = useRef(null);
|
||||
|
||||
// For displaying a pending 'loading' status
|
||||
const [isLoading, setIsLoading] = useState(false);
|
||||
const [isLoading, setIsLoading] = useState(true);
|
||||
|
||||
// Helper
|
||||
const getWindowSize = () => ({width: window.innerWidth, height: window.innerHeight});
|
||||
@ -169,6 +172,19 @@ const App = () => {
|
||||
URL.revokeObjectURL(downloadLink.href);
|
||||
};
|
||||
|
||||
// Save the current flow to localStorage for later recall. Useful to getting
|
||||
// back progress upon leaving the site / browser crash / system restart.
|
||||
const saveFlow = useCallback((rf_inst) => {
|
||||
const rf = rf_inst || rfInstance;
|
||||
if (!rf) return;
|
||||
// NOTE: This currently only saves the front-end state. Cache files
|
||||
// are not pulled or overwritten upon loading from localStorage.
|
||||
const flow = rf.toObject();
|
||||
localStorage.setItem('chainforge-flow', JSON.stringify(flow));
|
||||
console.log('Flow saved!');
|
||||
}, [rfInstance]);
|
||||
|
||||
// Triggered when user confirms 'New Flow' button
|
||||
const resetFlow = useCallback(() => {
|
||||
resetLLMColors();
|
||||
|
||||
@ -180,24 +196,33 @@ const App = () => {
|
||||
|
||||
setNodes(starting_nodes);
|
||||
setEdges([]);
|
||||
}, [setNodes, setEdges, resetLLMColors]);
|
||||
const saveFlow = useCallback(() => {
|
||||
if (!rfInstance) return;
|
||||
const flow = rfInstance.toObject();
|
||||
localStorage.setItem('chainforge-flow', JSON.stringify(flow));
|
||||
}, [rfInstance]);
|
||||
if (rfInstance)
|
||||
rfInstance.setViewport({x: 200, y: 80, zoom: 1});
|
||||
}, [setNodes, setEdges, resetLLMColors, rfInstance]);
|
||||
|
||||
const loadFlow = async (flow) => {
|
||||
const loadFlow = async (flow, rf_inst) => {
|
||||
if (flow) {
|
||||
const { x = 0, y = 0, zoom = 1 } = flow.viewport;
|
||||
// setViewport({ x, y, zoom });
|
||||
if (rf_inst) {
|
||||
if (flow.viewport)
|
||||
rf_inst.setViewport({x: flow.viewport.x || 0, y: flow.viewport.y || 0, zoom: 1});
|
||||
else
|
||||
rf_inst.setViewport({x:0, y:0, zoom:1});
|
||||
}
|
||||
resetLLMColors();
|
||||
setNodes(flow.nodes || []);
|
||||
setEdges(flow.edges || []);
|
||||
|
||||
// Save flow that user loaded to autosave cache, in case they refresh the browser
|
||||
localStorage.setItem('chainforge-flow', JSON.stringify(flow));
|
||||
}
|
||||
};
|
||||
const loadFlowFromCache = async () => {
|
||||
loadFlow(JSON.parse(localStorage.getItem('chainforge-flow')));
|
||||
const autosavedFlowExists = () => {
|
||||
return localStorage.getItem('chainforge-flow') !== null;
|
||||
};
|
||||
const loadFlowFromAutosave = async (rf_inst) => {
|
||||
const saved_flow = localStorage.getItem('chainforge-flow');
|
||||
if (saved_flow)
|
||||
loadFlow(JSON.parse(saved_flow), rf_inst);
|
||||
};
|
||||
|
||||
// Export / Import (from JSON)
|
||||
@ -251,11 +276,11 @@ const App = () => {
|
||||
}, handleError).catch(handleError);
|
||||
};
|
||||
|
||||
const importFlowFromJSON = (flowJSON) => {
|
||||
const importFlowFromJSON = useCallback((flowJSON) => {
|
||||
// Detect if there's no cache data
|
||||
if (!flowJSON.cache) {
|
||||
// Support for loading old flows w/o cache data:
|
||||
loadFlow(flowJSON);
|
||||
loadFlow(flowJSON, rfInstance);
|
||||
return;
|
||||
}
|
||||
|
||||
@ -267,13 +292,13 @@ const App = () => {
|
||||
// before we can load the flow itself...
|
||||
importCache(cache).then(() => {
|
||||
// We load the ReactFlow instance last
|
||||
loadFlow(flow);
|
||||
loadFlow(flow, rfInstance);
|
||||
}).catch(err => {
|
||||
// On an error, still try to load the flow itself:
|
||||
handleError("Error encountered when importing cache data:" + err.message + "\n\nTrying to load flow regardless...");
|
||||
loadFlow(flow);
|
||||
loadFlow(flow, rfInstance);
|
||||
});
|
||||
};
|
||||
}, [rfInstance]);
|
||||
|
||||
// Import a ChainForge flow from a file
|
||||
const importFlowFromFile = async () => {
|
||||
@ -312,9 +337,6 @@ const App = () => {
|
||||
|
||||
// Downloads the selected OpenAI eval file (preconverted to a .cforge flow)
|
||||
const importFlowFromOpenAIEval = (evalname) => {
|
||||
// Trigger the 'loading' modal
|
||||
setIsLoading(true);
|
||||
|
||||
fetch(BASE_URL + 'app/fetchOpenAIEval', {
|
||||
method: 'POST',
|
||||
headers: {'Content-Type': 'application/json', 'Access-Control-Allow-Origin': '*'},
|
||||
@ -343,7 +365,9 @@ const App = () => {
|
||||
|
||||
// Load flow from examples modal
|
||||
const onSelectExampleFlow = (name, example_category) => {
|
||||
console.log(name, example_category);
|
||||
// Trigger the 'loading' modal
|
||||
setIsLoading(true);
|
||||
|
||||
// Detect a special category of the example flow, and use the right loader for it:
|
||||
if (example_category === 'openai-eval') {
|
||||
importFlowFromOpenAIEval(name);
|
||||
@ -360,6 +384,9 @@ const App = () => {
|
||||
}, handleError).then(function(res) {
|
||||
return res.json();
|
||||
}, handleError).then(function(json) {
|
||||
// Close the loading modal
|
||||
setIsLoading(false);
|
||||
|
||||
if (!json)
|
||||
throw new Error('Request to fetch example flow was sent to backend server, but there was no response.');
|
||||
else if (json.error || !json.data)
|
||||
@ -384,6 +411,43 @@ const App = () => {
|
||||
confirmationModal.current.trigger();
|
||||
}, [confirmationModal, resetFlow, setConfirmationDialogProps]);
|
||||
|
||||
// Run once upon ReactFlow initialization
|
||||
const onInit = (rf_inst) => {
|
||||
setRfInstance(rf_inst);
|
||||
|
||||
// Autosave the flow to localStorage every minute:
|
||||
console.log('set autosaving interval');
|
||||
const interv = setInterval(() => saveFlow(rf_inst), 60000); // 60000 milliseconds = 1 minute
|
||||
setAutosavingInterval(interv);
|
||||
|
||||
// Attempt to load an autosaved flow, if one exists:
|
||||
if (autosavedFlowExists())
|
||||
loadFlowFromAutosave(rf_inst);
|
||||
else {
|
||||
// Create a default starting flow for new users
|
||||
// NOTE: We need to create a unique ID using the current date,
|
||||
// because of the way ReactFlow saves and restores states.
|
||||
const uid = (id) => `${id}-${Date.now()}`;
|
||||
setNodes([
|
||||
{ id: uid('prompt'), type: 'prompt', data: { prompt: 'What is the opening sentence of Pride and Prejudice?', n: 1 }, position: { x: 450, y: 200 } },
|
||||
{ id: uid('eval'), type: 'evaluator', data: { code: "def evaluate(response):\n return len(response.text)" }, position: { x: 820, y: 150 } },
|
||||
{ id: uid('textfields'), type: 'textfields', data: {}, position: { x: 80, y: 270 } },
|
||||
{ id: uid('vis'), type: 'vis', data: {}, position: { x: 1200, y: 250 } },
|
||||
{ id: uid('inspect'), type: 'inspect', data: {}, position: { x:820, y:400 } },
|
||||
]);
|
||||
}
|
||||
|
||||
// Turn off loading wheel
|
||||
setIsLoading(false);
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
// Cleanup the autosaving interval upon component unmount:
|
||||
return () => {
|
||||
clearInterval(autosavingInterval); // Clear the interval when the component is unmounted
|
||||
};
|
||||
}, []);
|
||||
|
||||
return (
|
||||
<div>
|
||||
<GlobalSettingsModal ref={settingsModal} />
|
||||
@ -406,7 +470,7 @@ const App = () => {
|
||||
// connectionLineStyle={connectionLineStyle}
|
||||
snapToGrid={true}
|
||||
snapGrid={snapGrid}
|
||||
onInit={setRfInstance}
|
||||
onInit={onInit}
|
||||
>
|
||||
<Background color="#999" gap={16} />
|
||||
<Controls showZoom={true} />
|
||||
@ -438,7 +502,7 @@ const App = () => {
|
||||
<div style={{position: 'fixed', right: '10px', top: '10px', zIndex: 8}}>
|
||||
<Button onClick={onClickNewFlow} size="sm" variant="outline" compact mr='xs' style={{float: 'left'}}> New Flow </Button>
|
||||
<Button onClick={onClickExamples} size="sm" variant="outline" compact mr='xs' style={{float: 'left'}}> Example Flows </Button>
|
||||
<Button onClick={onClickSettings} size="sm" variant="filled" compact><IconSettings size={"90%"} /></Button>
|
||||
<Button onClick={onClickSettings} size="sm" variant="outline" compact><IconSettings size={"90%"} /></Button>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
|
||||
@ -21,7 +21,7 @@ const AreYouSureModal = forwardRef(({title, message, onConfirm}, ref) => {
|
||||
}
|
||||
|
||||
return (
|
||||
<Modal opened={opened} onClose={close} title={title} styles={{header: {backgroundColor: 'orange', color: 'white'}, root: {position: 'relative', left: '-80px'}}}>
|
||||
<Modal opened={opened} onClose={close} title={title} styles={{header: {backgroundColor: 'red', color: 'white'}, root: {position: 'relative', left: '-80px'}}}>
|
||||
<Box maw={400} mx="auto" mt='md' mb='md'>
|
||||
<Text>{description}</Text>
|
||||
</Box>
|
||||
|
||||
@ -5,7 +5,7 @@
|
||||
* be deployed in multiple locations.
|
||||
*/
|
||||
import React, { useState, useEffect, useRef } from 'react';
|
||||
import { Collapse, MultiSelect } from '@mantine/core';
|
||||
import { Collapse, MultiSelect, ScrollArea } from '@mantine/core';
|
||||
import { useDisclosure } from '@mantine/hooks';
|
||||
import * as XLSX from 'xlsx';
|
||||
import useStore from './store';
|
||||
@ -322,7 +322,7 @@ const LLMResponseInspector = ({ jsonResponses, wideFormat }) => {
|
||||
setMultiSelectValue(new_val);
|
||||
};
|
||||
|
||||
return (<div>
|
||||
return (<div style={{height: '100%'}}>
|
||||
<MultiSelect ref={multiSelectRef}
|
||||
onChange={handleMultiSelectValueChange}
|
||||
className='nodrag nowheel inspect-multiselect'
|
||||
|
||||
25
chainforge/react-server/src/PromptNode.js
vendored
25
chainforge/react-server/src/PromptNode.js
vendored
@ -42,6 +42,13 @@ const ensureUniqueName = (_name, _prev_names) => {
|
||||
}
|
||||
return new_name;
|
||||
};
|
||||
const getUniqueLLMMetavarKey = (responses) => {
|
||||
const metakeys = new Set(responses.map(resp_obj => Object.keys(resp_obj.metavars)));
|
||||
let i = 0;
|
||||
while (`LLM_${i}` in metakeys)
|
||||
i += 1;
|
||||
return `LLM_${i}`;
|
||||
};
|
||||
|
||||
const PromptNode = ({ data, id }) => {
|
||||
|
||||
@ -413,6 +420,8 @@ const PromptNode = ({ data, id }) => {
|
||||
// On connect to the server, ask it to give us the current progress
|
||||
// for task 'queryllm' with id 'id', and stop when it reads progress >= 'max'.
|
||||
socket.on("connect", (msg) => {
|
||||
if (FINISHED_QUERY) return;
|
||||
|
||||
// Initialize progress bars to small amounts
|
||||
setProgress({ success: 2, error: 0 });
|
||||
setLLMItems(llmItemsCurrState.map(item => {
|
||||
@ -533,9 +542,23 @@ const PromptNode = ({ data, id }) => {
|
||||
setPromptTextOnLastRun(promptText);
|
||||
|
||||
// Save response texts as 'fields' of data, for any prompt nodes pulling the outputs
|
||||
// First we need to get a unique key for a unique metavar for the LLM set that produced these responses,
|
||||
// so we can keep track of 'upstream' LLMs (and plot against them) later on:
|
||||
const llm_metavar_key = getUniqueLLMMetavarKey(json.responses);
|
||||
setDataPropsForNode(id, {fields: json.responses.map(
|
||||
resp_obj => resp_obj['responses'].map(
|
||||
r => ({text: r, fill_history: resp_obj['vars']}))).flat()
|
||||
r => {
|
||||
// Carry over the response text and prompt fill history (vars):
|
||||
let o = {text: r, fill_history: resp_obj['vars']};
|
||||
|
||||
// Carry over any metavars
|
||||
o.metavars = resp_obj['metavars'] || {};
|
||||
|
||||
// Add a meta var to keep track of which LLM produced this response
|
||||
o.metavars[llm_metavar_key] = resp_obj['llm'];
|
||||
return o;
|
||||
}
|
||||
)).flat()
|
||||
});
|
||||
|
||||
// Ping any inspect nodes attached to this node to refresh their contents:
|
||||
|
||||
157
chainforge/react-server/src/VisNode.js
vendored
157
chainforge/react-server/src/VisNode.js
vendored
@ -50,12 +50,29 @@ const extractEvalResultsForMetric = (metric, responses) => {
|
||||
const areSetsEqual = (xs, ys) =>
|
||||
xs.size === ys.size &&
|
||||
[...xs].every((x) => ys.has(x));
|
||||
const genUniqueShortnames = (names) => {
|
||||
|
||||
function addLineBreaks(str, max_line_len) {
|
||||
let result = '';
|
||||
const is_alphabetical = (s) => /^[A-Za-z]$/.test(s);
|
||||
for (var i = 0; i < str.length; i++) {
|
||||
result += str[i];
|
||||
if ((i + 1) % max_line_len === 0) {
|
||||
const next_char = i+1 < str.length ? str[i+1] : '';
|
||||
result += (is_alphabetical(str[i]) && is_alphabetical(next_char) ? '-' : '') + '<br>';
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
const genUniqueShortnames = (names, max_chars_per_line=32) => {
|
||||
// Generate unique 'shortnames' to refer to each name:
|
||||
let past_shortnames_counts = {};
|
||||
let shortnames = {};
|
||||
const max_lines = 2;
|
||||
for (const name of names) {
|
||||
const sn = truncStr(name, 16);
|
||||
// Truncate string up to maximum num of chars
|
||||
let sn = truncStr(name, max_chars_per_line * max_lines - 3);
|
||||
// Add <br> tags to spread across multiple lines, where necessary
|
||||
sn = addLineBreaks(sn, max_chars_per_line);
|
||||
if (sn in past_shortnames_counts) {
|
||||
past_shortnames_counts[sn] += 1;
|
||||
shortnames[name] = sn + `(${past_shortnames_counts[sn]})`;
|
||||
@ -66,6 +83,20 @@ const genUniqueShortnames = (names) => {
|
||||
}
|
||||
return shortnames;
|
||||
};
|
||||
const calcMaxCharsPerLine = (shortnames) => {
|
||||
let max_chars = 1;
|
||||
for (let i = 0; i < shortnames.length; i++) {
|
||||
const sn = shortnames[i];
|
||||
if (sn.includes('<br>'))
|
||||
return sn.indexOf('<br>');
|
||||
else if (sn.length > max_chars)
|
||||
max_chars = sn.length;
|
||||
}
|
||||
return Math.max(max_chars, 9);
|
||||
};
|
||||
const calcLeftPaddingForYLabels = (shortnames) => {
|
||||
return calcMaxCharsPerLine(shortnames) * 7;
|
||||
}
|
||||
|
||||
const VisNode = ({ data, id }) => {
|
||||
|
||||
@ -129,9 +160,13 @@ const VisNode = ({ data, id }) => {
|
||||
const varcolors = colorPalettes.var; // ['#44d044', '#f1b933', '#e46161', '#8888f9', '#33bef0', '#bb55f9', '#cadefc', '#f8f398'];
|
||||
let spec = [];
|
||||
let layout = {
|
||||
autosize: true, title: '', margin: {
|
||||
autosize: true,
|
||||
dragmode: 'pan',
|
||||
title: '',
|
||||
margin: {
|
||||
l: 125, r: 0, b: 36, t: 20, pad: 6
|
||||
}
|
||||
},
|
||||
yaxis: {showgrid: true},
|
||||
};
|
||||
|
||||
// Bucket responses by LLM:
|
||||
@ -153,6 +188,14 @@ const VisNode = ({ data, id }) => {
|
||||
if (is_all_bools) typeof_eval_res = 'Boolean';
|
||||
}
|
||||
|
||||
// Check the max length of eval results, as if it's only 1 score per item (num of generations per prompt n=1),
|
||||
// we might want to plot the result differently:
|
||||
let max_num_results_per_prompt = 1;
|
||||
responses.forEach(res_obj => {
|
||||
if (res_obj.eval_res?.items?.length > max_num_results_per_prompt)
|
||||
max_num_results_per_prompt = res_obj.eval_res.items.length;
|
||||
});
|
||||
|
||||
let plot_legend = null;
|
||||
let metric_axes_labels = [];
|
||||
let num_metrics = 1;
|
||||
@ -220,8 +263,8 @@ const VisNode = ({ data, id }) => {
|
||||
name_idx += 1;
|
||||
}
|
||||
|
||||
// Set the left margin to fit the text
|
||||
layout.margin.l = Math.max(...y_items.map(i => i.length)) * 9;
|
||||
// Set the left margin to fit the yticks labels
|
||||
layout.margin.l = calcLeftPaddingForYLabels(Object.values(shortnames));
|
||||
|
||||
spec = [{
|
||||
type: 'bar',
|
||||
@ -267,8 +310,7 @@ const VisNode = ({ data, id }) => {
|
||||
names = new Set(responses.map(resp_to_x));
|
||||
}
|
||||
|
||||
const shortnames = genUniqueShortnames(names);
|
||||
|
||||
const shortnames = genUniqueShortnames(names);
|
||||
let name_idx = 0;
|
||||
for (const name of names) {
|
||||
let x_items = [];
|
||||
@ -292,7 +334,8 @@ const VisNode = ({ data, id }) => {
|
||||
// otherwise use the palette for displaying variables.
|
||||
const color = (group_type === 'llm') ?
|
||||
getColorForLLMAndSetIfNotFound(name)
|
||||
: varcolors[name_idx % varcolors.length];
|
||||
//: varcolors[name_idx % varcolors.length];
|
||||
: getColorForLLMAndSetIfNotFound(responses[0].llm);
|
||||
|
||||
if (typeof_eval_res === 'Boolean' || typeof_eval_res === 'Categorical') {
|
||||
// Plot a histogram for categorical data.
|
||||
@ -305,24 +348,41 @@ const VisNode = ({ data, id }) => {
|
||||
orientation: 'h',
|
||||
});
|
||||
layout.barmode = "stack";
|
||||
layout.yaxis = { showticklabels: true, dtick: 1, type: 'category' };
|
||||
layout.yaxis = { showticklabels: true, dtick: 1, type: 'category', showgrid: true };
|
||||
layout.xaxis = { title: { font: {size: 12}, text: "Number of 'true' values" }, ...layout.xaxis};
|
||||
} else {
|
||||
// Plot a boxplot for all other cases.
|
||||
spec.push({
|
||||
type: 'box',
|
||||
// x_items = [x_items.reduce((val, acc) => val + acc, 0)];
|
||||
let d = {
|
||||
name: shortnames[name],
|
||||
boxpoints: 'all',
|
||||
x: x_items,
|
||||
text: text_items,
|
||||
hovertemplate: '%{text}',
|
||||
orientation: 'h',
|
||||
marker: { color: color }
|
||||
});
|
||||
};
|
||||
|
||||
// If only one result, plot a bar chart:
|
||||
if (x_items.length === 1) {
|
||||
d.type = 'bar';
|
||||
d.textposition = 'none'; // hide the text which appears within each bar
|
||||
d.y = new Array(x_items.length).fill(shortnames[name]);
|
||||
} else {
|
||||
// If multiple eval results per response object (num generations per prompt n > 1),
|
||||
// plot box-and-whiskers to illustrate the variability:
|
||||
d.type = 'box';
|
||||
d.boxpoints = 'all';
|
||||
}
|
||||
spec.push(d);
|
||||
}
|
||||
|
||||
name_idx += 1;
|
||||
}
|
||||
layout.hovermode = 'closest';
|
||||
layout.showlegend = false;
|
||||
|
||||
// Set the left margin to fit the yticks labels
|
||||
layout.margin.l = calcLeftPaddingForYLabels(Object.values(shortnames));
|
||||
|
||||
if (metric_axes_labels.length > 0)
|
||||
layout.xaxis = {
|
||||
@ -364,10 +424,10 @@ const VisNode = ({ data, id }) => {
|
||||
orientation: 'h',
|
||||
});
|
||||
layout.barmode = "stack";
|
||||
layout.xaxis = { title: { font: {size: 12}, text: "Number of 'true' values" }, ...layout.xaxis};
|
||||
} else {
|
||||
// Plot a boxplot for all other cases.
|
||||
spec.push({
|
||||
type: 'box',
|
||||
// Plot a boxplot or bar chart for other cases.
|
||||
let d = {
|
||||
name: llm,
|
||||
marker: {color: getColorForLLMAndSetIfNotFound(llm)},
|
||||
x: x_items,
|
||||
@ -376,12 +436,31 @@ const VisNode = ({ data, id }) => {
|
||||
text: text_items,
|
||||
hovertemplate: '%{text} <b><i>(%{x})</i></b>',
|
||||
orientation: 'h',
|
||||
});
|
||||
};
|
||||
|
||||
// If only one result, plot a bar chart:
|
||||
if (max_num_results_per_prompt === 1) {
|
||||
d.type = 'bar';
|
||||
d.textposition = 'none'; // hide the text which appears within each bar
|
||||
} else {
|
||||
// If multiple eval results per response object (num generations per prompt n > 1),
|
||||
// plot box-and-whiskers to illustrate the variability:
|
||||
d.type = 'box';
|
||||
}
|
||||
|
||||
spec.push(d);
|
||||
layout.xaxis = {
|
||||
title: { font: {size: 12}, text: 'score' },
|
||||
...layout.axis
|
||||
};
|
||||
}
|
||||
});
|
||||
layout.boxmode = 'group';
|
||||
layout.bargap = 0.5;
|
||||
|
||||
// Set the left margin to fit the yticks labels
|
||||
layout.margin.l = calcLeftPaddingForYLabels(Object.values(shortnames));
|
||||
|
||||
if (metric_axes_labels.length > 0)
|
||||
layout.xaxis = {
|
||||
title: { font: {size: 12}, text: metric_axes_labels[0] },
|
||||
@ -585,7 +664,6 @@ const VisNode = ({ data, id }) => {
|
||||
if (!multiSelectVars || !multiSelectValue || !areSetsEqual(new Set(msvars.map(o => o.value)), new Set(multiSelectVars.map(o => o.value)))) {
|
||||
setMultiSelectValue([]);
|
||||
setMultiSelectVars(msvars);
|
||||
console.log('here');
|
||||
setDataPropsForNode(id, { vars: msvars, selected_vars: [] });
|
||||
}
|
||||
// From here a React effect will detect the changes to these values and display a new plot
|
||||
@ -638,14 +716,39 @@ const VisNode = ({ data, id }) => {
|
||||
nodeId={id}
|
||||
status={status}
|
||||
icon={'📊'} />
|
||||
<MultiSelect ref={multiSelectRef}
|
||||
onChange={handleMultiSelectValueChange}
|
||||
className='nodrag nowheel'
|
||||
data={multiSelectVars}
|
||||
placeholder="Pick params you wish to plot"
|
||||
size="sm"
|
||||
value={multiSelectValue}
|
||||
searchable />
|
||||
<div style={{display: 'flex', justifyContent: 'center', flexWrap: 'wrap'}}>
|
||||
<div style={{display: 'inline-flex', maxWidth: '50%'}}>
|
||||
<span style={{fontSize: '10pt', margin: '6pt 3pt 0 3pt', fontWeight: 'bold', whiteSpace: 'nowrap'}}>y-axis:</span>
|
||||
<MultiSelect ref={multiSelectRef}
|
||||
onChange={handleMultiSelectValueChange}
|
||||
className='nodrag nowheel'
|
||||
data={multiSelectVars}
|
||||
placeholder="Pick param to plot"
|
||||
size="xs"
|
||||
value={multiSelectValue}
|
||||
miw='80px'
|
||||
searchable />
|
||||
</div>
|
||||
<div style={{display: 'inline-flex', justifyContent: 'space-evenly', maxWidth: '30%', marginLeft: '10pt'}}>
|
||||
<span style={{fontSize: '10pt', margin: '6pt 3pt 0 0', fontWeight: 'bold', whiteSpace: 'nowrap'}}>x-axis:</span>
|
||||
<MultiSelect className='nodrag nowheel'
|
||||
data={['score']}
|
||||
size="xs"
|
||||
value={['score']}
|
||||
miw='80px'
|
||||
disabled />
|
||||
</div>
|
||||
<div style={{display: 'inline-flex', justifyContent: 'space-evenly', maxWidth: '30%', marginLeft: '10pt'}}>
|
||||
<span style={{fontSize: '10pt', margin: '6pt 3pt 0 0', fontWeight: 'bold', whiteSpace: 'nowrap'}}>group by:</span>
|
||||
<MultiSelect className='nodrag nowheel'
|
||||
data={['LLM']}
|
||||
size="xs"
|
||||
value={['LLM']}
|
||||
miw='80px'
|
||||
disabled />
|
||||
</div>
|
||||
</div>
|
||||
<hr />
|
||||
<div className="nodrag" ref={setPlotDivRef} style={{minWidth: '150px', minHeight: '100px'}}>
|
||||
{plotlySpec && plotlySpec.length > 0 ? <></> : placeholderText}
|
||||
<Plot
|
||||
|
||||
18
chainforge/react-server/src/store.js
vendored
18
chainforge/react-server/src/store.js
vendored
@ -9,21 +9,7 @@ import {
|
||||
// Where the ChainForge Flask server is being hosted.
|
||||
export const BASE_URL = 'http://localhost:8000/';
|
||||
|
||||
// We need to create a unique ID using the current date,
|
||||
// because of the way ReactFlow saves and restores states.
|
||||
const uid = (id) => `${id}-${Date.now()}`;
|
||||
|
||||
// Initial project settings
|
||||
const initprompt = uid('prompt');
|
||||
const initeval = uid('eval');
|
||||
const initialNodes = [
|
||||
{ id: initprompt, type: 'prompt', data: { prompt: 'What is an example of ownership and borrowing in Rust?', n: 1 }, position: { x: 450, y: 200 } },
|
||||
{ id: initeval, type: 'evaluator', data: { code: "def evaluate(response):\n return len(response.text)" }, position: { x: 820, y: 150 } },
|
||||
{ id: uid('textfields'), type: 'textfields', data: {}, position: { x: 80, y: 270 } },
|
||||
{ id: uid('vis'), type: 'vis', data: {}, position: { x: 1200, y: 250 } },
|
||||
{ id: uid('inspect'), type: 'inspect', data: {}, position: { x:820, y:400 } },
|
||||
];
|
||||
const initialEdges = [];
|
||||
const initialAPIKeys = {};
|
||||
const initialLLMColors = {};
|
||||
|
||||
@ -44,8 +30,8 @@ export const colorPalettes = {
|
||||
// A global store of variables, used for maintaining state
|
||||
// across ChainForge and ReactFlow components.
|
||||
const useStore = create((set, get) => ({
|
||||
nodes: initialNodes,
|
||||
edges: initialEdges,
|
||||
nodes: [],
|
||||
edges: [],
|
||||
|
||||
// Keeping track of LLM API keys
|
||||
apiKeys: initialAPIKeys,
|
||||
|
||||
@ -181,6 +181,9 @@
|
||||
/* #cef5dab1; */
|
||||
cursor: zoom-in;
|
||||
}
|
||||
.eval-inspect-response-footer button {
|
||||
cursor: zoom-in;
|
||||
}
|
||||
|
||||
.ace-editor-container {
|
||||
resize:vertical;
|
||||
@ -212,6 +215,10 @@
|
||||
opacity: 0.8;
|
||||
}
|
||||
|
||||
g.ytick text {
|
||||
line-height: 2px;
|
||||
}
|
||||
|
||||
.response-var-inline > .response-var-name {
|
||||
font-size: 8pt;
|
||||
font-weight: normal;
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user