mirror of
https://github.com/ianarawjo/ChainForge.git
synced 2025-03-14 16:26:45 +00:00
Let user group responses in InspectNodes
This commit is contained in:
Ian Arawjo
parent
f0c506d242
commit
c44de49615
@ -1,4 +1,4 @@
|
||||
import React, { useState, useEffect } from 'react';
|
||||
import React, { useState, useEffect, useRef } from 'react';
|
||||
import { Handle } from 'react-flow-renderer';
|
||||
import { Badge, MultiSelect } from '@mantine/core';
|
||||
import useStore from './store';
|
||||
@ -11,7 +11,14 @@ const truncStr = (s, maxLen) => {
|
||||
return s.substring(0, maxLen) + '...'
|
||||
else
|
||||
return s;
|
||||
}
|
||||
};
|
||||
const filterDict = (dict, keyFilterFunc) => {
|
||||
return Object.keys(dict).reduce((acc, key) => {
|
||||
if (keyFilterFunc(key) === true)
|
||||
acc[key] = dict[key];
|
||||
return acc;
|
||||
}, {});
|
||||
};
|
||||
const vars_to_str = (vars) => {
|
||||
const pairs = Object.keys(vars).map(varname => {
|
||||
const s = truncStr(vars[varname].trim(), 12);
|
||||
@ -36,6 +43,7 @@ const groupResponsesBy = (responses, keyFunc) => {
|
||||
const InspectorNode = ({ data, id }) => {
|
||||
|
||||
const [responses, setResponses] = useState([]);
|
||||
const [jsonResponses, setJSONResponses] = useState(null);
|
||||
const [pastInputs, setPastInputs] = useState([]);
|
||||
const inputEdgesForNode = useStore((state) => state.inputEdgesForNode);
|
||||
const setDataPropsForNode = useStore((state) => state.setDataPropsForNode);
|
||||
@ -44,6 +52,123 @@ const InspectorNode = ({ data, id }) => {
|
||||
const [multiSelectVars, setMultiSelectVars] = useState(data.vars || []);
|
||||
const [multiSelectValue, setMultiSelectValue] = useState(data.selected_vars || []);
|
||||
|
||||
// Update the visualization when the MultiSelect values change:
|
||||
useEffect(() => {
|
||||
if (!jsonResponses || (Array.isArray(jsonResponses) && jsonResponses.length === 0))
|
||||
return;
|
||||
|
||||
const responses = jsonResponses;
|
||||
const selected_vars = multiSelectValue;
|
||||
|
||||
// Find all LLMs in responses and store as array
|
||||
let found_llms = new Set();
|
||||
responses.forEach(res_obj =>
|
||||
found_llms.add(res_obj.llm));
|
||||
found_llms = Array.from(found_llms);
|
||||
|
||||
// Assign a color to each LLM in responses
|
||||
const llm_colors = ['#ace1aeb1', '#f1b963b1', '#e46161b1', '#f8f398b1', '#defcf9b1', '#cadefcb1', '#c3bef0b1', '#cca8e9b1'];
|
||||
const color_for_llm = (llm) => llm_colors[found_llms.indexOf(llm) % llm_colors.length];
|
||||
const response_box_colors = ['#ddd', '#eee', '#ddd', '#eee'];
|
||||
const rgroup_color = (depth) => response_box_colors[depth % response_box_colors.length];
|
||||
|
||||
const getHeaderBadge = (key, val) => {
|
||||
if (val) {
|
||||
const s = truncStr(val.trim(), 12);
|
||||
const txt = `${key} = '${s}'`;
|
||||
return (<Badge key={val} color="blue" size="xs">{txt}</Badge>);
|
||||
} else {
|
||||
return (<Badge key={'unspecified'} color="blue" size="xs">{`(unspecified ${key})`}</Badge>);
|
||||
}
|
||||
};
|
||||
|
||||
// Now we need to perform groupings by each var in the selected vars list,
|
||||
// nesting the groupings (preferrably with custom divs) and sorting within
|
||||
// each group by value of that group's var (so all same values are clumped together).
|
||||
// :: For instance, for varnames = ['LLM', '$var1', '$var2'] we should get back
|
||||
// :: nested divs first grouped by LLM (first level), then by var1, then var2 (deepest level).
|
||||
const groupByVars = (resps, varnames, eatenvars, header) => {
|
||||
if (resps.length === 0) return [];
|
||||
if (varnames.length === 0) {
|
||||
// Base case. Display n response(s) to each single prompt, back-to-back:
|
||||
const resp_boxes = resps.map((res_obj, res_idx) => {
|
||||
// Spans for actual individual response texts
|
||||
const ps = res_obj.responses.map((r, idx) =>
|
||||
(<pre className="small-response" key={idx}>{r}</pre>)
|
||||
);
|
||||
|
||||
// At the deepest level, there may still be some vars left over. We want to display these
|
||||
// as tags, too, so we need to display only the ones that weren't 'eaten' during the recursive call:
|
||||
// (e.g., the vars that weren't part of the initial 'varnames' list that form the groupings)
|
||||
const unused_vars = filterDict(res_obj.vars, v => !eatenvars.includes(v));
|
||||
const vars = vars_to_str(unused_vars);
|
||||
const var_tags = vars.map((v) =>
|
||||
(<Badge key={v} color="blue" size="xs">{v}</Badge>)
|
||||
);
|
||||
return (
|
||||
<div key={"r"+res_idx} className="response-box" style={{ backgroundColor: color_for_llm(res_obj.llm) }}>
|
||||
{var_tags}
|
||||
{eatenvars.includes('LLM') ?
|
||||
ps
|
||||
: (<div className="response-item-llm-name-wrapper">
|
||||
{ps}
|
||||
<h1>{res_obj.llm}</h1>
|
||||
</div>)
|
||||
}
|
||||
</div>
|
||||
);
|
||||
});
|
||||
const className = eatenvars.length > 0 ? "response-group" : "";
|
||||
const boxesClassName = eatenvars.length > 0 ? "response-boxes-wrapper" : "";
|
||||
return (
|
||||
<div className={className} style={{ backgroundColor: rgroup_color(eatenvars.length) }}>
|
||||
{header}
|
||||
<div className={boxesClassName}>
|
||||
{resp_boxes}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
// Bucket responses by the first var in the list, where
|
||||
// we also bucket any 'leftover' responses that didn't have the requested variable (a kind of 'soft fail')
|
||||
const group_name = varnames[0];
|
||||
const [grouped_resps, leftover_resps] = (group_name === 'LLM')
|
||||
? groupResponsesBy(resps, (r => r.llm))
|
||||
: groupResponsesBy(resps, (r => ((group_name in r.vars) ? r.vars[group_name] : null)));
|
||||
const get_header = (group_name === 'LLM')
|
||||
? ((key, val) => (<Badge key={val} color="violet" size="sm">{val}</Badge>))
|
||||
: ((key, val) => getHeaderBadge(key, val));
|
||||
|
||||
// Now produce nested divs corresponding to the groups
|
||||
const remaining_vars = varnames.slice(1);
|
||||
const updated_eatenvars = eatenvars.concat([group_name]);
|
||||
const grouped_resps_divs = Object.keys(grouped_resps).map(g => groupByVars(grouped_resps[g], remaining_vars, updated_eatenvars, get_header(group_name, g)));
|
||||
const leftover_resps_divs = leftover_resps.length > 0 ? groupByVars(leftover_resps, remaining_vars, updated_eatenvars, get_header(group_name, undefined)) : [];
|
||||
|
||||
return (<>
|
||||
{header ?
|
||||
(<div key={group_name} className="response-group" style={{ backgroundColor: rgroup_color(eatenvars.length) }}>
|
||||
{header}
|
||||
<div className="response-boxes-wrapper">
|
||||
{grouped_resps_divs}
|
||||
</div>
|
||||
</div>)
|
||||
: <div>{grouped_resps_divs}</div>}
|
||||
{leftover_resps_divs.length === 0 ? (<></>) : (
|
||||
<div key={'__unspecified_group'} className="response-group">
|
||||
{leftover_resps_divs}
|
||||
</div>
|
||||
)}
|
||||
</>);
|
||||
};
|
||||
|
||||
// Produce DIV elements grouped by selected vars
|
||||
const divs = groupByVars(responses, selected_vars, [], null);
|
||||
setResponses(divs);
|
||||
|
||||
}, [multiSelectValue, multiSelectVars]);
|
||||
|
||||
const handleOnConnect = () => {
|
||||
// Get the ids from the connected input nodes:
|
||||
const input_node_ids = inputEdgesForNode(id).map(e => e.source);
|
||||
@ -62,11 +187,10 @@ const InspectorNode = ({ data, id }) => {
|
||||
}).then(function(json) {
|
||||
console.log(json);
|
||||
if (json.responses && json.responses.length > 0) {
|
||||
const responses = json.responses;
|
||||
|
||||
// Find all vars in response
|
||||
// Find all vars in responses
|
||||
let found_vars = new Set();
|
||||
responses.forEach(res_obj => {
|
||||
json.responses.forEach(res_obj => {
|
||||
Object.keys(res_obj.vars).forEach(v => {
|
||||
found_vars.add(v);
|
||||
});
|
||||
@ -86,108 +210,7 @@ const InspectorNode = ({ data, id }) => {
|
||||
selected_vars = ['LLM'];
|
||||
}
|
||||
|
||||
// Now we need to perform groupings by each var in the selected vars list,
|
||||
// nesting the groupings (preferrably with custom divs) and sorting within
|
||||
// each group by value of that group's var (so all same values are clumped together).
|
||||
// :: For instance, for varnames = ['LLM', '$var1', '$var2'] we should get back
|
||||
// :: nested divs first grouped by LLM (first level), then by var1, then var2 (deepest level).
|
||||
/**
|
||||
const groupByVars = (resps, varnames, eatenvars) => {
|
||||
if (resps.length === 0) return [];
|
||||
if (varnames.length === 0) {
|
||||
// Base case. Display n response(s) to each single prompt, back-to-back:
|
||||
return resps.map((res_obj, res_idx) => {
|
||||
// Spans for actual individual response texts
|
||||
const ps = res_obj.responses.map((r, idx) =>
|
||||
(<pre className="small-response" key={idx}>{r}</pre>)
|
||||
);
|
||||
|
||||
// At the deepest level, there may still be some vars left over. We want to display these
|
||||
// as tags, too, so we need to display only the ones that weren't 'eaten' during the recursive call:
|
||||
// (e.g., the vars that weren't part of the initial 'varnames' list that form the groupings)
|
||||
const vars = vars_to_str(res_obj.vars.filter(v => !eatenvars.includes(v)));
|
||||
const var_tags = vars.map((v) =>
|
||||
(<Badge key={v} color="blue" size="xs">{v}</Badge>)
|
||||
);
|
||||
return (
|
||||
<div key={"r"+res_idx} className="response-box" style={{ backgroundColor: colorForLLM(res_obj.llm) }}>
|
||||
{var_tags}
|
||||
{ps}
|
||||
</div>
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
// Bucket responses by the first var in the list, where
|
||||
// we also bucket any 'leftover' responses that didn't have the requested variable (a kind of 'soft fail')
|
||||
const group_name = varnames[0];
|
||||
const [grouped_resps, leftover_resps] = (group_name === 'LLM')
|
||||
? groupResponsesBy(resps, (r => r.llm))
|
||||
: groupResponsesBy(resps, (r => ((group_name in r.vars) ? r.vars[group_name] : null)));
|
||||
// Now produce nested divs corresponding to the groups
|
||||
const remaining_vars = varnames.slice(1);
|
||||
const updated_eatenvars = eatenvars.concat([group_name]);
|
||||
const grouped_resps_divs = grouped_resps.map(g => groupByVars(g, remaining_vars, updated_eatenvars));
|
||||
const leftover_resps_divs = leftover_resps.length > 0 ? groupByVars(leftover_resps, remaining_vars, updated_eatenvars) : [];
|
||||
|
||||
return (<>
|
||||
<div key={group_name} className="response-group">
|
||||
<h1>{group_name}</h1>
|
||||
{grouped_resps_divs}
|
||||
</div>
|
||||
{leftover_resps_divs.length === 0 ? (<></>) : (
|
||||
<div key={'__unspecified_group'} className="response-group">
|
||||
{leftover_resps_divs}
|
||||
</div>
|
||||
)}
|
||||
</>);
|
||||
};
|
||||
|
||||
// Produce DIV elements grouped by selected vars
|
||||
groupByVars(responses, selected_vars, []);
|
||||
**/
|
||||
|
||||
// Bucket responses by LLM:
|
||||
const responses_by_llm = groupResponsesBy(responses, (r => r.llm));
|
||||
|
||||
const colors = ['#ace1aeb1', '#f1b963b1', '#e46161b1', '#f8f398b1', '#defcf9b1', '#cadefcb1', '#c3bef0b1', '#cca8e9b1'];
|
||||
setResponses(Object.keys(responses_by_llm).map((llm, llm_idx) => {
|
||||
const res_divs = responses_by_llm[llm].map((res_obj, res_idx) => {
|
||||
const ps = res_obj.responses.map((r, idx) =>
|
||||
(<pre className="small-response" key={idx}>{r}</pre>)
|
||||
);
|
||||
const vars = vars_to_str(res_obj.vars);
|
||||
const var_tags = vars.map((v) => (
|
||||
<Badge key={v} color="blue" size="xs">{v}</Badge>
|
||||
));
|
||||
return (
|
||||
<div key={"r"+res_idx} className="response-box" style={{ backgroundColor: colors[llm_idx % colors.length] }}>
|
||||
{var_tags}
|
||||
{ps}
|
||||
</div>
|
||||
);
|
||||
});
|
||||
return (
|
||||
<div key={llm} className="llm-response-container">
|
||||
<h1>{llm}</h1>
|
||||
{res_divs}
|
||||
</div>
|
||||
);
|
||||
}));
|
||||
|
||||
// setVarSelects(Object.keys(tempvars).map(v => {
|
||||
// const options = Array.from(tempvars[v]).map((val, idx) => (
|
||||
// <option value={val} key={idx}>{val}</option>
|
||||
// ));
|
||||
// return (
|
||||
// <div key={v}>
|
||||
// <label htmlFor={v}>{v}: </label>
|
||||
// <select name={v} id={v} onChange={handleVarValueSelect}>
|
||||
// {options}
|
||||
// </select>
|
||||
// </div>
|
||||
// );
|
||||
// }));
|
||||
setJSONResponses(json.responses);
|
||||
}
|
||||
});
|
||||
}
|
||||
@ -206,20 +229,33 @@ const InspectorNode = ({ data, id }) => {
|
||||
setDataPropsForNode(id, { refresh: false });
|
||||
handleOnConnect();
|
||||
}
|
||||
}, [data, id, handleOnConnect, setDataPropsForNode]);
|
||||
}, [data, id, handleOnConnect, setDataPropsForNode]);
|
||||
|
||||
// When the user clicks an item in the drop-down,
|
||||
// we want to autoclose the multiselect drop-down:
|
||||
const multiSelectRef = useRef(null);
|
||||
const handleMultiSelectValueChange = (new_val) => {
|
||||
if (multiSelectRef) {
|
||||
multiSelectRef.current.blur();
|
||||
}
|
||||
setMultiSelectValue(new_val);
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="inspector-node cfnode">
|
||||
<NodeLabel title={data.title || 'Inspect Node'}
|
||||
nodeId={id}
|
||||
icon={'🔍'} />
|
||||
<MultiSelect onChange={setMultiSelectValue}
|
||||
<MultiSelect ref={multiSelectRef}
|
||||
onChange={handleMultiSelectValueChange}
|
||||
className='nodrag nowheel'
|
||||
label={<span style={{marginTop: '0px', fontWeight: 'normal'}}>Group responses by (order matters):</span>}
|
||||
data={multiSelectVars}
|
||||
placeholder="Pick vars to group responses, in order of importance"
|
||||
size="xs"
|
||||
value={multiSelectValue}
|
||||
searchable />
|
||||
clearSearchOnChange={true}
|
||||
clearSearchOnBlur={true} />
|
||||
<div className="inspect-response-container nowheel nodrag">
|
||||
{responses}
|
||||
</div>
|
||||
|
||||
@ -165,9 +165,9 @@
|
||||
border-radius: 5px;
|
||||
}
|
||||
.inspect-response-container {
|
||||
overflow-y: auto;
|
||||
overflow-y: scroll;
|
||||
min-width: 150px;
|
||||
max-width: 450px;
|
||||
max-width: 650px;
|
||||
max-height: 650px;
|
||||
resize: both;
|
||||
}
|
||||
@ -194,6 +194,42 @@
|
||||
padding-bottom: 0px;
|
||||
color: #222;
|
||||
}
|
||||
|
||||
.llm-group-header {
|
||||
font-weight: 400;
|
||||
font-size: 10pt;
|
||||
margin: 6px 8px 4px 8px;
|
||||
padding-top: 2px;
|
||||
padding-bottom: 0px;
|
||||
color: #222;
|
||||
}
|
||||
|
||||
.response-group {
|
||||
margin: 2px 0px 8px 0px;
|
||||
padding: 2px 2px 2px 2px;
|
||||
/* border-radius: 7px; */
|
||||
}
|
||||
.response-boxes-wrapper {
|
||||
margin-top: 4px;
|
||||
padding-left: 10px;
|
||||
border-left-width: 2px;
|
||||
border-left-style: solid;
|
||||
border-left-color: #bbb;
|
||||
}
|
||||
.response-item-llm-name-wrapper {
|
||||
padding-bottom: 0px;
|
||||
}
|
||||
.response-item-llm-name-wrapper h1 {
|
||||
font-size: 8pt;
|
||||
font-weight: bold;
|
||||
font-style: italic;
|
||||
color: #000;
|
||||
opacity: 0.7;
|
||||
text-align: right;
|
||||
padding-right: 4px;
|
||||
margin: 0px;
|
||||
}
|
||||
|
||||
.response-preview-container {
|
||||
margin: 10px -9px -9px -9px;
|
||||
max-height: 100px;
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user