mirror of
https://github.com/ianarawjo/ChainForge.git
synced 2025-03-14 08:16:37 +00:00
Add # special fill variables to prompt templating.
This commit is contained in:
Ian Arawjo
parent
ece74e696e
commit
4730009a1c
@ -1,15 +1,15 @@
|
||||
{
|
||||
"files": {
|
||||
"main.css": "/static/css/main.a4e8271c.css",
|
||||
"main.js": "/static/js/main.b59fa9d7.js",
|
||||
"main.js": "/static/js/main.05c73878.js",
|
||||
"static/js/787.4c72bb55.chunk.js": "/static/js/787.4c72bb55.chunk.js",
|
||||
"index.html": "/index.html",
|
||||
"main.a4e8271c.css.map": "/static/css/main.a4e8271c.css.map",
|
||||
"main.b59fa9d7.js.map": "/static/js/main.b59fa9d7.js.map",
|
||||
"main.05c73878.js.map": "/static/js/main.05c73878.js.map",
|
||||
"787.4c72bb55.chunk.js.map": "/static/js/787.4c72bb55.chunk.js.map"
|
||||
},
|
||||
"entrypoints": [
|
||||
"static/css/main.a4e8271c.css",
|
||||
"static/js/main.b59fa9d7.js"
|
||||
"static/js/main.05c73878.js"
|
||||
]
|
||||
}
|
||||
@ -1 +1 @@
|
||||
<!doctype html><html lang="en"><head><meta charset="utf-8"/><script async src="https://www.googletagmanager.com/gtag/js?id=G-RN3FDBLMCR"></script><script>function gtag(){dataLayer.push(arguments)}window.dataLayer=window.dataLayer||[],gtag("js",new Date),gtag("config","G-RN3FDBLMCR")</script><link rel="icon" href="/favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#000000"/><meta name="description" content="A visual programming environment for prompt engineering"/><link rel="apple-touch-icon" href="/logo192.png"/><link rel="manifest" href="/manifest.json"/><title>ChainForge</title><script defer="defer" src="/static/js/main.b59fa9d7.js"></script><link href="/static/css/main.a4e8271c.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
|
||||
<!doctype html><html lang="en"><head><meta charset="utf-8"/><script async src="https://www.googletagmanager.com/gtag/js?id=G-RN3FDBLMCR"></script><script>function gtag(){dataLayer.push(arguments)}window.dataLayer=window.dataLayer||[],gtag("js",new Date),gtag("config","G-RN3FDBLMCR")</script><link rel="icon" href="/favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#000000"/><meta name="description" content="A visual programming environment for prompt engineering"/><link rel="apple-touch-icon" href="/logo192.png"/><link rel="manifest" href="/manifest.json"/><title>ChainForge</title><script defer="defer" src="/static/js/main.05c73878.js"></script><link href="/static/css/main.a4e8271c.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
2
chainforge/react-server/src/EvaluatorNode.js
vendored
2
chainforge/react-server/src/EvaluatorNode.js
vendored
@ -239,7 +239,7 @@ const EvaluatorNode = ({ data, id }) => {
|
||||
|
||||
const showResponseInspector = useCallback(() => {
|
||||
if (inspectModal && inspectModal.current && lastResponses)
|
||||
inspectModal.current.trigger();
|
||||
inspectModal.current.trigger();
|
||||
}, [inspectModal, lastResponses]);
|
||||
|
||||
const default_header = (progLang === 'python') ?
|
||||
|
||||
2
chainforge/react-server/src/LLMEvalNode.js
vendored
2
chainforge/react-server/src/LLMEvalNode.js
vendored
@ -55,7 +55,7 @@ const LLMEvaluatorNode = ({ data, id }) => {
|
||||
const handleError = (err) => {
|
||||
setStatus('error');
|
||||
setProgress(undefined);
|
||||
alertModal.current.trigger(err);
|
||||
alertModal.current.trigger(err?.error || err);
|
||||
};
|
||||
|
||||
// Fetch info about the number of queries we'll need to make
|
||||
|
||||
@ -29,7 +29,7 @@ const defaultColumns = [
|
||||
},
|
||||
{
|
||||
key: 'answer',
|
||||
header: 'Expected Answer',
|
||||
header: 'Expected',
|
||||
},
|
||||
];
|
||||
|
||||
|
||||
@ -28,6 +28,10 @@ export const extractBracketedSubstrings = (text) => {
|
||||
}
|
||||
prev_c = c;
|
||||
}
|
||||
|
||||
// Ignore any varnames that begin with the special character #:
|
||||
capture_groups = capture_groups.filter(s => (s.length === 0 || s[0] !== '#'))
|
||||
|
||||
return capture_groups;
|
||||
};
|
||||
|
||||
|
||||
@ -916,7 +916,9 @@ export async function evalWithLLM(id: string,
|
||||
|
||||
// We need to keep track of the index of each response in the response object.
|
||||
// We can generate var dicts with metadata to store the indices:
|
||||
let inputs = resp_objs.map((obj, i) => obj.responses.map((r: string, j: number) => ({text: r, fill_history: {}, metavars: { i, j }}))).flat();
|
||||
let inputs = resp_objs.map((obj, __i) => obj.responses.map(
|
||||
(r: string, __j: number) => ({text: r, fill_history: obj.vars, metavars: { ...obj.metavars, __i, __j }})
|
||||
)).flat();
|
||||
|
||||
// Now run all inputs through the LLM grader!:
|
||||
const {responses, errors} = await queryLLM(`eval-${id}-${cache_id}`, [llm], 1, root_prompt, { input: inputs }, undefined, undefined, undefined, progress_listener);
|
||||
@ -928,15 +930,15 @@ export async function evalWithLLM(id: string,
|
||||
// Now we need to apply each response as an eval_res (a score) back to each response object,
|
||||
// using the aforementioned mapping metadata:
|
||||
responses.forEach((r: StandardizedLLMResponse) => {
|
||||
let resp_obj = resp_objs[r.metavars.i];
|
||||
let resp_obj = resp_objs[r.metavars.__i];
|
||||
if (resp_obj.eval_res !== undefined)
|
||||
resp_obj.eval_res.items[r.metavars.j] = r.responses[0];
|
||||
resp_obj.eval_res.items[r.metavars.__j] = r.responses[0];
|
||||
else {
|
||||
resp_obj.eval_res = {
|
||||
items: [],
|
||||
dtype: 'Categorical',
|
||||
};
|
||||
resp_obj.eval_res.items[r.metavars.j] = r.responses[0];
|
||||
resp_obj.eval_res.items[r.metavars.__j] = r.responses[0];
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
@ -115,6 +115,33 @@ export class StringTemplate {
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Finds all unfilled variables in the template string.
|
||||
*
|
||||
* For instance, if the string is "The {place} had {food}",
|
||||
* then ["place", "food"] will be returned.
|
||||
*/
|
||||
get_vars(): Array<string> {
|
||||
let template = this.val;
|
||||
let varnames: Array<string> = [];
|
||||
let prev_c = '';
|
||||
let group_start_idx = -1;
|
||||
for (let i = 0; i < template.length; i += 1) {
|
||||
const c = template.charAt(i);
|
||||
if (prev_c !== '\\') { // Skip escaped braces
|
||||
if (group_start_idx === -1 && c === '{') // Identify the start of a capture {group}
|
||||
group_start_idx = i;
|
||||
else if (group_start_idx > -1 && c === '}') { // Identify the end of a capture {group}
|
||||
if (group_start_idx + 1 < i) // Ignore {} empty braces
|
||||
varnames.push(template.substring(group_start_idx+1, i));
|
||||
group_start_idx = -1;
|
||||
}
|
||||
}
|
||||
prev_c = c;
|
||||
}
|
||||
return varnames;
|
||||
}
|
||||
|
||||
toString(): string {
|
||||
return this.val;
|
||||
}
|
||||
@ -352,8 +379,29 @@ export class PromptPermutationGenerator {
|
||||
return true; // done
|
||||
}
|
||||
|
||||
for (let p of this._gen_perm(template, Object.keys(paramDict), paramDict))
|
||||
for (let p of this._gen_perm(template, Object.keys(paramDict), paramDict)) {
|
||||
// Special variables {#...} denotes filling a variable from a matching var in fill_history or metavars.
|
||||
// Find any special variables:
|
||||
const unfilled_vars = (new StringTemplate(p.template)).get_vars();
|
||||
let special_vars_to_fill: {[key: string]: string} = {};
|
||||
for (const v of unfilled_vars) {
|
||||
if (v.length > 0 && v[0] === '#') { // special template variables must begin with #
|
||||
const svar = v.substring(1);
|
||||
if (svar in p.fill_history)
|
||||
special_vars_to_fill[v] = p.fill_history[svar];
|
||||
else if (svar in p.metavars)
|
||||
special_vars_to_fill[v] = p.metavars[svar];
|
||||
else
|
||||
console.warn(`Could not find a value to fill special var ${v} in prompt template.`);
|
||||
}
|
||||
}
|
||||
// Fill any special variables, using the fill history of the template in question:
|
||||
if (Object.keys(special_vars_to_fill).length > 0)
|
||||
p.template = new StringTemplate(p.template).safe_substitute(special_vars_to_fill);
|
||||
|
||||
// Yield the final prompt template
|
||||
yield p;
|
||||
}
|
||||
return true; // done
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user