ExternalVendorCode/ChainForge
1
0
Fork 0
mirror of https://github.com/ianarawjo/ChainForge.git synced 2025-03-14 00:06:33 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Add Example flows pane and other minor improvements (#59)

Browse Source 
* Example flows pane.

* Better Add+ Node UI.

* Added system message comparison example (see .

* Let user switch to gpt4 if initial model is gpt3.5
This commit is contained in:
ianarawjo 2023-06-03 14:32:37 -04:00 committed by GitHub
parent 7eb5aaa26d
commit c6da2314e7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 5519 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
MANIFEST.in
View File

@ -1,2 +1,3 @@
graft chainforge/react-server/build
graft chainforge/examples
include README.md

0
examples/basic-comparison.cforge → chainforge/examples/basic-comparison.cforge
View File

0
examples/chaining-prompts.cforge → chainforge/examples/chaining-prompts.cforge
View File

5259
chainforge/examples/comparing-system-msg.cforge Normal file
View File

File diff suppressed because it is too large Load Diff

0
examples/prompt-injection-test.cforge → chainforge/examples/prompt-injection-test.cforge
View File

43
chainforge/flask_app.py
View File

@ -18,8 +18,9 @@ app = Flask(__name__, static_folder=STATIC_DIR, template_folder=BUILD_DIR)
# Set up CORS for specific routes
cors = CORS(app, resources={r"/*": {"origins": "*"}})
# The cache base directory
# The cache and examples files base directories
CACHE_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'cache')
EXAMPLES_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'examples')
# Serve React app (static; no hot reloading)
@app.route("/")
@ -837,6 +838,46 @@ def importCache():
return ret
@app.route('/app/fetchExampleFlow', methods=['POST'])
def fetchExampleFlow():
"""
Fetches the example flow data, given its filename. The filename should be the
name of a file in the examples/ folder of the package.
Used for loading examples in the Example Flow modal.
POST'd data should be in form:
{
name: <str> # The filename (without .cforge extension)
}
"""
# Verify post'd data
data = request.get_json()
if 'name' not in data:
return jsonify({'error': 'Missing "name" parameter to fetchExampleFlow.'})
# Verify 'examples' directory exists:
if not os.path.isdir(EXAMPLES_DIR):
dirpath = os.path.dirname(os.path.realpath(__file__))
return jsonify({'error': f'Could not find an examples/ directory at path {dirpath}'})
# Check if the file is there:
filepath = os.path.join(EXAMPLES_DIR, data['name'] + '.cforge')
if not os.path.isfile(filepath):
return jsonify({'error': f"Could not find an example flow named {data['name']}"})
# Load the file and return its data:
try:
with open(filepath, 'r', encoding='utf-8') as f:
filedata = json.load(f)
except Exception as e:
return jsonify({'error': f"Error parsing example flow at {filepath}: {str(e)}"})
ret = jsonify({'data': filedata})
ret.headers.add('Access-Control-Allow-Origin', '*')
return ret
def run_server(host="", port=8000, cmd_args=None):
if cmd_args is not None and cmd_args.dummy_responses:
global PromptLLM

6
chainforge/react-server/build/asset-manifest.json
View File

@ -1,15 +1,15 @@
{
"files": {
"main.css": "/static/css/main.ce7de22c.css",
"main.js": "/static/js/main.1cce1776.js",
"main.js": "/static/js/main.ea9594fa.js",
"static/js/787.4c72bb55.chunk.js": "/static/js/787.4c72bb55.chunk.js",
"index.html": "/index.html",
"main.ce7de22c.css.map": "/static/css/main.ce7de22c.css.map",
"main.1cce1776.js.map": "/static/js/main.1cce1776.js.map",
"main.ea9594fa.js.map": "/static/js/main.ea9594fa.js.map",
"787.4c72bb55.chunk.js.map": "/static/js/787.4c72bb55.chunk.js.map"
},
"entrypoints": [
"static/css/main.ce7de22c.css",
"static/js/main.1cce1776.js"
"static/js/main.ea9594fa.js"
]
}

2
chainforge/react-server/build/index.html
View File

@ -1 +1 @@
<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="/favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#000000"/><meta name="description" content="Web site created using create-react-app"/><link rel="apple-touch-icon" href="/logo192.png"/><link rel="manifest" href="/manifest.json"/><title>Chainforge</title><script defer="defer" src="/static/js/main.1cce1776.js"></script><link href="/static/css/main.ce7de22c.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="/favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#000000"/><meta name="description" content="Web site created using create-react-app"/><link rel="apple-touch-icon" href="/logo192.png"/><link rel="manifest" href="/manifest.json"/><title>Chainforge</title><script defer="defer" src="/static/js/main.ea9594fa.js"></script><link href="/static/css/main.ce7de22c.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>

6
chainforge/react-server/build/static/js/main.1cce1776.js → chainforge/react-server/build/static/js/main.ea9594fa.js
View File

File diff suppressed because one or more lines are too long

0
chainforge/react-server/build/static/js/main.1cce1776.js.LICENSE.txt → chainforge/react-server/build/static/js/main.ea9594fa.js.LICENSE.txt
View File

2
chainforge/react-server/build/static/js/main.1cce1776.js.map → chainforge/react-server/build/static/js/main.ea9594fa.js.map
View File

File diff suppressed because one or more lines are too long

171
chainforge/react-server/src/App.js vendored
View File

@ -8,8 +8,8 @@ import ReactFlow, {
useReactFlow,
useViewport
} from 'react-flow-renderer';
import { Button } from '@mantine/core';
import { IconSettings } from '@tabler/icons-react';
import { Button, Menu } from '@mantine/core';
import { IconSettings, IconTextPlus, IconTerminal, IconCsv, IconSettingsAutomation } from '@tabler/icons-react';
import TextFieldsNode from './TextFieldsNode'; // Import a custom node
import PromptNode from './PromptNode';
import EvaluatorNode from './EvaluatorNode';
@ -19,6 +19,7 @@ import ScriptNode from './ScriptNode';
import AlertModal from './AlertModal';
import CsvNode from './CsvNode';
import GlobalSettingsModal from './GlobalSettingsModal';
import ExampleFlowsModal from './ExampleFlowsModal';
import './text-fields-node.css';
// State management (from https://reactflow.dev/docs/guides/state-management/)
@ -62,6 +63,9 @@ const App = () => {
// For modal popup to set global settings like API keys
const settingsModal = useRef(null);
// For modal popup of example flows
const examplesModal = useRef(null);
// For displaying error messages to user
const alertModal = useRef(null);
@ -106,11 +110,21 @@ const App = () => {
addNode({ id: 'csvNode-'+Date.now(), type: 'csv', data: {}, position: {x: x-200, y:y-100} });
};
const onClickExamples = () => {
if (examplesModal && examplesModal.current)
examplesModal.current.trigger();
};
const onClickSettings = () => {
if (settingsModal && settingsModal.current)
settingsModal.current.trigger();
};
const handleError = (err) => {
if (alertModal.current)
alertModal.current.trigger(err.message);
console.error(err.message);
};
/**
* SAVING / LOADING, IMPORT / EXPORT (from JSON)
*/
@ -186,32 +200,54 @@ const App = () => {
downloadJSON(flow_and_cache, `flow-${Date.now()}.cforge`);
});
}, [rfInstance, nodes]);
const importFlow = async (event) => {
// Create helper function for saving imported cache'd data to backend
const rejected = (err) => {
if (alertModal.current)
alertModal.current.trigger(err.message);
console.error(err.message);
};
const importCache = (cache_data) => {
return fetch(BASE_URL + 'app/importCache', {
method: 'POST',
headers: {'Content-Type': 'application/json', 'Access-Control-Allow-Origin': '*'},
body: JSON.stringify({
'files': cache_data,
}),
}, rejected).then(function(res) {
return res.json();
}, rejected).then(function(json) {
if (!json || json.result === undefined)
throw new Error('Request to import cache data was sent and received by backend server, but there was no response.');
else if (json.error || json.result === false)
throw new Error('Error importing cache data:' + json.error);
// Done!
}, rejected).catch(rejected);
};
// Import data to the cache stored on the local filesystem (in backend)
const importCache = (cache_data) => {
return fetch(BASE_URL + 'app/importCache', {
method: 'POST',
headers: {'Content-Type': 'application/json', 'Access-Control-Allow-Origin': '*'},
body: JSON.stringify({
'files': cache_data,
}),
}, handleError).then(function(res) {
return res.json();
}, handleError).then(function(json) {
if (!json || json.result === undefined)
throw new Error('Request to import cache data was sent and received by backend server, but there was no response.');
else if (json.error || json.result === false)
throw new Error('Error importing cache data:' + json.error);
// Done!
}, handleError).catch(handleError);
};
const importFlowFromJSON = (flowJSON) => {
// Detect if there's no cache data
if (!flowJSON.cache) {
// Support for loading old flows w/o cache data:
loadFlow(flowJSON);
return;
}
// Then we need to extract the JSON of the flow vs the cache data
const flow = flowJSON.flow;
const cache = flowJSON.cache;
// We need to send the cache data to the backend first,
// before we can load the flow itself...
importCache(cache).then(() => {
// We load the ReactFlow instance last
loadFlow(flow);
}).catch(err => {
// On an error, still try to load the flow itself:
handleError("Error encountered when importing cache data:" + err.message + "\n\nTrying to load flow regardless...");
loadFlow(flow);
});
};
// Import a ChainForge flow from a file
const importFlowFromFile = async () => {
// Create an input element with type "file" and accept only JSON files
const input = document.createElement("input");
input.type = "file";
@ -229,30 +265,11 @@ const App = () => {
// We try to parse the JSON response
const flow_and_cache = JSON.parse(reader.result);
// Detect if there's no cache data
if (!flow_and_cache.cache) {
// Support for loading old flows w/o cache data:
loadFlow(flow_and_cache);
return;
}
// Then we need to extract the JSON of the flow vs the cache data
const flow = flow_and_cache.flow;
const cache = flow_and_cache.cache;
// We need to send the cache data to the backend first,
// before we can load the flow itself...
importCache(cache).then(() => {
// We load the ReactFlow instance last
loadFlow(flow);
}).catch(err => {
// On an error, still try to load the flow itself:
rejected("Error encountered when importing cache data:" + err.message + "\n\nTrying to load flow regardless...");
loadFlow(flow);
});
// Import it to React Flow and import cache data on the backend
importFlowFromJSON(flow_and_cache);
} catch (error) {
console.error("Error parsing JSON file:", error);
handleError(error);
}
});
@ -264,10 +281,34 @@ const App = () => {
input.click();
};
// Load flow from examples modal
const onSelectExampleFlow = (filename) => {
// Fetch the example flow data from the backend
fetch(BASE_URL + 'app/fetchExampleFlow', {
method: 'POST',
headers: {'Content-Type': 'application/json', 'Access-Control-Allow-Origin': '*'},
body: JSON.stringify({
'name': filename,
}),
}, handleError).then(function(res) {
return res.json();
}, handleError).then(function(json) {
if (!json)
throw new Error('Request to fetch example flow was sent to backend server, but there was no response.');
else if (json.error || !json.data)
throw new Error('Error importing example flow:' + json.error);
// We have the data, import it:
importFlowFromJSON(json.data);
}, handleError).catch(handleError);
};
return (
<div>
<GlobalSettingsModal ref={settingsModal} />
<AlertModal ref={alertModal} />
<ExampleFlowsModal ref={examplesModal} onSelect={onSelectExampleFlow} />
<div style={{ height: '100vh', width: '100%', backgroundColor: '#eee' }}>
<ReactFlow
onNodesChange={onNodesChange}
@ -290,20 +331,30 @@ const App = () => {
</ReactFlow>
</div>
<div id="custom-controls" style={{position: 'fixed', left: '10px', top: '10px', zIndex:8}}>
<button onClick={addTextFieldsNode}>Add text fields node</button>
<button onClick={addPromptNode}>Add prompt node</button>
<button onClick={addEvalNode}>Add evaluator node</button>
<button onClick={addVisNode}>Add vis node</button>
<button onClick={addInspectNode}>Add inspect node</button>
<button onClick={addScriptNode}>Add script node</button>
<button onClick={addCsvNode}>Add csv node</button>
{/* <button onClick={saveFlow} style={{marginLeft: '12px'}}>Save</button>
<button onClick={loadFlowFromCache}>Load</button> */}
<button onClick={exportFlow} style={{marginLeft: '12px'}}>Export</button>
<button onClick={importFlow}>Import</button>
<Menu transitionProps={{ transition: 'pop-top-left' }}
position="top-start"
width={220}
closeOnClickOutside={true}
>
<Menu.Target>
<Button size="sm" variant="gradient" compact mr='sm'>Add Node +</Button>
</Menu.Target>
<Menu.Dropdown>
<Menu.Item onClick={addTextFieldsNode} icon={<IconTextPlus size="16px" />}> TextFields </Menu.Item>
<Menu.Item onClick={addPromptNode} icon={'💬'}> Prompt Node </Menu.Item>
<Menu.Item onClick={addEvalNode} icon={<IconTerminal size="16px" />}> Evaluator Node </Menu.Item>
<Menu.Item onClick={addVisNode} icon={'📊'}> Vis Node </Menu.Item>
<Menu.Item onClick={addInspectNode} icon={'🔍'}> Inspect Node </Menu.Item>
<Menu.Item onClick={addCsvNode} icon={<IconCsv size="16px" />}> CSV Node </Menu.Item>
<Menu.Item onClick={addScriptNode} icon={<IconSettingsAutomation size="16px" />}> Global Scripts </Menu.Item>
</Menu.Dropdown>
</Menu>
<Button onClick={exportFlow} size="sm" variant="outline" compact mr='xs'>Export</Button>
<Button onClick={importFlowFromFile} size="sm" variant="outline" compact>Import</Button>
</div>
<div style={{position: 'fixed', right: '10px', top: '10px', zIndex: 8}}>
<Button onClick={onClickSettings} size="sm" variant="gray" compact><IconSettings size={"100%"} /></Button>
<Button onClick={onClickExamples} size="sm" variant="outline" compact mr='xs' style={{float: 'left'}}> Example Flows </Button>
<Button onClick={onClickSettings} size="sm" variant="outline" compact><IconSettings size={"100%"} /></Button>
</div>
</div>
);

84
chainforge/react-server/src/ExampleFlowsModal.js vendored Normal file
View File

@ -0,0 +1,84 @@
import React, { useState, useEffect, useRef, useCallback, forwardRef, useImperativeHandle } from 'react';
import { SimpleGrid, Card, Modal, Image, Group, Text, Button, Badge } from '@mantine/core';
import { useDisclosure } from '@mantine/hooks';
import { IconChartDots3 } from '@tabler/icons-react';
/** Example flows to help users get started and see what CF can do */
const ExampleFlowCard = ({ title, description, buttonText, filename, onSelect }) => {
return (
<Card shadow="sm" padding="lg" radius="md" withBorder >
{/* <Card.Section>
<Image
src="..."
height={160}
alt="Alt text"
/>
</Card.Section> */}
<Text mb="xs" weight={500}>{title}</Text>
<Text size="sm" color="dimmed" lh={1.3}>
{description}
</Text>
<Button onClick={() => onSelect(filename)} variant="light" color="blue" fullWidth mt="md" radius="md">
{buttonText ? buttonText : 'Try me'}
</Button>
</Card>
);
};
const ExampleFlowsModal = forwardRef((props, ref) => {
// Mantine modal popover for alerts
const [opened, { open, close }] = useDisclosure(false);
// Callback for when an example flow is selected. Passed the name of the selected flow.
const onSelect = props.onSelect ? (
(filename) => {close(); props.onSelect(filename);}
) : undefined;
// This gives the parent access to triggering the modal alert
const trigger = () => {
open();
};
useImperativeHandle(ref, () => ({
trigger,
}));
return (
<Modal size='xl' opened={opened} onClose={close} title={<div><IconChartDots3 size={24} style={{position:'relative', marginRight: '8px', top: '4px'}} /><span style={{fontSize: '14pt'}}>Example Flows</span></div>} closeOnClickOutside={true} style={{position: 'relative', 'left': '-100px'}}>
<SimpleGrid cols={3} spacing='sm' verticalSpacing='sm'>
<ExampleFlowCard title="Compare length of responses across LLMs"
description="A simple evaluation with a prompt template, some inputs, and three models to prompt. Visualizes variability in response length."
filename="basic-comparison"
onSelect={onSelect}
/>
<ExampleFlowCard title="Robustness to prompt injection attacks"
description="Get a sense of different model's robustness against prompt injection attacks."
filename="prompt-injection-test"
onSelect={onSelect}
/>
<ExampleFlowCard title="Use an LLM as an evaluator"
description="Chain one prompt into another to extract entities from a text response. Plots number of entities."
filename="chaining-prompts"
onSelect={onSelect}
/>
<ExampleFlowCard title="Compare system messages"
description="Compares response quality across different ChatGPT system prompts. Visualizes how well it sticks to the instructions to only print Racket code."
filename="comparing-system-msg"
onSelect={onSelect}
/>
{/* <ExampleFlowCard title="Test mathematical ability"
description="Evaluate the ability of different LLMs to perform basic math and get the correct answer. Showcases chaining prompt templates and using prompt variables in Evaluate nodes."
/>
<ExampleFlowCard title="Does it conform to spec?"
description="Test how well a prompt and model conforms to a specification (instructed to format its output a certain way). Extracts and parses JSON outputs."
/> */}
</SimpleGrid>
</Modal>
);
});
export default ExampleFlowsModal;

22
chainforge/react-server/src/ModelSettingSchemas.js vendored
View File

@ -20,7 +20,7 @@ export const AvailableLLMs = [
];
const ChatGPTSettings = {
fullName: "GPT-3.5 (ChatGPT)",
fullName: "GPT-3.5+ (OpenAI)",
schema: {
"type": "object",
"required": [
@ -36,8 +36,8 @@ const ChatGPTSettings = {
"model": {
"type": "string",
"title": "Model Version",
"description": "Select a version of GPT3.5 to query. For more details on the differences, see the OpenAI API documentation. (Note that all ChainForge OpenAI calls use the Chat Completions API; we intend to support just Completions in the future.)",
"enum": ["gpt-3.5-turbo", "gpt-3.5-turbo-0301", "text-davinci-003", "text-davinci-002", "code-davinci-002"],
"description": "Select an OpenAI model to query. For more details on the differences, see the OpenAI API documentation.",
"enum": ["gpt-3.5-turbo", "gpt-3.5-turbo-0301", "gpt-4", "gpt-4-0314", "gpt-4-32k", "gpt-4-32k-0314", "text-davinci-003", "text-davinci-002", "code-davinci-002"],
"default": "gpt-3.5-turbo"
},
"system_msg": {
@ -158,7 +158,7 @@ const ChatGPTSettings = {
};
const GPT4Settings = {
fullName: "GPT-4",
fullName: ChatGPTSettings.fullName,
schema: {
"type": "object",
"required": [
@ -173,15 +173,17 @@ const GPT4Settings = {
"default": "GPT-4"
},
"model": {
"type": "string",
"title": "Model Version",
"description": "Select a version of GPT-4 to query. For more details on the differences, see the OpenAI API documentation. (Note that all ChainForge OpenAI calls use the Chat Completions API; we intend to support just Completions in the future.)",
"enum": ["gpt-4", "gpt-4-0314", "gpt-4-32k", "gpt-4-32k-0314"],
...ChatGPTSettings.schema.properties.model,
"default": "gpt-4"
},
}
}
},
uiSchema: ChatGPTSettings.uiSchema,
uiSchema: {
...ChatGPTSettings.uiSchema,
"model": {
"ui:help": "Defaults to gpt-4.",
},
},
postprocessors: ChatGPTSettings.postprocessors,
};

3
setup.py
View File

@ -6,7 +6,7 @@ def readme():
setup(
name='chainforge',
version='0.1.3',
version='0.1.3.1',
packages=find_packages(),
author="Ian Arawjo",
description="A Visual Programming Environment for Prompt Engineering",
@ -28,6 +28,7 @@ setup(
"anthropic",
"google-generativeai",
"dalaipy>=2.0.2",
"mistune>=2.0", # for LLM response markdown parsing
],
entry_points={
'console_scripts': [