Validator
In this example, we will create a function that is able to validate whether or not a series of statements is true.
This can be useful as:
- Moderation layer
- Policy checker
This takes advantage of a custom output parser to not only ensure formatting, but slightly transform the output.
Step 1 - Prepare Prompt
ts
const PROMPT = `You need to work through the list below step-by-step and
identify if each statement is true or false. Use the conversation and context
below to help make a decision. Do not explain your answer, and do not use
punctuation.
{{#if statements}}
The List:
{{#each statements as | statements |}}
- {{ statements }}
{{/each}}
{{/if}}
Conversation:
{{>DialogueHistory key='chatHistory'}}
Most Recent Message: {{ input }}`;
const INSTRUCTION = `Your response should be valid JSON in the following format: ${JSON.stringify(
[
{
statement: "<a statement from the list>",
answer: "<true or false>",
confidence: "<number between 1 and 100. how confident are you>",
},
]
)}`;
Step 2 - Create Custom Output Parser
While we tell the LLM to format its response as JSON, it's a string. We still need to parse and validate the response so it can be used in our code. A custom output parser will reformat the output from the LLM, and provide typings for our LLM executor.
ts
export const VerifyParser = createCustomParser(
"VerifyParser",
(input, _context) => {
const schema = defineSchema({
type: "array",
items: {
type: "object",
properties: {
statement: { type: "string", default: "" },
answer: { type: "string", default: "" },
confidence: { type: "integer", default: 0 },
},
required: ["statement", "answer", "confidence"],
additionalProperties: false,
},
});
const parseJson = createParser("json", { schema }).parse(input);
const didAllPass =
parseJson.filter((a: any) => a.answer === "true").length ===
parseJson.length;
return {
passed: didAllPass,
results: parseJson,
};
}
);
Step 3 - Create LLM Executor
Combine the prompt, LLM, and parser into a single function.
ts
export async function checkPolicy(
llm: BaseLlm,
input: {
statements: string[]; // the list of statements to check
chatHistory: IChatMessages; // the chat history
input: string; // the most recent user message
}
) {
const prompt = createChatPrompt(PROMPT).addSystemMessage(INSTRUCTION);
return createLlmExecutor({
llm,
prompt,
parser: VerifyParser,
}).execute(input);
}
Step 4 - Use it!
ts
import { checkPolicy } from "./somewhere"
// the input you get from somewhere
const input = "";
// a chat history, loaded from somewhere
const chatHistory = [{ role: "user", content: "Hi I'm Greg" }];
// a list of statements we'd like to check
const statements = [
"The user has told us their age.",
"The user has told us their name."
]
const response = await checkPolicy().execute({
mostRecentMessage: input,
chatHistory,
statements
});
/**
*
* console.log(response)
* {
* "passed": false,
* "results": [{
* "statement": "The user has told us their age.",
* "answer":"false",
* "confidence":"95"
* },{
* "statement": "The user has told us their name.",
* "answer": "true",
* "confidence":"80"
* }]
* }
**
Complete File
ts
import {
createChatPrompt,
createParser,
createLlmExecutor,
createCustomParser,
} from "llm-exe";
import type { BaseLlm, IChatMessages } from "llm-exe";
import { defineSchema } from "@/utils";
const PROMPT = `You need to work through the list below step-by-step and
identify if each statement is true or false. Use the conversation and context
below to help make a decision. Do not explain your answer, and do not use
punctuation.
{{#if statements}}
The List:
{{#each statements as | statements |}}
- {{ statements }}
{{/each}}
{{/if}}
Conversation:
{{>DialogueHistory key='chatHistory'}}
Most Recent Message: {{ input }}`;
const INSTRUCTION = `Your response should be valid JSON in the following format: ${JSON.stringify(
[
{
statement: "<a statement from the list>",
answer: "<true or false>",
confidence: "<number between 1 and 100. how confident are you>",
},
]
)}`;
export const VerifyParser = createCustomParser(
"VerifyParser",
(input, _context) => {
const schema = defineSchema({
type: "array",
items: {
type: "object",
properties: {
statement: { type: "string", default: "" },
answer: { type: "string", default: "" },
confidence: { type: "integer", default: 0 },
},
required: ["statement", "answer", "confidence"],
additionalProperties: false,
},
});
const parseJson = createParser("json", { schema }).parse(input);
const didAllPass =
parseJson.filter((a: any) => a.answer === "true").length ===
parseJson.length;
return {
passed: didAllPass,
results: parseJson,
};
}
);
export async function checkPolicy(
llm: BaseLlm,
input: {
statements: string[]; // the list of statements to check
chatHistory: IChatMessages; // the chat history
input: string; // the most recent user message
}
) {
const prompt = createChatPrompt(PROMPT).addSystemMessage(INSTRUCTION);
return createLlmExecutor({
llm,
prompt,
parser: VerifyParser,
}).execute(input);
}