Validator

In this example, we will create a function that is able to validate whether or not a series of statements is true.

This can be useful as:

• Moderation layer
• Policy checker

This takes advantage of a custom output parser to not only ensure formatting, but slightly transform the output.

Step 1 - Prepare Prompt

const PROMPT = `You need to work through the list below step-by-step and
identify if each statement is true or false. Use the conversation and context
below to help make a decision. Do not explain your answer, and do not use 
punctuation.
  
{{#if statements}}
The List:
{{#each statements as | statements |}}
- {{ statements }}
{{/each}}
{{/if}}

Conversation:
{{>DialogueHistory key='chatHistory'}}

Most Recent Message: {{ input }}`;

const INSTRUCTION = `Your response should be valid JSON in the following format: ${JSON.stringify(
  [
    {
      statement: "<a statement from the list>",
      answer: "<true or false>",
      confidence: "<number between 1 and 100. how confident are you>",
    },
  ]
)}`;

Step 2 - Create Custom Output Parser

While we tell the LLM to format its response as JSON, it's a string. We still need to parse and validate the response so it can be used in our code. A custom output parser will reformat the output from the LLM, and provide typings for our LLM executor.

export const VerifyParser = createCustomParser(
  "VerifyParser",
  (input, _context) => {
    const schema = defineSchema({
      type: "array",
      items: {
        type: "object",
        properties: {
          statement: { type: "string", default: "" },
          answer: { type: "string", default: "" },
          confidence: { type: "integer", default: 0 },
        },
        required: ["statement", "answer", "confidence"],
        additionalProperties: false,
      },
    });

    const parseJson = createParser("json", { schema }).parse(input);

    const didAllPass =
      parseJson.filter((a: any) => a.answer === "true").length ===
      parseJson.length;

    return {
      passed: didAllPass,
      results: parseJson,
    };
  }
);

Step 3 - Create LLM Executor

Combine the prompt, LLM, and parser into a single function.

export async function checkPolicy(llm: BaseLlm) {
  const prompt = createChatPrompt(PROMPT).addSystemMessage(INSTRUCTION);
  return createLlmExecutor({
    llm,
    prompt,
    parser: VerifyParser
  });
}

Step 4 - Use it!

import { checkPolicy } from "./somewhere"

// the input you get from somewhere
const input = "";

// a chat history, loaded from somewhere
const chatHistory = [{ role: "user", content: "Hi I'm Greg" }];

// a list of statements we'd like to check
const statements = [
    "The user has told us their age.",
     "The user has told us their name."
]

const response = await checkPolicy().execute({
    mostRecentMessage: input,
    chatHistory,
    statements
});

/**
 * 
 * console.log(response)
 * {
 *   "passed": false,
 *   "results": [{
 *     "statement": "The user has told us their age.",
 *     "answer":"false",
 *     "confidence":"95"
 *    },{
 *      "statement": "The user has told us their name.",
 *      "answer": "true",
 *      "confidence":"80"
 *    }]
 * }
 ** 

Complete File

import { createLlmExecutor } from "@/executor";
import { BaseLlm } from "@/llm";
import { createCustomParser, createParser } from "@/parser";
import { createChatPrompt } from "@/prompt";
import { defineSchema } from "@/utils";

const PROMPT = `You need to work through the list below step-by-step and
identify if each statement is true or false. Use the conversation and context
below to help make a decision. Do not explain your answer, and do not use 
punctuation.
  
{{#if statements}}
The List:
{{#each statements as | statements |}}
- {{ statements }}
{{/each}}
{{/if}}

Conversation:
{{>DialogueHistory key='chatHistory'}}

Most Recent Message: {{ input }}`;

const INSTRUCTION = `Your response should be valid JSON in the following format: ${JSON.stringify(
  [
    {
      statement: "<a statement from the list>",
      answer: "<true or false>",
      confidence: "<number between 1 and 100. how confident are you>",
    },
  ]
)}`;

export const VerifyParser = createCustomParser(
  "VerifyParser",
  (input, _context) => {
    const schema = defineSchema({
      type: "array",
      items: {
        type: "object",
        properties: {
          statement: { type: "string", default: "" },
          answer: { type: "string", default: "" },
          confidence: { type: "integer", default: 0 },
        },
        required: ["statement", "answer", "confidence"],
        additionalProperties: false,
      },
    });

    const parseJson = createParser("json", { schema }).parse(input);

    const didAllPass =
      parseJson.filter((a: any) => a.answer === "true").length ===
      parseJson.length;

    return {
      passed: didAllPass,
      results: parseJson,
    };
  }
);

export async function checkPolicy(llm: BaseLlm) {
  const prompt = createChatPrompt(PROMPT).addSystemMessage(INSTRUCTION);
  return createLlmExecutor({
    llm,
    prompt,
    parser: VerifyParser
  });
}