Simple Combining
In this example, we will create a single function that wraps 2 executor functions. The function will be provided a block of code, and will output some Jest tests. Instead of just saying 'write me jest tests', we will have the LLM plan a bit in advance.
This can be useful as:
- Planning any ideas
- Iterating on plans
This is really just a simple loop that combines 2 functions, but using LLM executors keeps the code simple.
Step 1 - Create LLM Executor to make list of test cases given a function
ts
const PROMPT = `You are a senior typescript developer. I need you make
a concise list of test cases that need to be written for the function below.
We need to write an extensive test suite that covers all paths and edge-cases.
Do not write any tests yet, we need to plan first.
Here is the function:
{{>MarkdownCode language='typescript' code=functionToTest}}`;
const INSTRUCTION = `Don't explain yourself or ask questions. Think step
by step through how the function works, then respond with a plain-text
list of tests cases we need to write.
Do not list more than {{numberOfTestCases}} test cases.
For Example:
- <explain a test case we need to write>
- <explain a test case we need to write>`;
export function llmExecutorThatMakesAListOfTestCases<T extends {
functionToTest: string;
numberOfTestCases: number;
}>(llm: BaseLlm, input: T) {
return createLlmExecutor({
prompt: createChatPrompt<T>(PROMPT).addUserMessage(INSTRUCTION),
parser: createParser("listToArray"),
llm,
}).execute(input)
}Step 2 - Create LLM Executor to write single test case
ts
const PROMPT2 = `You are a senior typescript developer. You need to write
a single Jest test for the function below.
Here is the function the test needs to be written for:
{{>MarkdownCode language='typescript' code=functionToTest}}`;
const INSTRUCTION2 = `Here is the test you need to write:
{{testRequirement}}
Reply only with the test case. Do not explain yourself or ask questions.
For example:
\`\`\`typescript
it('<a description of what the test covers>', async () => {
// Your test code here
});
\`\`\``;
export function llmExecutorThatWritesTests<T extends {
testRequirement: string;
functionToTest: string;
}>(llm: BaseLlm, input: T) {
const prompt = createChatPrompt<T>(PROMPT2, { allowUnsafeUserTemplate: true })
.addUserMessage(INSTRUCTION2);
const parser = createParser("markdownCodeBlock");
return createLlmExecutor({
prompt,
parser,
llm,
}).execute(input)
}Step 3 - Combine into single method
Combine the prompt, LLM, and parser into a single function.
ts
export async function generateTestSuite(functionToTest: string) {
const tests = [];
const requirements = await llmExecutorThatMakesAListOfTestCases(
llm, {
numberOfTestCases: 5,
functionToTest,
});
for (const testRequirement of requirements) {
const test = await llmExecutorThatWritesTests(llm, {
functionToTest,
testRequirement,
});
if (test?.code) {
tests.push(test.code);
}
}
return { requirements, tests};
}Step 4 - Use it!
ts
import { generateTestSuite } from "./somewhere"
// the input you get from somewhere
const functionToTest = `function add(a: number, b: number){
return a + b;
}`;
const response = await generateTestSuite({
functionToTest
});
/**
*
* console.log(response)
* {
"requirements": [
"Test case 1: Test adding two positive numbers",
"Test case 2: Test adding two negative numbers",
"Test case 3: Test adding a positive number and a negative number",
"Test case 4: Test adding zero to a positive number",
"Test case 5: Test adding zero to a negative number",
"Test case 6: Test adding zero to zero",
"Test case 7: Test adding a positive number to the maximum safe integer value",
"Test case 8: Test adding a negative number to the minimum safe integer value",
"Test case 9: Test adding a positive number to Infinity",
"Test case 10: Test adding a negative number to -Infinity",
"Test case 11: Test adding a positive number to NaN",
"Test case 12: Test adding a negative number to NaN",
"Test case 13: Test adding a positive number to a string",
"Test case 14: Test adding a negative number to a string",
"Test case 15: Test adding a positive number to an empty string",
"Test case 16: Test adding a negative number to an empty string",
"Test case 17: Test adding a positive number to null",
"Test case 18: Test adding a negative number to null",
"Test case 19: Test adding a positive number to undefined",
"Test case 20: Test adding a negative number to undefined"
],
"tests": [
"it('should add two positive numbers', () => {\n // Arrange\n const a = 2;\n const b = 3;\n\n // Act\n const result = add(a, b);\n\n // Assert\n expect(result).toBe(5);\n});",
"it('Test adding two negative numbers', () => {\n expect(add(-5, -10)).toBe(-15);\n});",
"it('should add a positive number and a negative number', () => {\n const result = add(5, -3);\n expect(result).toBe(2);\n});",
"it('should add zero to a positive number', () => {\n // Arrange\n const a = 5;\n const b = 0;\n\n // Act\n const result = add(a, b);\n\n // Assert\n expect(result).toBe(5);\n});",
"it('should add zero to a negative number', () => {\n const result = add(-5, 0);\n expect(result).toBe(-5);\n});",
"it('Test adding zero to zero', () => {\n expect(add(0, 0)).toBe(0);\n});",
"it('Test adding a positive number to the maximum safe integer value', () => {\n const result = add(Number.MAX_SAFE_INTEGER, 5);\n expect(result).toBe(Number.MAX_SAFE_INTEGER + 5);\n});",
"it('Test adding a negative number to the minimum safe integer value', () => {\n const result = add(Number.MIN_SAFE_INTEGER, -5);\n expect(result).toBe(Number.MIN_SAFE_INTEGER - 5);\n});",
"it('should add a positive number to Infinity', () => {\n expect(add(5, Infinity)).toBe(Infinity);\n});",
"it('Test adding a negative number to -Infinity', () => {\n expect(add(-5, -Infinity)).toBe(-Infinity);\n});",
"it('Test adding a positive number to NaN', () => {\n const result = add(5, NaN);\n expect(result).toBeNaN();\n});",
"it('Test adding a negative number to NaN', () => {\n const result = add(-5, NaN);\n expect(result).toBeNaN();\n});",
"it('should add a positive number to a string', () => {\n const result = add(5, '10');\n expect(result).toBe('510');\n});",
"it('should add a negative number to a string', () => {\n const result = add(-5, '10');\n expect(result).toBe('5-10');\n});",
"it('should add a positive number to an empty string', () => {\n const result = add(5, '');\n expect(result).toBe('5');\n});",
"it('should add a negative number to an empty string', () => {\n const result = add(-5, 0);\n expect(result).toBe(-5);\n});",
"it('should add a positive number to null', () => {\n const result = add(5, null);\n expect(result).toBe(5);\n});",
"it('should add a negative number to null', () => {\n const result = add(-5, null);\n expect(result).toBe(-5);\n});",
"it('Test adding a positive number to undefined', () => {\n expect(add(5, undefined)).toBeNaN();\n});",
"it('Test adding a negative number to undefined', () => {\n expect(add(-5, undefined)).toBeNaN();\n});"
]
}
**
*/Complete File
ts
import { createLlmExecutor } from "@/executor";
import { BaseLlm } from "@/types";
import { createParser } from "@/parser";
import { createChatPrompt } from "@/prompt";
import { useLlm } from "@/llm";
const llm = useLlm("openai.chat-mock.v1", { model: "something" });
// #region StepOne
const PROMPT = `You are a senior typescript developer. I need you make
a concise list of test cases that need to be written for the function below.
We need to write an extensive test suite that covers all paths and edge-cases.
Do not write any tests yet, we need to plan first.
Here is the function:
{{>MarkdownCode language='typescript' code=functionToTest}}`;
const INSTRUCTION = `Don't explain yourself or ask questions. Think step
by step through how the function works, then respond with a plain-text
list of tests cases we need to write.
Do not list more than {{numberOfTestCases}} test cases.
For Example:
- <explain a test case we need to write>
- <explain a test case we need to write>`;
export function llmExecutorThatMakesAListOfTestCases<T extends {
functionToTest: string;
numberOfTestCases: number;
}>(llm: BaseLlm, input: T) {
return createLlmExecutor({
prompt: createChatPrompt<T>(PROMPT).addUserMessage(INSTRUCTION),
parser: createParser("listToArray"),
llm,
}).execute(input)
}
// #endregion StepOne
// #region StepTwo
const PROMPT2 = `You are a senior typescript developer. You need to write
a single Jest test for the function below.
Here is the function the test needs to be written for:
{{>MarkdownCode language='typescript' code=functionToTest}}`;
const INSTRUCTION2 = `Here is the test you need to write:
{{testRequirement}}
Reply only with the test case. Do not explain yourself or ask questions.
For example:
\`\`\`typescript
it('<a description of what the test covers>', async () => {
// Your test code here
});
\`\`\``;
export function llmExecutorThatWritesTests<T extends {
testRequirement: string;
functionToTest: string;
}>(llm: BaseLlm, input: T) {
const prompt = createChatPrompt<T>(PROMPT2, { allowUnsafeUserTemplate: true })
.addUserMessage(INSTRUCTION2);
const parser = createParser("markdownCodeBlock");
return createLlmExecutor({
prompt,
parser,
llm,
}).execute(input)
}
// #endregion StepTwo
// #region StepThree
export async function generateTestSuite(functionToTest: string) {
const tests = [];
const requirements = await llmExecutorThatMakesAListOfTestCases(
llm, {
numberOfTestCases: 5,
functionToTest,
});
for (const testRequirement of requirements) {
const test = await llmExecutorThatWritesTests(llm, {
functionToTest,
testRequirement,
});
if (test?.code) {
tests.push(test.code);
}
}
return { requirements, tests};
}
// #endregion StepThree
(async () => {
const functionToTest = `function add(a: number, b: number){
return a + b;
}`;
const results = await generateTestSuite(functionToTest);
console.log(results);
})();