Get Started with Evaluation

from langfuse import get_client, Evaluation
from langfuse.openai import OpenAI

langfuse = get_client()

def my_task(*, item, **kwargs):
    response = OpenAI().chat.completions.create(
        model="gpt-4.1",
        messages=[{"role": "user", "content": item["input"]}],
    )
    return response.choices[0].message.content

def check_answer(*, output, expected_output, **kwargs):
    is_correct = expected_output.lower() in output.lower()
    return Evaluation(name="correctness", value=1.0 if is_correct else 0.0)

result = langfuse.run_experiment(
    name="My First Experiment",
    data=[
        {"input": "What is the capital of France?", "expected_output": "Paris"},
        {"input": "What is the capital of Germany?", "expected_output": "Berlin"},
    ],
    task=my_task,
    evaluators=[check_answer],
)

print(result.format())

import { OpenAI } from "openai";
import { NodeSDK } from "@opentelemetry/sdk-node";
import { LangfuseClient, ExperimentItem } from "@langfuse/client";
import { observeOpenAI } from "@langfuse/openai";
import { LangfuseSpanProcessor } from "@langfuse/otel";

const otelSdk = new NodeSDK({ spanProcessors: [new LangfuseSpanProcessor()] });
otelSdk.start();

const langfuse = new LangfuseClient();

const testData: ExperimentItem[] = [
  { input: "What is the capital of France?", expectedOutput: "Paris" },
  { input: "What is the capital of Germany?", expectedOutput: "Berlin" },
];

const myTask = async (item: ExperimentItem) => {
  const response = await observeOpenAI(new OpenAI()).chat.completions.create({
    model: "gpt-4.1",
    messages: [{ role: "user", content: item.input as string }],
  });
  return response.choices[0].message.content;
};

const checkAnswer = async ({ output, expectedOutput }) => ({
  name: "correctness",
  value: expectedOutput && output.toLowerCase().includes(expectedOutput.toLowerCase()) ? 1.0 : 0.0,
});

const result = await langfuse.experiment.run({
  name: "My First Experiment",
  data: testData,
  task: myTask,
  evaluators: [checkAnswer],
});

console.log(await result.format());
await otelSdk.shutdown();