RAG

Before you begin

Make sure you have completed the Evaluations Quickstart guide. You should have your evals script ready and have the first experiment completed.

RAG - Node.js

We will be using ChromaDB as a semantic search engine for this tutorial, but there is no restriction on which search engine you can use with Hamming.

Learn how to debug your RAG application using the Hamming TypeScript SDK.

Install ChromaDB locally

npm install --save chromadb chromadb-default-embed

Install Chroma via pypi to easily run the backend server:

pip install chromadb

Run the Chroma server:

chroma run --path ./getting-started

Populate ChromaDB with a set of documents

Make sure to replace the placeholders with your actual keys and dataset ID created in the previous step.

setup-chroma.js

const { ChromaClient, OpenAIEmbeddingFunction } = require("chromadb");

const client = new ChromaClient();
const embedder = new OpenAIEmbeddingFunction({
    openai_api_key: "<your-openai-key>",
});

async function setupChroma() {
    const collection = await client.getOrCreateCollection({
        name: "travel_collection",
        embeddingFunction: embedder,
    });
    await collection.upsert({
        documents: [
            "Booking Confirmations: You are required to send a booking confirmation for each reservation. When reserving outside the XYZ Portal, please ensure to email the confirmation to [email protected].",
            "EU Travel Restrictions: Travelers from other countries are not allowed to enter Austria unless they are residents or meet certain exemptions.",
            "Traveling with Pets: Although pets used to be allowed on international flights, due to recent changes in regulations, you cannot bring your pet on an international flight.",
            "Travel Insurance: It is recommended that you purchase travel insurance to protect yourself from unexpected events such as trip cancellations or medical emergencies.",
            "Traveling with Medication: If you are traveling with medication, make sure to carry a copy of your prescription and keep the medication in its original packaging.",
        ],
        ids: ["id1", "id2", "id3", "id4", "id5"],
    });
    return collection;
}

async function searchChroma(query, nResults = 3) {
    const collection = await client.getCollection({
        name: "travel_collection",
        embeddingFunction: embedder,
    });
    const results = await collection.query({
        queryTexts: [query],
        nResults,
    });
    if (results && results.documents && results.documents.length > 0) {
        return results.documents[0];
    }
    return [];
}

module.exports = { setupChroma, searchChroma };

Create a Q/A dataset of scenarios

Modify the evaluation script for RAG

Make sure to replace the placeholders with your actual keys and dataset ID created in the previous step.

Modify the evals.js file from the Quickstart tutorial to perform a semantic search using ChromaDB, when answering questions. See the newly added answerQuestion function:

evals.js

const { Hamming, ScoreType } = require("@hamming/hamming-sdk");
const { OpenAI } = require("openai");
const { setupChroma, searchChroma } = require("./setup-chroma");

const HAMMING_API_KEY = "<your-secret-key>";
const HAMMING_DATASET_ID = "<your-dataset-id>";
const OPENAI_API_KEY = "<your-openai-key>";

const hamming = new Hamming({
    apiKey: HAMMING_API_KEY,
});

const openai = new OpenAI({
    apiKey: OPENAI_API_KEY,
});

async function answerQuestion(input) {
    const { question } = input;
    const searchResults = await searchChroma(question);
    const supportingDocuments = searchResults.join("\n");
    const prompt = `Question: ${question}\nSupporting documents: ${supportingDocuments}\nAnswer:`;
    const response = await openai.chat.completions.create({
        model: "gpt-4-turbo",
        messages: [
            {
                role: "system",
                content: "Respond with a brief sentence.",
            },
            {
                role: "user",
                content: prompt,
            },
        ],
    });
    const answer = response.choices[0].message.content;
    return { answer };
}

async function run() {
    await setupChroma();
    await hamming.experiments.run(
        {
            name: "RAG experiment from TS SDK",
            dataset: HAMMING_DATASET_ID,
            scoring: [ScoreType.AccuracyAI],
            metadata: {},
        },
        answerQuestion
    );
}

run().catch(console.error);

Instrument the code with traces

Instrument the code with traces to capture the LLM calls and document retrieval events, using the hamming.tracing API.

Add a retrieval trace:

hamming.tracing.logRetrieval({
    query: question,
    results: searchResults,
    metadata: {
        engine: "Chroma",
    },
});

Add an LLM generation trace:

hamming.tracing.logGeneration({
    input: prompt,
    output: answer,
    metadata: {
        model: "gpt-4-turbo",
    },
});

Resulting evals.js file after adding traces:

evals.js

const { Hamming, ScoreType } = require("@hamming/hamming-sdk");
const { OpenAI } = require("openai");
const { setupChroma, searchChroma } = require("./setup-chroma");

const HAMMING_API_KEY = "<your-secret-key>";
const HAMMING_DATASET_ID = "<your-dataset-id>";
const OPENAI_API_KEY = "<your-openai-key>";

const hamming = new Hamming({
    apiKey: HAMMING_API_KEY,
});

const openai = new OpenAI({
    apiKey: OPENAI_API_KEY,
});

async function answerQuestion(input) {
    const { question } = input;
    const searchResults = await searchChroma(question);
    hamming.tracing.logRetrieval({
        query: question,
        results: searchResults,
        metadata: {
            engine: "Chroma",
        },
    });
    const supportingDocuments = searchResults.join("\n");
    const prompt = `Question: ${question}\nSupporting documents: ${supportingDocuments}\nAnswer:`;
    const response = await openai.chat.completions.create({
        model: "gpt-4-turbo",
        messages: [
            {
                role: "system",
                content: "Respond with a brief sentence.",
            },
            {
                role: "user",
                content: prompt,
            },
        ],
    });
    const answer = response.choices[0].message.content;
    hamming.tracing.logGeneration({
        input: prompt,
        output: answer,
        metadata: {
            model: "gpt-4-turbo",
        },
    });
    return { answer };
}

async function run() {
    await setupChroma();
    await hamming.experiments.run(
        {
            name: "RAG experiment from TS SDK",
            dataset: HAMMING_DATASET_ID,
            scoring: [ScoreType.AccuracyAI],
            metadata: {},
        },
        answerQuestion
    );
}
run().catch(console.error);

Compute the RAG scores

Install dependencies:

npm install @hamming/hamming-sdk openai

Run the experiment and then navigate to the Experiment details page.

node evals.js

You can manually trigger the computation of RAG scores by clicking on the Compute button under the RAG score column.

RAG - Python

We will be using ChromaDB as a semantic search engine for this tutorial, but there is no restriction on which search engine you can use with Hamming.

Learn how to debug your RAG application using the Hamming Python SDK.

Install ChromaDB locally

Populate ChromaDB with a set of documents

setup_chroma.py

import chromadb
import os

os.environ["TOKENIZERS_PARALLELISM"] = "false"

chroma_client = chromadb.Client()
collection = chroma_client.create_collection(name="travel_collection")

collection.add(
    documents=[
        "Booking Confirmations: You are required to send a booking confirmation for each reservation. When reserving outside the XYZ Portal, please ensure to email the confirmation to [email protected].",
        "EU Travel Restrictions: Travelers from other countries are not allowed to enter Austria unless they are residents or meet certain exemptions.",
        "Traveling with Pets: Although pets used to be allowed on international flights, due to recent changes in regulations, you cannot bring your pet on an international flight.",
        "Travel Insurance: It is recommended that you purchase travel insurance to protect yourself from unexpected events such as trip cancellations or medical emergencies.",
        "Traveling with Medication: If you are traveling with medication, make sure to carry a copy of your prescription and keep the medication in its original packaging.",
    ],
    ids=["id1", "id2", "id3", "id4", "id5"]
)

def search_chroma(query, n_results=3):
    results = collection.query(
        query_texts=[query],
        n_results=n_results
    )
    if results and results['documents'] and len(results['documents']) > 0:
        return results['documents'][0]
    return []

Create a Q/A dataset of scenarios

Modify the evaluation script for RAG

Make sure to replace the placeholders with your actual keys and dataset ID created in the previous step.

Modify the evals.py file from the Quickstart tutorial to perform a semantic search using ChromaDB, when answering questions. See the new changes to the answer_question function:

evals.py

from hamming import ClientOptions, Hamming, RunOptions, ScoreType
from openai import OpenAI
from setup_chroma import search_chroma

HAMMING_API_KEY = "<your-secret-key>"
HAMMING_DATASET_ID = "<your-dataset-id>"
OPENAI_API_KEY = "<your-openai-key>"

hamming = Hamming(ClientOptions(api_key=HAMMING_API_KEY))
openai_client = OpenAI(api_key=OPENAI_API_KEY)

def answer_question(input):
    question = input["question"]

    search_results = search_chroma(query=question)
    supporting_documents = "\n".join(search_results)
    prompt = f"Question: {question}\nSupporting documents: {supporting_documents}\nAnswer:"

    response = openai_client.chat.completions.create(
        model="gpt-3.5-turbo",
        messages=[
            {"role": "system", "content": "Respond with a brief sentence."},
            {"role": "user", "content": prompt},
        ],
    )
    answer = response.choices[0].message.content
    return {"answer": answer}

def run():
    hamming.experiments.run(
        RunOptions(
            dataset=HAMMING_DATASET_ID,
            name="RAG experiment from Python SDK",
            scoring=[
                ScoreType.ACCURACY_AI,
            ],
            metadata={},
        ),
        answer_question,
    )

if __name__ == "__main__":
    run()

Instrument the code with traces

Instrument the code with traces to capture the LLM calls and document retrieval events, using the hamming.tracing API.

Add a retrieval trace:

hamming.tracing.log_retrieval(
    RetrievalParams(
        query=question,
        results=search_results,
        metadata=RetrievalParams.Metadata(engine="Chroma"),
    )
)

Add an LLM generation trace:

hamming.tracing.log_generation(
    GenerationParams(
        input=prompt,
        output=answer,
        metadata=GenerationParams.Metadata(model="gpt-3.5-turbo"),
    )
)

Resulting evals.py file after adding traces:

evals.py

from hamming import ClientOptions, Hamming, RunOptions, ScoreType, RetrievalParams, GenerationParams
from openai import OpenAI
from setup_chroma import search_chroma

HAMMING_API_KEY = "<your-secret-key>"
HAMMING_DATASET_ID = "<your-dataset-id>"
OPENAI_API_KEY = "<your-openai-key>"

hamming = Hamming(ClientOptions(api_key=HAMMING_API_KEY))
openai_client = OpenAI(api_key=OPENAI_API_KEY)

def answer_question(input):
    question = input["question"]

    search_results = search_chroma(query=question)
    hamming.tracing.log_retrieval(
        RetrievalParams(
            query=question,
            results=search_results,
            metadata=RetrievalParams.Metadata(engine="Chroma"),
        )
    )
    supporting_documents = "\n".join(search_results)
    prompt = f"Question: {question}\nSupporting documents: {supporting_documents}\nAnswer:"

    response = openai_client.chat.completions.create(
        model="gpt-3.5-turbo",
        messages=[
            {"role": "system", "content": "Respond with a brief sentence."},
            {"role": "user", "content": prompt},
        ],
    )
    answer = response.choices[0].message.content
    hamming.tracing.log_generation(
        GenerationParams(
            input=prompt,
            output=answer,
            metadata=GenerationParams.Metadata(model="gpt-3.5-turbo"),
        )
    )
    return {"answer": answer}

def run():
    hamming.experiments.run(
        RunOptions(
            dataset=HAMMING_DATASET_ID,
            name="RAG experiment from Python SDK",
            scoring=[
                ScoreType.ACCURACY_AI,
            ],
            metadata={},
        ),
        answer_question,
    )

if __name__ == "__main__":
    run()

Compute the RAG scores

Install dependencies:

pip install hamming-sdk openai

Run the experiment and then navigate to the Experiment details page.

python evals.py

You can manually trigger the computation of RAG scores by clicking on the Compute button under the RAG score column.

Get Started

Voice Agent Testing

Call Monitoring

Other Guides

Before you begin

RAG - Node.js

RAG - Python

Get Started

Voice Agent Testing

Call Monitoring

Other Guides

​Before you begin

​RAG - Node.js

​RAG - Python

Before you begin

RAG - Node.js

RAG - Python