Advanced
RAG
Advanced
RAG
Debug your RAG application using Hamming.
Before you begin
Make sure you have completed the Evaluations Quickstart guide. You should have your evals script ready and have the first experiment completed.
RAG - Node.js
We will be using ChromaDB as a semantic search engine for this tutorial, but there is no restriction on which search engine you can use with Hamming.
Learn how to debug your RAG application using the Hamming TypeScript SDK.
npm install --save chromadb chromadb-default-embed
Install Chroma via pypi to easily run the backend server:
pip install chromadb
Run the Chroma server:
chroma run --path ./getting-started
Make sure to replace the placeholders with your actual keys and dataset ID created in the previous step.
setup-chroma.js
const { ChromaClient, OpenAIEmbeddingFunction } = require("chromadb");
const client = new ChromaClient();
const embedder = new OpenAIEmbeddingFunction({
openai_api_key: "<your-openai-key>",
});
async function setupChroma() {
const collection = await client.getOrCreateCollection({
name: "travel_collection",
embeddingFunction: embedder,
});
await collection.upsert({
documents: [
"Booking Confirmations: You are required to send a booking confirmation for each reservation. When reserving outside the XYZ Portal, please ensure to email the confirmation to [email protected].",
"EU Travel Restrictions: Travelers from other countries are not allowed to enter Austria unless they are residents or meet certain exemptions.",
"Traveling with Pets: Although pets used to be allowed on international flights, due to recent changes in regulations, you cannot bring your pet on an international flight.",
"Travel Insurance: It is recommended that you purchase travel insurance to protect yourself from unexpected events such as trip cancellations or medical emergencies.",
"Traveling with Medication: If you are traveling with medication, make sure to carry a copy of your prescription and keep the medication in its original packaging.",
],
ids: ["id1", "id2", "id3", "id4", "id5"],
});
return collection;
}
async function searchChroma(query, nResults = 3) {
const collection = await client.getCollection({
name: "travel_collection",
embeddingFunction: embedder,
});
const results = await collection.query({
queryTexts: [query],
nResults,
});
if (results && results.documents && results.documents.length > 0) {
return results.documents[0];
}
return [];
}
module.exports = { setupChroma, searchChroma };
This dataset contains a list of questions and answers. The answers can be found within the ChromaDB collection we just created.
- Download our Q/A dataset file.
- Navigate to Create new dataset and use the drag and drop box to upload the file.
- For Input Columns, select “question”, and for Output Columns, select “answer”.
- Name it “Q/A Dataset” and click Create.
- Copy the dataset ID from by clicking on the Copy ID button.
Make sure to replace the placeholders with your actual keys and dataset ID created in the previous step.
answerQuestion function:evals.js
const { Hamming, ScoreType } = require("@hamming/hamming-sdk");
const { OpenAI } = require("openai");
const { setupChroma, searchChroma } = require("./setup-chroma");
const HAMMING_API_KEY = "<your-secret-key>";
const HAMMING_DATASET_ID = "<your-dataset-id>";
const OPENAI_API_KEY = "<your-openai-key>";
const hamming = new Hamming({
apiKey: HAMMING_API_KEY,
});
const openai = new OpenAI({
apiKey: OPENAI_API_KEY,
});
async function answerQuestion(input) {
const { question } = input;
const searchResults = await searchChroma(question);
const supportingDocuments = searchResults.join("\n");
const prompt = `Question: ${question}\nSupporting documents: ${supportingDocuments}\nAnswer:`;
const response = await openai.chat.completions.create({
model: "gpt-4-turbo",
messages: [
{
role: "system",
content: "Respond with a brief sentence.",
},
{
role: "user",
content: prompt,
},
],
});
const answer = response.choices[0].message.content;
return { answer };
}
async function run() {
await setupChroma();
await hamming.experiments.run(
{
name: "RAG experiment from TS SDK",
dataset: HAMMING_DATASET_ID,
scoring: [ScoreType.AccuracyAI],
metadata: {},
},
answerQuestion
);
}
run().catch(console.error);
Instrument the code with traces to capture the LLM calls and document retrieval events, using the hamming.tracing API.
- Add a retrieval trace:
hamming.tracing.logRetrieval({
query: question,
results: searchResults,
metadata: {
engine: "Chroma",
},
});
- Add an LLM generation trace:
hamming.tracing.logGeneration({
input: prompt,
output: answer,
metadata: {
model: "gpt-4-turbo",
},
});
Resulting evals.js file after adding traces:
evals.js
const { Hamming, ScoreType } = require("@hamming/hamming-sdk");
const { OpenAI } = require("openai");
const { setupChroma, searchChroma } = require("./setup-chroma");
const HAMMING_API_KEY = "<your-secret-key>";
const HAMMING_DATASET_ID = "<your-dataset-id>";
const OPENAI_API_KEY = "<your-openai-key>";
const hamming = new Hamming({
apiKey: HAMMING_API_KEY,
});
const openai = new OpenAI({
apiKey: OPENAI_API_KEY,
});
async function answerQuestion(input) {
const { question } = input;
const searchResults = await searchChroma(question);
hamming.tracing.logRetrieval({
query: question,
results: searchResults,
metadata: {
engine: "Chroma",
},
});
const supportingDocuments = searchResults.join("\n");
const prompt = `Question: ${question}\nSupporting documents: ${supportingDocuments}\nAnswer:`;
const response = await openai.chat.completions.create({
model: "gpt-4-turbo",
messages: [
{
role: "system",
content: "Respond with a brief sentence.",
},
{
role: "user",
content: prompt,
},
],
});
const answer = response.choices[0].message.content;
hamming.tracing.logGeneration({
input: prompt,
output: answer,
metadata: {
model: "gpt-4-turbo",
},
});
return { answer };
}
async function run() {
await setupChroma();
await hamming.experiments.run(
{
name: "RAG experiment from TS SDK",
dataset: HAMMING_DATASET_ID,
scoring: [ScoreType.AccuracyAI],
metadata: {},
},
answerQuestion
);
}
run().catch(console.error);
Install dependencies:
npm install @hamming/hamming-sdk openai
Run the experiment and then navigate to the Experiment details page.
node evals.js
You can manually trigger the computation of RAG scores by clicking on the Compute button under the RAG score column.
RAG - Python
We will be using ChromaDB as a semantic search engine for this tutorial, but there is no restriction on which search engine you can use with Hamming.
Learn how to debug your RAG application using the Hamming Python SDK.
pip install chromadb
setup_chroma.py
import chromadb
import os
os.environ["TOKENIZERS_PARALLELISM"] = "false"
chroma_client = chromadb.Client()
collection = chroma_client.create_collection(name="travel_collection")
collection.add(
documents=[
"Booking Confirmations: You are required to send a booking confirmation for each reservation. When reserving outside the XYZ Portal, please ensure to email the confirmation to [email protected].",
"EU Travel Restrictions: Travelers from other countries are not allowed to enter Austria unless they are residents or meet certain exemptions.",
"Traveling with Pets: Although pets used to be allowed on international flights, due to recent changes in regulations, you cannot bring your pet on an international flight.",
"Travel Insurance: It is recommended that you purchase travel insurance to protect yourself from unexpected events such as trip cancellations or medical emergencies.",
"Traveling with Medication: If you are traveling with medication, make sure to carry a copy of your prescription and keep the medication in its original packaging.",
],
ids=["id1", "id2", "id3", "id4", "id5"]
)
def search_chroma(query, n_results=3):
results = collection.query(
query_texts=[query],
n_results=n_results
)
if results and results['documents'] and len(results['documents']) > 0:
return results['documents'][0]
return []
This dataset contains a list of questions and answers. The answers can be found within the ChromaDB collection we just created.
- Download our Q/A dataset file.
- Navigate to Create new dataset and use the drag and drop box to upload the file.
- For Input Columns, select “question”, and for Output Columns, select “answer”.
- Name it “Q/A Dataset” and click Create.
- Copy the dataset ID from by clicking on the Copy ID button.
Make sure to replace the placeholders with your actual keys and dataset ID created in the previous step.
answer_question function:evals.py
from hamming import ClientOptions, Hamming, RunOptions, ScoreType
from openai import OpenAI
from setup_chroma import search_chroma
HAMMING_API_KEY = "<your-secret-key>"
HAMMING_DATASET_ID = "<your-dataset-id>"
OPENAI_API_KEY = "<your-openai-key>"
hamming = Hamming(ClientOptions(api_key=HAMMING_API_KEY))
openai_client = OpenAI(api_key=OPENAI_API_KEY)
def answer_question(input):
question = input["question"]
search_results = search_chroma(query=question)
supporting_documents = "\n".join(search_results)
prompt = f"Question: {question}\nSupporting documents: {supporting_documents}\nAnswer:"
response = openai_client.chat.completions.create(
model="gpt-3.5-turbo",
messages=[
{"role": "system", "content": "Respond with a brief sentence."},
{"role": "user", "content": prompt},
],
)
answer = response.choices[0].message.content
return {"answer": answer}
def run():
hamming.experiments.run(
RunOptions(
dataset=HAMMING_DATASET_ID,
name="RAG experiment from Python SDK",
scoring=[
ScoreType.ACCURACY_AI,
],
metadata={},
),
answer_question,
)
if __name__ == "__main__":
run()
Instrument the code with traces to capture the LLM calls and document retrieval events, using the hamming.tracing API.
- Add a retrieval trace:
hamming.tracing.log_retrieval(
RetrievalParams(
query=question,
results=search_results,
metadata=RetrievalParams.Metadata(engine="Chroma"),
)
)
- Add an LLM generation trace:
hamming.tracing.log_generation(
GenerationParams(
input=prompt,
output=answer,
metadata=GenerationParams.Metadata(model="gpt-3.5-turbo"),
)
)
Resulting evals.py file after adding traces:
evals.py
from hamming import ClientOptions, Hamming, RunOptions, ScoreType, RetrievalParams, GenerationParams
from openai import OpenAI
from setup_chroma import search_chroma
HAMMING_API_KEY = "<your-secret-key>"
HAMMING_DATASET_ID = "<your-dataset-id>"
OPENAI_API_KEY = "<your-openai-key>"
hamming = Hamming(ClientOptions(api_key=HAMMING_API_KEY))
openai_client = OpenAI(api_key=OPENAI_API_KEY)
def answer_question(input):
question = input["question"]
search_results = search_chroma(query=question)
hamming.tracing.log_retrieval(
RetrievalParams(
query=question,
results=search_results,
metadata=RetrievalParams.Metadata(engine="Chroma"),
)
)
supporting_documents = "\n".join(search_results)
prompt = f"Question: {question}\nSupporting documents: {supporting_documents}\nAnswer:"
response = openai_client.chat.completions.create(
model="gpt-3.5-turbo",
messages=[
{"role": "system", "content": "Respond with a brief sentence."},
{"role": "user", "content": prompt},
],
)
answer = response.choices[0].message.content
hamming.tracing.log_generation(
GenerationParams(
input=prompt,
output=answer,
metadata=GenerationParams.Metadata(model="gpt-3.5-turbo"),
)
)
return {"answer": answer}
def run():
hamming.experiments.run(
RunOptions(
dataset=HAMMING_DATASET_ID,
name="RAG experiment from Python SDK",
scoring=[
ScoreType.ACCURACY_AI,
],
metadata={},
),
answer_question,
)
if __name__ == "__main__":
run()
Install dependencies:
pip install hamming-sdk openai
Run the experiment and then navigate to the Experiment details page.
python evals.py
You can manually trigger the computation of RAG scores by clicking on the Compute button under the RAG score column.
On this page