Evaluations

npm install openai

pip install hamming-sdk

from hamming import ClientOptions, Hamming, RunOptions, ScoreType
from openai import OpenAI

HAMMING_API_KEY = "<your-secret-key>"
HAMMING_DATASET_ID = "<your-dataset-id>"
OPENAI_API_KEY = "<your-openai-key>"

hamming = Hamming(ClientOptions(api_key=HAMMING_API_KEY))
openai_client = OpenAI(api_key=OPENAI_API_KEY)


def answer_question(input):
    question = input["question"]
    response = openai_client.chat.completions.create(
        model="gpt-3.5-turbo",
        messages=[
            {"role": "system", "content": "Respond with a brief sentence."},
            {"role": "user", "content": question},
        ],
    )
    answer = response.choices[0].message.content
    return {"answer": answer}


def run():
    hamming.experiments.run(
        RunOptions(
            dataset=HAMMING_DATASET_ID,
            name="Example experiment from Python SDK",
            scoring=[
                ScoreType.ACCURACY_AI,
            ],
            metadata={},
        ),
        answer_question,
    )


if __name__ == "__main__":
    run()

pip install openai

python evals.py