Consistency based examples

from typing import List
from pydantic import BaseModel, Field
import instructor
from openai import OpenAI

class Response(BaseModel):
    content: str = Field(description="The model's response to the prompt")
    confidence: float = Field(description="Confidence score between 0 and 1")

client = instructor.from_openai(OpenAI())

def generate_responses(prompt: str, n: int = 3) -> List[Response]:
    responses = []
    for _ in range(n):
        response = client.chat.completions.create(
            model="gpt-4",
            messages=[{"role": "user", "content": prompt}],
            response_model=Response
        )
        responses.append(response)
    return responses

import numpy as np
from scipy.stats import entropy

def calculate_metrics(responses: List[Response]) -> tuple[float, float]:
    # Calculate entropy
    confidences = [r.confidence for r in responses]
    entropy_score = entropy(confidences)

    # Calculate repetitiveness
    unique_responses = len(set(r.content for r in responses))
    repetitiveness = 1 - (unique_responses / len(responses))

    return entropy_score, repetitiveness

from typing import List, Tuple
from pydantic import BaseModel, Field
import instructor
from openai import OpenAI
import numpy as np
from scipy.stats import entropy

class Example(BaseModel):
    text: str
    score: float = Field(description="Combined quality score")
    entropy: float = Field(description="Entropy of responses")
    repetitiveness: float = Field(description="Repetitiveness of responses")

class COSPSelector:
    def __init__(self, client: OpenAI, n_samples: int = 3):
        self.client = instructor.from_openai(client)
        self.n_samples = n_samples

    def generate_responses(self, prompt: str) -> List[Response]:
        return [
            self.client.chat.completions.create(
                model="gpt-4",
                messages=[{"role": "user", "content": prompt}],
                response_model=Response
            )
            for _ in range(self.n_samples)
        ]

    def calculate_metrics(self, responses: List[Response]) -> Tuple[float, float]:
        confidences = [r.confidence for r in responses]
        entropy_score = entropy(confidences)

        unique_responses = len(set(r.content for r in responses))
        repetitiveness = 1 - (unique_responses / len(responses))

        return entropy_score, repetitiveness

    def select_examples(self, candidates: List[str], k: int) -> List[Example]:
        examples = []

        for text in candidates:
            responses = self.generate_responses(text)
            entropy_score, repetitiveness = self.calculate_metrics(responses)

            # Combined score (lower is better)
            score = entropy_score - repetitiveness

            examples.append(Example(
                text=text,
                score=score,
                entropy=entropy_score,
                repetitiveness=repetitiveness
            ))

        # Sort by score (lower is better) and select top k
        return sorted(examples, key=lambda x: x.score)[:k]

# Initialize COSP selector
client = OpenAI()
selector = COSPSelector(client)

# Candidate examples
candidates = [
    "The quick brown fox jumps over the lazy dog",
    "Machine learning is a subset of artificial intelligence",
    "Python is a high-level programming language",
    # ... more examples
]

# Select best examples
best_examples = selector.select_examples(candidates, k=3)

# Use selected examples in your prompt
selected_texts = [ex.text for ex in best_examples]
prompt = f"""Use these examples to guide your response:

Examples:
{chr(10).join(f'- {text}' for text in selected_texts)}

Now, please respond to: [your query here]
"""