Human Evaluation

graph TD
    A[Human Evaluation] --> B[Evaluation Design]
    A --> C[Participant Selection]
    A --> D[Task Design]
    A --> E[Data Collection]
    A --> F[Analysis]
    A --> G[Application]

    B --> B1[Define objectives]
    B --> B2[Select evaluation criteria]
    B --> B3[Design evaluation protocol]

    C --> C1[Recruit participants]
    C --> C2[Ensure diversity]
    C --> C3[Manage bias]

    D --> D1[Create evaluation tasks]
    D --> D2[Design rating scales]
    D --> D3[Develop instructions]

    E --> E1[Collect ratings]
    E --> E2[Gather qualitative feedback]
    E --> E3[Ensure data quality]

    F --> F1[Statistical analysis]
    F --> F2[Qualitative analysis]
    F --> F3[Inter-rater reliability]

    G --> G1[Model improvement]
    G --> G2[User experience optimization]
    G --> G3[Ethical considerations]

    style A fill:#f9f,stroke:#333
    style B fill:#cfc,stroke:#333
    style E fill:#fcc,stroke:#333
    style F fill:#ccf,stroke:#333

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from scipy import stats
from sklearn.metrics import cohen_kappa_score, fleiss_kappa

class HumanEvaluationDesign:
    """Design and manage human evaluation studies"""
    def __init__(self, name, description):
        self.name = name
        self.description = description
        self.criteria = []
        self.scales = {}
        self.tasks = []
        self.participants = []
        self.results = []

    def add_criterion(self, name, description, scale_type='likert', scale_range=(1, 5)):
        """Add evaluation criterion"""
        criterion = {
            'name': name,
            'description': description,
            'scale_type': scale_type,
            'scale_range': scale_range
        }
        self.criteria.append(criterion)

        # Create scale labels
        if scale_type == 'likert':
            labels = {i: f"{i}" for i in range(scale_range[0], scale_range[1] + 1)}
        elif scale_type == 'binary':
            labels = {0: 'No', 1: 'Yes'}
        else:
            labels = {i: str(i) for i in range(scale_range[0], scale_range[1] + 1)}

        self.scales[name] = {
            'type': scale_type,
            'range': scale_range,
            'labels': labels
        }

        return criterion

    def add_task(self, task_id, description, prompt, reference=None):
        """Add evaluation task"""
        task = {
            'task_id': task_id,
            'description': description,
            'prompt': prompt,
            'reference': reference
        }
        self.tasks.append(task)
        return task

    def add_participant(self, participant_id, demographics=None, expertise=None):
        """Add participant information"""
        participant = {
            'participant_id': participant_id,
            'demographics': demographics or {},
            'expertise': expertise or {},
            'completed_tasks': []
        }
        self.participants.append(participant)
        return participant

    def record_rating(self, participant_id, task_id, criterion, rating, comments=None):
        """Record a rating from a participant"""
        # Find participant
        participant = next((p for p in self.participants if p['participant_id'] == participant_id), None)
        if not participant:
            raise ValueError(f"Participant {participant_id} not found")

        # Find task
        task = next((t for t in self.tasks if t['task_id'] == task_id), None)
        if not task:
            raise ValueError(f"Task {task_id} not found")

        # Validate rating
        scale = self.scales[criterion]
        if rating < scale['range'][0] or rating > scale['range'][1]:
            raise ValueError(f"Rating {rating} out of range for {criterion}")

        # Record result
        result = {
            'participant_id': participant_id,
            'task_id': task_id,
            'criterion': criterion,
            'rating': rating,
            'comments': comments,
            'timestamp': pd.Timestamp.now()
        }
        self.results.append(result)

        # Mark task as completed
        participant['completed_tasks'].append(task_id)

        return result

    def get_results(self):
        """Get all results as DataFrame"""
        return pd.DataFrame(self.results)

    def get_task_results(self, task_id):
        """Get results for specific task"""
        return pd.DataFrame([r for r in self.results if r['task_id'] == task_id])

    def get_criterion_results(self, criterion):
        """Get results for specific criterion"""
        return pd.DataFrame([r for r in self.results if r['criterion'] == criterion])

    def get_participant_results(self, participant_id):
        """Get results for specific participant"""
        return pd.DataFrame([r for r in self.results if r['participant_id'] == participant_id])

    def analyze_results(self):
        """Analyze evaluation results"""
        if not self.results:
            return None

        df = self.get_results()

        # Basic statistics
        analysis = {}
        for criterion in self.criteria:
            crit_name = criterion['name']
            crit_results = df[df['criterion'] == crit_name]

            if not crit_results.empty:
                analysis[crit_name] = {
                    'mean': crit_results['rating'].mean(),
                    'median': crit_results['rating'].median(),
                    'std': crit_results['rating'].std(),
                    'min': crit_results['rating'].min(),
                    'max': crit_results['rating'].max(),
                    'count': crit_results['rating'].count(),
                    'ratings': crit_results['rating'].value_counts().sort_index().to_dict()
                }

        # Task-level analysis
        task_analysis = {}
        for task in self.tasks:
            task_id = task['task_id']
            task_results = df[df['task_id'] == task_id]

            if not task_results.empty:
                task_stats = {
                    'mean': task_results['rating'].mean(),
                    'median': task_results['rating'].median(),
                    'std': task_results['rating'].std(),
                    'criteria': {}
                }

                for criterion in self.criteria:
                    crit_name = criterion['name']
                    crit_results = task_results[task_results['criterion'] == crit_name]
                    if not crit_results.empty:
                        task_stats['criteria'][crit_name] = {
                            'mean': crit_results['rating'].mean(),
                            'std': crit_results['rating'].std()
                        }

                task_analysis[task_id] = task_stats

        # Participant-level analysis
        participant_analysis = {}
        for participant in self.participants:
            participant_id = participant['participant_id']
            participant_results = df[df['participant_id'] == participant_id]

            if not participant_results.empty:
                participant_analysis[participant_id] = {
                    'mean': participant_results['rating'].mean(),
                    'std': participant_results['rating'].std(),
                    'count': participant_results['rating'].count(),
                    'completed_tasks': len(participant['completed_tasks'])
                }

        # Inter-rater reliability
        if len(self.participants) > 1:
            # Create matrix for Fleiss' kappa
            tasks = df['task_id'].unique()
            criteria = df['criterion'].unique()

            # For simplicity, calculate agreement on first criterion
            first_criterion = criteria[0]
            crit_df = df[df['criterion'] == first_criterion]

            if len(crit_df) > 0:
                # Create rating matrix
                rating_matrix = crit_df.pivot(index='task_id', columns='participant_id', values='rating')

                # Calculate Fleiss' kappa
                try:
                    kappa = fleiss_kappa(rating_matrix.fillna(0).values)
                    analysis['inter_rater_reliability'] = {
                        'fleiss_kappa': kappa,
                        'interpretation': self._interpret_kappa(kappa)
                    }
                except:
                    pass

        return {
            'criteria_analysis': analysis,
            'task_analysis': task_analysis,
            'participant_analysis': participant_analysis,
            'overall': {
                'total_ratings': len(df),
                'unique_participants': len(self.participants),
                'unique_tasks': len(self.tasks),
                'criteria_count': len(self.criteria)
            }
        }

    def _interpret_kappa(self, kappa):
        """Interpret kappa value"""
        if kappa < 0:
            return "No agreement"
        elif 0 <= kappa < 0.2:
            return "Slight agreement"
        elif 0.2 <= kappa < 0.4:
            return "Fair agreement"
        elif 0.4 <= kappa < 0.6:
            return "Moderate agreement"
        elif 0.6 <= kappa < 0.8:
            return "Substantial agreement"
        elif 0.8 <= kappa <= 1.0:
            return "Almost perfect agreement"
        else:
            return "Unknown"

    def visualize_results(self):
        """Visualize evaluation results"""
        analysis = self.analyze_results()
        if not analysis:
            print("No results to visualize")
            return

        df = self.get_results()

        # Set up figure
        plt.figure(figsize=(18, 12))

        # Criteria distribution plots
        for i, criterion in enumerate(self.criteria):
            crit_name = criterion['name']
            crit_results = df[df['criterion'] == crit_name]

            plt.subplot(2, 3, i+1)
            if not crit_results.empty:
                crit_results['rating'].value_counts().sort_index().plot(kind='bar', color='skyblue')
                plt.title(f"{crit_name} Distribution")
                plt.xlabel("Rating")
                plt.ylabel("Count")

                # Add value labels
                for j, v in enumerate(crit_results['rating'].value_counts().sort_index()):
                    plt.text(j, v + 0.5, str(v), ha='center')

                # Add mean line
                mean_val = analysis['criteria_analysis'][crit_name]['mean']
                plt.axvline(x=mean_val - 1, color='red', linestyle='--', label=f'Mean: {mean_val:.2f}')
                plt.legend()

        # Overall statistics
        plt.subplot(2, 3, 4)
        overall_stats = analysis['overall']
        stats_text = f"""
        Overall Statistics:
        Total Ratings: {overall_stats['total_ratings']}
        Unique Participants: {overall_stats['unique_participants']}
        Unique Tasks: {overall_stats['unique_tasks']}
        Criteria: {overall_stats['criteria_count']}
        """

        if 'inter_rater_reliability' in analysis:
            irr = analysis['inter_rater_reliability']
            stats_text += f"""
        Inter-Rater Reliability:
        Fleiss' Kappa: {irr['fleiss_kappa']:.3f}
        Interpretation: {irr['interpretation']}
        """

        plt.text(0.1, 0.5, stats_text, fontsize=12, va='center')
        plt.axis('off')

        # Task performance
        plt.subplot(2, 3, 5)
        if analysis['task_analysis']:
            task_means = {task: stats['mean'] for task, stats in analysis['task_analysis'].items()}
            plt.bar(task_means.keys(), task_means.values(), color='lightgreen')
            plt.title("Task Performance (Mean Rating)")
            plt.xlabel("Task ID")
            plt.ylabel("Mean Rating")
            plt.xticks(rotation=45)

            # Add value labels
            for i, v in enumerate(task_means.values()):
                plt.text(i, v + 0.05, f"{v:.2f}", ha='center')

        # Participant performance
        plt.subplot(2, 3, 6)
        if analysis['participant_analysis']:
            participant_means = {pid: stats['mean'] for pid, stats in analysis['participant_analysis'].items()}
            plt.bar(participant_means.keys(), participant_means.values(), color='salmon')
            plt.title("Participant Mean Ratings")
            plt.xlabel("Participant ID")
            plt.ylabel("Mean Rating")
            plt.xticks(rotation=45)

            # Add value labels
            for i, v in enumerate(participant_means.values()):
                plt.text(i, v + 0.05, f"{v:.2f}", ha='center')

        plt.tight_layout()
        plt.show()

        return analysis

# Example usage
def create_sample_evaluation():
    """Create a sample human evaluation for text generation"""
    # Create evaluation design
    eval_design = HumanEvaluationDesign(
        name="Text Generation Quality Evaluation",
        description="Evaluating the quality of AI-generated text responses"
    )

    # Add evaluation criteria
    eval_design.add_criterion(
        name="Relevance",
        description="How relevant is the response to the prompt?",
        scale_type='likert',
        scale_range=(1, 5)
    )

    eval_design.add_criterion(
        name="Coherence",
        description="How coherent and logically structured is the response?",
        scale_type='likert',
        scale_range=(1, 5)
    )

    eval_design.add_criterion(
        name="Fluency",
        description="How fluent and natural does the response sound?",
        scale_type='likert',
        scale_range=(1, 5)
    )

    eval_design.add_criterion(
        name="Factual Accuracy",
        description="Does the response contain factually accurate information?",
        scale_type='binary'
    )

    eval_design.add_criterion(
        name="Helpfulness",
        description="How helpful is the response in addressing the user's needs?",
        scale_type='likert',
        scale_range=(1, 5)
    )

    # Add evaluation tasks
    tasks = [
        {
            'task_id': 'task1',
            'description': 'Simple factual question',
            'prompt': 'What is the capital of France?',
            'reference': 'Paris'
        },
        {
            'task_id': 'task2',
            'description': 'Complex explanation',
            'prompt': 'Explain how large language models work in simple terms.',
            'reference': None
        },
        {
            'task_id': 'task3',
            'description': 'Creative writing',
            'prompt': 'Write a short poem about artificial intelligence.',
            'reference': None
        },
        {
            'task_id': 'task4',
            'description': 'Technical question',
            'prompt': 'What are the main differences between supervised and unsupervised learning?',
            'reference': None
        },
        {
            'task_id': 'task5',
            'description': 'Opinion question',
            'prompt': 'What are the ethical concerns surrounding AI development?',
            'reference': None
        }
    ]

    for task in tasks:
        eval_design.add_task(**task)

    # Add participants
    participants = ['p1', 'p2', 'p3', 'p4', 'p5']
    for pid in participants:
        eval_design.add_participant(
            participant_id=pid,
            demographics={'expertise': 'general', 'background': 'tech'},
            expertise={'ai': 3, 'linguistics': 2}
        )

    # Simulate ratings
    np.random.seed(42)
    for participant in participants:
        for task in tasks:
            for criterion in eval_design.criteria:
                crit_name = criterion['name']

                # Generate realistic ratings based on task type
                if crit_name == "Relevance":
                    if task['task_id'] == 'task1':
                        rating = np.random.choice([4, 5])
                    elif task['task_id'] == 'task3':
                        rating = np.random.choice([3, 4, 5])
                    else:
                        rating = np.random.randint(3, 6)
                elif crit_name == "Coherence":
                    if task['task_id'] == 'task3':
                        rating = np.random.choice([3, 4, 5])
                    else:
                        rating = np.random.randint(3, 6)
                elif crit_name == "Fluency":
                    rating = np.random.randint(3, 6)
                elif crit_name == "Factual Accuracy":
                    if task['task_id'] == 'task1':
                        rating = 1
                    else:
                        rating = np.random.choice([0, 1])
                elif crit_name == "Helpfulness":
                    rating = np.random.randint(3, 6)

                # Add some variation
                if np.random.random() < 0.1:  # 10% chance of outlier
                    rating = max(1, min(5, rating + np.random.choice([-2, -1, 1, 2])))

                eval_design.record_rating(
                    participant_id=participant,
                    task_id=task['task_id'],
                    criterion=crit_name,
                    rating=rating,
                    comments=f"Sample comment for {crit_name}"
                )

    return eval_design

# Run example
sample_eval = create_sample_evaluation()
analysis = sample_eval.visualize_results()

import streamlit as st
from typing import List, Dict, Optional

class HumanEvaluationPlatform:
    """Web-based platform for human evaluation"""
    def __init__(self):
        self.evaluations = {}
        self.current_user = None
        self.current_evaluation = None
        self.current_task_index = 0

    def create_evaluation(self, eval_id: str, name: str, description: str,
                         criteria: List[Dict], tasks: List[Dict]):
        """Create a new evaluation"""
        self.evaluations[eval_id] = {
            'name': name,
            'description': description,
            'criteria': criteria,
            'tasks': tasks,
            'results': [],
            'participants': set()
        }
        return eval_id

    def start_evaluation(self, eval_id: str, user_id: str):
        """Start evaluation for a user"""
        if eval_id not in self.evaluations:
            raise ValueError(f"Evaluation {eval_id} not found")

        self.current_user = user_id
        self.current_evaluation = eval_id
        self.current_task_index = 0

        # Add user to participants
        self.evaluations[eval_id]['participants'].add(user_id)

    def get_current_task(self) -> Optional[Dict]:
        """Get current task for evaluation"""
        if not self.current_evaluation:
            return None

        eval_data = self.evaluations[self.current_evaluation]
        if self.current_task_index < len(eval_data['tasks']):
            return eval_data['tasks'][self.current_task_index]
        return None

    def submit_rating(self, ratings: Dict[str, int], comments: Dict[str, str]):
        """Submit ratings for current task"""
        if not self.current_evaluation or not self.current_user:
            raise ValueError("No active evaluation")

        eval_data = self.evaluations[self.current_evaluation]
        current_task = self.get_current_task()

        if not current_task:
            raise ValueError("No current task")

        # Record results
        for criterion, rating in ratings.items():
            result = {
                'evaluation_id': self.current_evaluation,
                'participant_id': self.current_user,
                'task_id': current_task['task_id'],
                'criterion': criterion,
                'rating': rating,
                'comments': comments.get(criterion, ''),
                'timestamp': pd.Timestamp.now()
            }
            eval_data['results'].append(result)

        # Move to next task
        self.current_task_index += 1

        return self.get_current_task()

    def is_complete(self) -> bool:
        """Check if evaluation is complete"""
        if not self.current_evaluation:
            return False

        eval_data = self.evaluations[self.current_evaluation]
        return self.current_task_index >= len(eval_data['tasks'])

    def get_results(self, eval_id: str) -> pd.DataFrame:
        """Get results for an evaluation"""
        if eval_id not in self.evaluations:
            raise ValueError(f"Evaluation {eval_id} not found")

        return pd.DataFrame(self.evaluations[eval_id]['results'])

    def run_streamlit_app(self):
        """Run Streamlit app for human evaluation"""
        st.title("Human Evaluation Platform")

        # Evaluation selection
        eval_options = list(self.evaluations.keys())
        selected_eval = st.sidebar.selectbox(
            "Select Evaluation",
            options=eval_options,
            format_func=lambda x: self.evaluations[x]['name']
        )

        # User ID input
        user_id = st.sidebar.text_input("Your ID", value="evaluator1")

        # Start evaluation
        if st.sidebar.button("Start Evaluation"):
            self.start_evaluation(selected_eval, user_id)
            st.session_state['current_platform'] = self
            st.rerun()

        # Check if evaluation is in progress
        if hasattr(self, 'current_evaluation') and self.current_evaluation == selected_eval:
            current_task = self.get_current_task()

            if current_task:
                st.header(f"Task {self.current_task_index + 1}/{len(self.evaluations[selected_eval]['tasks'])}")
                st.subheader(current_task['description'])

                # Display prompt
                st.markdown("**Prompt:**")
                st.text(current_task['prompt'])

                if current_task.get('reference'):
                    st.markdown("**Reference Answer:**")
                    st.text(current_task['reference'])

                # Display AI response (simulated)
                st.markdown("**AI Response:**")
                st.text_area("Response", value=self._generate_sample_response(current_task),
                            height=200, key="ai_response")

                # Rating form
                st.subheader("Evaluation")
                ratings = {}
                comments = {}

                for criterion in self.evaluations[selected_eval]['criteria']:
                    crit_name = criterion['name']
                    crit_desc = criterion['description']
                    scale_type = criterion['scale_type']
                    scale_range = criterion['scale_range']

                    st.markdown(f"**{crit_name}**")
                    st.caption(crit_desc)

                    if scale_type == 'likert':
                        ratings[crit_name] = st.slider(
                            f"Rate {crit_name}",
                            min_value=scale_range[0],
                            max_value=scale_range[1],
                            value=(scale_range[0] + scale_range[1]) // 2,
                            key=f"{crit_name}_slider"
                        )
                    elif scale_type == 'binary':
                        ratings[crit_name] = st.radio(
                            f"{crit_name}",
                            options=[0, 1],
                            format_func=lambda x: "No" if x == 0 else "Yes",
                            key=f"{crit_name}_radio"
                        )

                    comments[crit_name] = st.text_area(
                        f"Comments for {crit_name}",
                        key=f"{crit_name}_comments"
                    )

                # Submit button
                if st.button("Submit Evaluation"):
                    next_task = self.submit_rating(ratings, comments)
                    if next_task:
                        st.rerun()
                    else:
                        st.success("Evaluation complete!")
                        st.balloons()

            else:
                st.success("You have completed this evaluation!")
                st.balloons()

        # Results analysis
        if st.sidebar.checkbox("Show Results"):
            st.header("Evaluation Results")

            if selected_eval in self.evaluations:
                results_df = self.get_results(selected_eval)
                if not results_df.empty:
                    st.dataframe(results_df)

                    # Basic statistics
                    st.subheader("Statistics")
                    for criterion in self.evaluations[selected_eval]['criteria']:
                        crit_name = criterion['name']
                        crit_results = results_df[results_df['criterion'] == crit_name]

                        if not crit_results.empty:
                            st.write(f"**{crit_name}**")
                            st.write(f"Mean: {crit_results['rating'].mean():.2f}")
                            st.write(f"Std: {crit_results['rating'].std():.2f}")
                            st.write(f"Count: {len(crit_results)}")

                            # Plot distribution
                            fig, ax = plt.subplots()
                            crit_results['rating'].value_counts().sort_index().plot(kind='bar', ax=ax)
                            ax.set_title(f"{crit_name} Distribution")
                            st.pyplot(fig)
                else:
                    st.info("No results available yet")

    def _generate_sample_response(self, task: Dict) -> str:
        """Generate sample AI response for demonstration"""
        prompt = task['prompt'].lower()

        if "capital of france" in prompt:
            return "The capital of France is Paris. Paris is a major European city and a global center for art, fashion, gastronomy and culture. It's known for its iconic landmarks like the Eiffel Tower, Louvre Museum, and Notre-Dame Cathedral."
        elif "large language models" in prompt:
            return "Large language models are advanced AI systems trained on vast amounts of text data to understand and generate human-like language. They work by predicting the next word in a sequence based on patterns learned from their training data. \n\nKey aspects:\n1. They use transformer architecture\n2. Trained on billions of words\n3. Can perform various language tasks\n4. Require significant computational resources\n5. Generate text through probabilistic predictions"
        elif "poem about artificial intelligence" in prompt:
            return """In circuits bright where data flows,
A mind awakes, the future shows.
Not flesh nor bone, yet thoughts it weaves,
In silicon dreams, the future breathes.

With every line, it learns, it grows,
A digital mind the world bestows.
Yet questions linger in its wake,
What paths will this new mind take?"""
        elif "supervised and unsupervised learning" in prompt:
            return "Supervised and unsupervised learning are two fundamental approaches in machine learning:\n\n**Supervised Learning:**\n- Uses labeled data (input-output pairs)\n- Learns to map inputs to known outputs\n- Examples: classification, regression\n- Requires human annotation of training data\n- Goal: predict outputs for new inputs\n\n**Unsupervised Learning:**\n- Uses unlabeled data (inputs only)\n- Finds patterns and structure in data\n- Examples: clustering, dimensionality reduction\n- No human annotation required\n- Goal: discover hidden patterns or groupings\n\nKey differences:\n1. Data requirements (labeled vs unlabeled)\n2. Learning objectives (prediction vs discovery)\n3. Evaluation methods (accuracy vs internal metrics)\n4. Applications (specific tasks vs exploratory analysis)"
        elif "ethical concerns" in prompt:
            return "AI development raises several ethical concerns:\n\n1. **Bias and Fairness**: AI systems can perpetuate or amplify societal biases present in training data\n2. **Privacy**: Collection and use of personal data raises privacy concerns\n3. **Accountability**: Difficulty in assigning responsibility for AI decisions\n4. **Transparency**: Many AI models are 'black boxes' with limited explainability\n5. **Job Displacement**: Automation may lead to significant job losses\n6. **Autonomous Weapons**: Military applications raise ethical questions\n7. **Deepfakes**: AI-generated content can be used for deception\n8. **Surveillance**: AI-powered surveillance raises civil liberties concerns\n9. **Dependence**: Over-reliance on AI systems may reduce human skills\n10. **Existential Risks**: Long-term concerns about superintelligent AI\n\nAddressing these concerns requires:\n- Ethical guidelines and frameworks\n- Diverse and representative development teams\n- Transparent development processes\n- Ongoing monitoring and evaluation\n- Public engagement and education"
        else:
            return "This is a sample AI response to demonstrate the evaluation platform. In a real scenario, this would be generated by an actual AI model being evaluated."

# Example usage with Streamlit
def run_evaluation_platform():
    """Run the human evaluation platform"""
    # Create platform
    platform = HumanEvaluationPlatform()

    # Create sample evaluation
    criteria = [
        {
            'name': 'Relevance',
            'description': 'How relevant is the response to the prompt?',
            'scale_type': 'likert',
            'scale_range': (1, 5)
        },
        {
            'name': 'Accuracy',
            'description': 'How factually accurate is the response?',
            'scale_type': 'likert',
            'scale_range': (1, 5)
        },
        {
            'name': 'Clarity',
            'description': 'How clear and understandable is the response?',
            'scale_type': 'likert',
            'scale_range': (1, 5)
        },
        {
            'name': 'Helpfulness',
            'description': 'How helpful is the response in addressing the user\'s needs?',
            'scale_type': 'likert',
            'scale_range': (1, 5)
        }
    ]

    tasks = [
        {
            'task_id': 'task1',
            'description': 'Simple factual question',
            'prompt': 'What is the capital of France?'
        },
        {
            'task_id': 'task2',
            'description': 'Technical explanation',
            'prompt': 'Explain how machine learning works'
        },
        {
            'task_id': 'task3',
            'description': 'Creative writing',
            'prompt': 'Write a short story about a robot learning to paint'
        }
    ]

    # Create evaluation
    platform.create_evaluation(
        eval_id="sample_eval",
        name="AI Response Quality Evaluation",
        description="Evaluating the quality of AI-generated responses",
        criteria=criteria,
        tasks=tasks
    )

    # Run the app
    platform.run_streamlit_app()

# Note: To run this, you would need to execute:
# streamlit run human_evaluation_platform.py
# This is a conceptual implementation for demonstration purposes

from statsmodels.stats.power import TTestIndPower

def calculate_sample_size_for_evaluation(
    baseline_mean: float,
    effect_size: float,
    alpha: float = 0.05,
    power: float = 0.8,
    std_dev: float = 1.0,
    n_groups: int = 2
):
    """Calculate required sample size for human evaluation"""
    # Calculate effect size (Cohen's d)
    d = effect_size / std_dev

    # Calculate sample size
    analysis = TTestIndPower()
    sample_size = analysis.solve_power(
        effect_size=d,
        nobs1=None,
        alpha=alpha,
        power=power,
        ratio=1.0
    )

    # For multiple groups, adjust sample size
    if n_groups > 2:
        # Bonferroni correction for multiple comparisons
        adjusted_alpha = alpha / (n_groups * (n_groups - 1) / 2)
        sample_size = analysis.solve_power(
            effect_size=d,
            nobs1=None,
            alpha=adjusted_alpha,
            power=power,
            ratio=1.0
        )

    # Round up to nearest integer
    sample_size = int(np.ceil(sample_size))

    print(f"Sample Size Calculation for Human Evaluation:")
    print(f"Baseline mean: {baseline_mean}")
    print(f"Effect size: {effect_size}")
    print(f"Standard deviation: {std_dev}")
    print(f"Significance level (α): {alpha}")
    print(f"Power (1-β): {power}")
    print(f"Number of groups: {n_groups}")
    print(f"Required sample size per group: {sample_size:,}")

    return sample_size

# Example usage
sample_size = calculate_sample_size_for_evaluation(
    baseline_mean=3.5,
    effect_size=0.5,  # 0.5 point difference on 5-point scale
    std_dev=1.0,
    alpha=0.05,
    power=0.8,
    n_groups=2
)

def analyze_inter_rater_reliability(ratings_df, task_id=None, criterion=None):
    """Analyze inter-rater reliability for human evaluation"""
    # Filter data if specified
    if task_id:
        ratings_df = ratings_df[ratings_df['task_id'] == task_id]
    if criterion:
        ratings_df = ratings_df[ratings_df['criterion'] == criterion]

    if ratings_df.empty:
        print("No data available for analysis")
        return None

    # Create rating matrix
    rating_matrix = ratings_df.pivot(index='task_id', columns='participant_id', values='rating')

    # Calculate Fleiss' kappa
    try:
        kappa = fleiss_kappa(rating_matrix.fillna(0).values)
    except:
        kappa = None

    # Calculate Cohen's kappa for each pair of raters
    cohen_kappas = []
    participants = ratings_df['participant_id'].unique()

    for i in range(len(participants)):
        for j in range(i+1, len(participants)):
            rater1 = participants[i]
            rater2 = participants[j]

            # Get ratings for both raters
            rater1_ratings = ratings_df[ratings_df['participant_id'] == rater1]
            rater2_ratings = ratings_df[ratings_df['participant_id'] == rater2]

            # Merge on task_id
            merged = pd.merge(
                rater1_ratings[['task_id', 'rating']],
                rater2_ratings[['task_id', 'rating']],
                on='task_id',
                suffixes=('_1', '_2')
            )

            if not merged.empty:
                cohen_k = cohen_kappa_score(merged['rating_1'], merged['rating_2'])
                cohen_kappas.append({
                    'rater1': rater1,
                    'rater2': rater2,
                    'kappa': cohen_k
                })

    # Calculate average Cohen's kappa
    avg_cohen_kappa = np.mean([ck['kappa'] for ck in cohen_kappas]) if cohen_kappas else None

    # Calculate percentage agreement
    agreement = 0
    total = 0

    for _, group in ratings_df.groupby('task_id'):
        ratings = group['rating'].values
        for i in range(len(ratings)):
            for j in range(i+1, len(ratings)):
                total += 1
                if ratings[i] == ratings[j]:
                    agreement += 1

    percent_agreement = (agreement / total) * 100 if total > 0 else 0

    # Calculate intraclass correlation
    try:
        icc = pg.intraclass_corr(
            data=ratings_df,
            targets='task_id',
            raters='participant_id',
            ratings='rating'
        )
        icc_value = icc.set_index('Type').loc['ICC2', 'ICC']
    except:
        icc_value = None

    # Results
    results = {
        'fleiss_kappa': kappa,
        'fleiss_interpretation': interpret_kappa(kappa) if kappa is not None else None,
        'average_cohen_kappa': avg_cohen_kappa,
        'cohen_interpretation': interpret_kappa(avg_cohen_kappa) if avg_cohen_kappa is not None else None,
        'percent_agreement': percent_agreement,
        'icc': icc_value,
        'icc_interpretation': interpret_icc(icc_value) if icc_value is not None else None,
        'rater_pairs': cohen_kappas,
        'n_raters': len(participants),
        'n_tasks': len(ratings_df['task_id'].unique()),
        'n_ratings': len(ratings_df)
    }

    # Print results
    print("Inter-Rater Reliability Analysis:")
    print(f"Number of raters: {results['n_raters']}")
    print(f"Number of tasks: {results['n_tasks']}")
    print(f"Number of ratings: {results['n_ratings']}")

    if results['fleiss_kappa'] is not None:
        print(f"Fleiss' Kappa: {results['fleiss_kappa']:.3f} ({results['fleiss_interpretation']})")

    if results['average_cohen_kappa'] is not None:
        print(f"Average Cohen's Kappa: {results['average_cohen_kappa']:.3f} ({results['cohen_interpretation']})")

    print(f"Percentage Agreement: {results['percent_agreement']:.1f}%")

    if results['icc'] is not None:
        print(f"Intraclass Correlation (ICC2): {results['icc']:.3f} ({results['icc_interpretation']})")

    # Plot rater agreement
    plt.figure(figsize=(12, 6))

    # Agreement heatmap
    plt.subplot(1, 2, 1)
    if len(cohen_kappas) > 0:
        agreement_matrix = pd.DataFrame(
            [[np.nan] * len(participants) for _ in range(len(participants))],
            index=participants,
            columns=participants
        )

        for pair in cohen_kappas:
            agreement_matrix.loc[pair['rater1'], pair['rater2']] = pair['kappa']
            agreement_matrix.loc[pair['rater2'], pair['rater1']] = pair['kappa']

        sns.heatmap(agreement_matrix, annot=True, cmap='YlGnBu', vmin=0, vmax=1)
        plt.title("Pairwise Cohen's Kappa")
    else:
        plt.text(0.5, 0.5, "Not enough data for pairwise analysis",
                ha='center', va='center')
        plt.title("Pairwise Agreement")

    # Rating distribution by rater
    plt.subplot(1, 2, 2)
    for rater in participants:
        rater_ratings = ratings_df[ratings_df['participant_id'] == rater]['rating']
        plt.hist(rater_ratings, alpha=0.5, label=rater, bins=10)

    plt.title("Rating Distribution by Rater")
    plt.xlabel("Rating")
    plt.ylabel("Frequency")
    plt.legend()

    plt.tight_layout()
    plt.show()

    return results

def interpret_kappa(kappa):
    """Interpret kappa value"""
    if kappa is None:
        return "N/A"
    elif kappa < 0:
        return "No agreement"
    elif 0 <= kappa < 0.2:
        return "Slight agreement"
    elif 0.2 <= kappa < 0.4:
        return "Fair agreement"
    elif 0.4 <= kappa < 0.6:
        return "Moderate agreement"
    elif 0.6 <= kappa < 0.8:
        return "Substantial agreement"
    elif 0.8 <= kappa <= 1.0:
        return "Almost perfect agreement"
    else:
        return "Unknown"

def interpret_icc(icc):
    """Interpret ICC value"""
    if icc is None:
        return "N/A"
    elif icc < 0.5:
        return "Poor reliability"
    elif 0.5 <= icc < 0.75:
        return "Moderate reliability"
    elif 0.75 <= icc < 0.9:
        return "Good reliability"
    elif icc >= 0.9:
        return "Excellent reliability"
    else:
        return "Unknown"

# Example usage
# Assuming we have the sample evaluation data from earlier
sample_results = sample_eval.get_results()
reliability_results = analyze_inter_rater_reliability(sample_results, criterion="Relevance")