chatbot.py

import requests
import json
import gradio as gr
from utils import format_as_chat, generate_payload

# Endpoint_url
post_url = 'https://uf9t072wj5ki2ho4.eu-west-1.aws.endpoints.huggingface.cloud/generate'
# Headers
headers = {
    'Content-Type': 'application/json'
}


def echo(message: str, history: list):
    # Concatenate history and message
    formatted_prompt = format_as_chat(message, history)
    # Generate the payload dictionary
    payload = generate_payload(formatted_prompt)

    # Post to API
    response = requests.post(
        post_url,
        data=json.dumps(payload),
        headers=headers
    )

    # Format the reply
    reply = response.json()['generated_text']
    reply = reply.replace('assistant', '').strip()

    return reply


demo = gr.ChatInterface(fn=echo, examples=["wassup?", "grüezi", "もしもし"],
                        title="Llama 3 8B Instruct")
demo.launch()