Подключение к LLM API

# .env.example (коммитим в git — без реальных значений!)
OPENAI_API_KEY=sk-...
ANTHROPIC_API_KEY=sk-ant-...
GOOGLE_API_KEY=AIza...
GROQ_API_KEY=gsk_...

# Для Ollama (локально)
OLLAMA_BASE_URL=http://localhost:11434

# Выбор провайдера по умолчанию
DEFAULT_PROVIDER=anthropic
DEFAULT_MODEL=claude-opus-4-6

# Лимиты
MAX_TOKENS=2048
REQUEST_TIMEOUT=60

from pydantic import Field
from pydantic_settings import BaseSettings, SettingsConfigDict
from pydantic import SecretStr

class LLMSettings(BaseSettings):
    model_config = SettingsConfigDict(
        env_file=".env",
        env_file_encoding="utf-8",
        extra="ignore",
    )

    # API ключи — SecretStr скрывает значение в логах
    openai_api_key: SecretStr | None = None
    anthropic_api_key: SecretStr | None = None
    google_api_key: SecretStr | None = None
    groq_api_key: SecretStr | None = None

    # Настройки по умолчанию
    default_provider: str = "anthropic"
    default_model: str = "claude-opus-4-6"
    max_tokens: int = 2048
    request_timeout: int = 60
    ollama_base_url: str = "http://localhost:11434"

    def get_key(self, provider: str) -> str:
        """Возвращает ключ провайдера или бросает ошибку."""
        key_map = {
            "openai":    self.openai_api_key,
            "anthropic": self.anthropic_api_key,
            "google":    self.google_api_key,
            "groq":      self.groq_api_key,
        }
        secret = key_map.get(provider)
        if secret is None:
            raise ValueError(f"API key for '{provider}' is not configured")
        return secret.get_secret_value()

    def validate_keys(self) -> list[str]:
        """Возвращает список доступных провайдеров."""
        available = []
        if self.openai_api_key:    available.append("openai")
        if self.anthropic_api_key: available.append("anthropic")
        if self.google_api_key:    available.append("google")
        if self.groq_api_key:      available.append("groq")
        available.append("ollama")  # локальный, без ключа
        return available

# Синглтон — читаем один раз при старте
settings = LLMSettings()

# Проверка при запуске
available = settings.validate_keys()
print(f"Available providers: {available}")

import openai
from openai import AsyncOpenAI
import httpx

def make_openai_client(
    api_key: str | None = None,
    base_url: str | None = None,          # для OpenAI-совместимых API
    timeout: float = 60.0,
    max_retries: int = 3,
) -> AsyncOpenAI:
    """
    Создаём async OpenAI клиент с правильными настройками.
    base_url позволяет переключиться на Ollama, Groq, LiteLLM и т.д.
    """
    return AsyncOpenAI(
        api_key=api_key or settings.get_key("openai"),
        base_url=base_url,
        timeout=httpx.Timeout(
            connect=10.0,
            read=timeout,
            write=30.0,
            pool=5.0,
        ),
        max_retries=max_retries,           # встроенный exponential backoff
        http_client=httpx.AsyncClient(
            limits=httpx.Limits(
                max_connections=20,
                max_keepalive_connections=10,
            )
        ),
    )

# Синглтон-клиенты — создаём один раз, переиспользуем
_openai_client: AsyncOpenAI | None = None

def get_openai_client() -> AsyncOpenAI:
    global _openai_client
    if _openai_client is None:
        _openai_client = make_openai_client()
    return _openai_client

# ── Базовый запрос ──
async def openai_chat(
    messages: list[dict],
    model: str = "gpt-4o",
    temperature: float = 0.7,
    max_tokens: int = 1024,
    **kwargs,
) -> str:
    client = get_openai_client()
    response = await client.chat.completions.create(
        model=model,
        messages=messages,
        temperature=temperature,
        max_tokens=max_tokens,
        **kwargs,
    )
    return response.choices[0].message.content

# ── Запрос с метаданными ──
from dataclasses import dataclass

@dataclass
class LLMResponse:
    text: str
    model: str
    input_tokens: int
    output_tokens: int
    finish_reason: str

    @property
    def total_tokens(self) -> int:
        return self.input_tokens + self.output_tokens

async def openai_chat_full(
    messages: list[dict],
    model: str = "gpt-4o",
    **kwargs,
) -> LLMResponse:
    client = get_openai_client()
    r = await client.chat.completions.create(
        model=model,
        messages=messages,
        **kwargs,
    )
    choice = r.choices[0]
    return LLMResponse(
        text=choice.message.content,
        model=r.model,
        input_tokens=r.usage.prompt_tokens,
        output_tokens=r.usage.completion_tokens,
        finish_reason=choice.finish_reason,
    )

import anthropic
from anthropic import AsyncAnthropic
import httpx

def make_anthropic_client(
    api_key: str | None = None,
    timeout: float = 60.0,
    max_retries: int = 3,
) -> AsyncAnthropic:
    return AsyncAnthropic(
        api_key=api_key or settings.get_key("anthropic"),
        timeout=httpx.Timeout(
            connect=10.0,
            read=timeout,
            write=30.0,
            pool=5.0,
        ),
        max_retries=max_retries,
        http_client=httpx.AsyncClient(
            limits=httpx.Limits(
                max_connections=20,
                max_keepalive_connections=10,
            )
        ),
    )

_anthropic_client: AsyncAnthropic | None = None

def get_anthropic_client() -> AsyncAnthropic:
    global _anthropic_client
    if _anthropic_client is None:
        _anthropic_client = make_anthropic_client()
    return _anthropic_client

async def anthropic_chat(
    messages: list[dict],
    system: str = "",
    model: str = "claude-opus-4-6",
    temperature: float = 0.7,
    max_tokens: int = 1024,
    **kwargs,
) -> LLMResponse:
    client = get_anthropic_client()
    r = await client.messages.create(
        model=model,
        system=system,
        messages=messages,
        temperature=temperature,
        max_tokens=max_tokens,
        **kwargs,
    )
    return LLMResponse(
        text=r.content[0].text,
        model=r.model,
        input_tokens=r.usage.input_tokens,
        output_tokens=r.usage.output_tokens,
        finish_reason=r.stop_reason,
    )

# Установка (Linux/macOS)
curl -fsSL https://ollama.com/install.sh | sh

# Скачать и запустить модель
ollama pull llama3.2          # 2GB, быстрая
ollama pull mistral           # 4GB, хороший баланс
ollama pull qwen2.5-coder     # специально для кода

# Запустить сервер (по умолчанию порт 11434)
ollama serve

# Проверить доступные модели
ollama list

# Запросить напрямую через curl
curl http://localhost:11434/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "llama3.2",
    "messages": [{"role": "user", "content": "Привет!"}]
  }'

from openai import AsyncOpenAI

# Ollama принимает OpenAI-формат — меняем только base_url
ollama_client = AsyncOpenAI(
    api_key="ollama",                            # любая строка, не проверяется
    base_url="http://localhost:11434/v1",
)

async def ollama_chat(
    prompt: str,
    model: str = "llama3.2",
    system: str = "Ты — полезный ассистент.",
) -> str:
    response = await ollama_client.chat.completions.create(
        model=model,
        messages=[
            {"role": "system", "content": system},
            {"role": "user",   "content": prompt},
        ],
        max_tokens=1024,
    )
    return response.choices[0].message.content

# Получить список доступных моделей
import httpx

async def list_ollama_models(base_url: str = "http://localhost:11434") -> list[str]:
    async with httpx.AsyncClient() as client:
        r = await client.get(f"{base_url}/api/tags")
        r.raise_for_status()
        return [m["name"] for m in r.json()["models"]]

models = await list_ollama_models()
print(models)  # ['llama3.2:latest', 'mistral:latest', ...]

pip install litellm[proxy]

# Конфиг litellm_config.yaml
cat > litellm_config.yaml << 'EOF'
model_list:
  - model_name: gpt-4o
    litellm_params:
      model: openai/gpt-4o
      api_key: os.environ/OPENAI_API_KEY

  - model_name: claude-3-opus
    litellm_params:
      model: anthropic/claude-opus-4-6
      api_key: os.environ/ANTHROPIC_API_KEY

  - model_name: llama3
    litellm_params:
      model: ollama/llama3.2
      api_base: http://localhost:11434

  - model_name: fast
    litellm_params:
      model: groq/llama-3.3-70b-versatile
      api_key: os.environ/GROQ_API_KEY
EOF

# Запуск прокси
litellm --config litellm_config.yaml --port 4000

from openai import AsyncOpenAI

# Подключаемся к LiteLLM proxy как к OpenAI
litellm_client = AsyncOpenAI(
    api_key="sk-any-string",             # LiteLLM принимает любой ключ
    base_url="http://localhost:4000",
)

async def agent_call(
    messages: list[dict],
    model: str = "claude-3-opus",        # маппится в litellm_config
) -> str:
    r = await litellm_client.chat.completions.create(
        model=model,
        messages=messages,
        max_tokens=1024,
    )
    return r.choices[0].message.content

# Переключаемся на другую модель одной строкой
answer_fast    = await agent_call(messages, model="fast")      # Groq
answer_local   = await agent_call(messages, model="llama3")    # Ollama
answer_quality = await agent_call(messages, model="claude-3-opus")  # Anthropic

import asyncio
import logging
import time
from dataclasses import dataclass, field
from typing import AsyncIterator

import anthropic
import openai

logger = logging.getLogger(__name__)

# Стоимость на 1M токенов (в USD, данные на 2025)
TOKEN_PRICES: dict[str, dict[str, float]] = {
    "gpt-4o":            {"input": 2.50,  "output": 10.00},
    "gpt-4o-mini":       {"input": 0.15,  "output": 0.60},
    "claude-opus-4-6":   {"input": 15.00, "output": 75.00},
    "claude-sonnet-4-6": {"input": 3.00,  "output": 15.00},
    "claude-haiku-4-5":  {"input": 0.80,  "output": 4.00},
}

@dataclass
class UsageStats:
    """Накопленная статистика использования."""
    total_requests: int = 0
    total_input_tokens: int = 0
    total_output_tokens: int = 0
    total_cost_usd: float = 0.0
    errors: int = 0

    def add(self, response: LLMResponse) -> None:
        self.total_requests += 1
        self.total_input_tokens  += response.input_tokens
        self.total_output_tokens += response.output_tokens
        prices = TOKEN_PRICES.get(response.model, {"input": 0, "output": 0})
        cost = (
            response.input_tokens  * prices["input"]  / 1_000_000 +
            response.output_tokens * prices["output"] / 1_000_000
        )
        self.total_cost_usd += cost

    def report(self) -> str:
        return (
            f"Requests: {self.total_requests} | "
            f"Tokens: {self.total_input_tokens}↑ {self.total_output_tokens}↓ | "
            f"Cost: ${self.total_cost_usd:.4f}"
        )


class UnifiedLLMClient:
    """
    Единый клиент для работы с любым LLM-провайдером.
    Поддерживает: anthropic, openai, ollama, groq (через openai-совместимый API).
    """

    def __init__(self, settings: LLMSettings):
        self.settings = settings
        self.usage = UsageStats()
        self._anthropic: anthropic.AsyncAnthropic | None = None
        self._openai: openai.AsyncOpenAI | None = None
        self._ollama: openai.AsyncOpenAI | None = None
        self._groq: openai.AsyncOpenAI | None = None

    def _get_anthropic(self) -> anthropic.AsyncAnthropic:
        if self._anthropic is None:
            self._anthropic = anthropic.AsyncAnthropic(
                api_key=self.settings.get_key("anthropic"),
                max_retries=3,
            )
        return self._anthropic

    def _get_openai(self) -> openai.AsyncOpenAI:
        if self._openai is None:
            self._openai = openai.AsyncOpenAI(
                api_key=self.settings.get_key("openai"),
                max_retries=3,
            )
        return self._openai

    def _get_ollama(self) -> openai.AsyncOpenAI:
        if self._ollama is None:
            self._ollama = openai.AsyncOpenAI(
                api_key="ollama",
                base_url=f"{self.settings.ollama_base_url}/v1",
                max_retries=1,
            )
        return self._ollama

    def _get_groq(self) -> openai.AsyncOpenAI:
        if self._groq is None:
            self._groq = openai.AsyncOpenAI(
                api_key=self.settings.get_key("groq"),
                base_url="https://api.groq.com/openai/v1",
                max_retries=3,
            )
        return self._groq

    def _detect_provider(self, model: str) -> str:
        """Определяем провайдера по имени модели."""
        if model.startswith("claude"):   return "anthropic"
        if model.startswith("gpt") or model.startswith("o1") or model.startswith("o3"):
            return "openai"
        if model.startswith("llama") or model.startswith("mistral") or model.startswith("qwen"):
            return "ollama"
        if "groq" in model:              return "groq"
        return self.settings.default_provider

    async def chat(
        self,
        messages: list[dict],
        model: str | None = None,
        system: str = "",
        temperature: float = 0.7,
        max_tokens: int | None = None,
        provider: str | None = None,
    ) -> LLMResponse:
        model = model or self.settings.default_model
        max_tokens = max_tokens or self.settings.max_tokens
        provider = provider or self._detect_provider(model)

        t0 = time.monotonic()
        try:
            if provider == "anthropic":
                response = await self._chat_anthropic(messages, model, system, temperature, max_tokens)
            else:
                # OpenAI, Ollama, Groq — все через OpenAI-совместимый интерфейс
                response = await self._chat_openai(messages, model, system, temperature, max_tokens, provider)

            self.usage.add(response)
            elapsed = time.monotonic() - t0
            logger.debug(
                "LLM call: model=%s tokens=%d/%d elapsed=%.2fs",
                response.model, response.input_tokens, response.output_tokens, elapsed,
            )
            return response

        except Exception as e:
            self.usage.errors += 1
            logger.error("LLM error (provider=%s model=%s): %s", provider, model, e)
            raise

    async def _chat_anthropic(
        self,
        messages: list[dict],
        model: str,
        system: str,
        temperature: float,
        max_tokens: int,
    ) -> LLMResponse:
        r = await self._get_anthropic().messages.create(
            model=model,
            system=system,
            messages=messages,
            temperature=temperature,
            max_tokens=max_tokens,
        )
        return LLMResponse(
            text=r.content[0].text,
            model=r.model,
            input_tokens=r.usage.input_tokens,
            output_tokens=r.usage.output_tokens,
            finish_reason=r.stop_reason,
        )

    async def _chat_openai(
        self,
        messages: list[dict],
        model: str,
        system: str,
        temperature: float,
        max_tokens: int,
        provider: str,
    ) -> LLMResponse:
        client = {
            "openai": self._get_openai,
            "ollama": self._get_ollama,
            "groq":   self._get_groq,
        }[provider]()

        full_messages = (
            [{"role": "system", "content": system}] + messages
            if system else messages
        )
        r = await client.chat.completions.create(
            model=model,
            messages=full_messages,
            temperature=temperature,
            max_tokens=max_tokens,
        )
        return LLMResponse(
            text=r.choices[0].message.content,
            model=r.model,
            input_tokens=r.usage.prompt_tokens,
            output_tokens=r.usage.completion_tokens,
            finish_reason=r.choices[0].finish_reason,
        )

    async def stream(
        self,
        messages: list[dict],
        model: str | None = None,
        system: str = "",
        **kwargs,
    ) -> AsyncIterator[str]:
        model = model or self.settings.default_model
        provider = self._detect_provider(model)

        if provider == "anthropic":
            client = self._get_anthropic()
            async with client.messages.stream(
                model=model, system=system, messages=messages,
                max_tokens=kwargs.get("max_tokens", self.settings.max_tokens),
            ) as s:
                async for chunk in s.text_stream:
                    yield chunk
        else:
            oai_client = {
                "openai": self._get_openai,
                "ollama": self._get_ollama,
                "groq":   self._get_groq,
            }[provider]()
            msgs = ([{"role": "system", "content": system}] + messages) if system else messages
            async with await oai_client.chat.completions.create(
                model=model, messages=msgs, stream=True,
                max_tokens=kwargs.get("max_tokens", self.settings.max_tokens),
            ) as s:
                async for chunk in s:
                    delta = chunk.choices[0].delta.content
                    if delta:
                        yield delta

    def print_usage(self) -> None:
        print(self.usage.report())


# ── Использование ──
llm = UnifiedLLMClient(settings)

# Anthropic
r = await llm.chat(
    system="Ты — ассистент.",
    messages=[{"role": "user", "content": "Привет!"}],
    model="claude-opus-4-6",
)

# OpenAI — без изменения кода
r = await llm.chat(
    system="Ты — ассистент.",
    messages=[{"role": "user", "content": "Привет!"}],
    model="gpt-4o",
)

# Ollama локально
r = await llm.chat(
    system="Ты — ассистент.",
    messages=[{"role": "user", "content": "Привет!"}],
    model="llama3.2",
)

llm.print_usage()
# → Requests: 3 | Tokens: 312↑ 156↓ | Cost: $0.0048

import asyncio
import time
from collections import deque

class RateLimiter:
    """
    Токен-ведро (token bucket) для rate limiting.
    Отслеживает RPM (requests per minute) и TPM (tokens per minute).
    """

    def __init__(self, rpm: int, tpm: int = 1_000_000):
        self.rpm = rpm
        self.tpm = tpm
        self._request_timestamps: deque[float] = deque()
        self._token_timestamps: deque[tuple[float, int]] = deque()
        self._lock = asyncio.Lock()

    async def acquire(self, estimated_tokens: int = 1000) -> None:
        """Ждёт, пока rate limit позволит сделать запрос."""
        async with self._lock:
            now = time.monotonic()
            window = 60.0

            # Очищаем старые записи (старше 1 минуты)
            while self._request_timestamps and now - self._request_timestamps[0] > window:
                self._request_timestamps.popleft()
            while self._token_timestamps and now - self._token_timestamps[0][0] > window:
                self._token_timestamps.popleft()

            # Считаем текущее использование
            current_rpm = len(self._request_timestamps)
            current_tpm = sum(t for _, t in self._token_timestamps)

            # Ждём если превышен лимит
            if current_rpm >= self.rpm:
                oldest = self._request_timestamps[0]
                wait = window - (now - oldest) + 0.1
                await asyncio.sleep(wait)

            elif current_tpm + estimated_tokens > self.tpm:
                oldest_tok = self._token_timestamps[0][0]
                wait = window - (now - oldest_tok) + 0.1
                await asyncio.sleep(wait)

            # Регистрируем запрос
            now = time.monotonic()
            self._request_timestamps.append(now)
            self._token_timestamps.append((now, estimated_tokens))

# Rate limiter для каждого провайдера
anthropic_limiter = RateLimiter(rpm=50, tpm=100_000)
openai_limiter    = RateLimiter(rpm=500, tpm=800_000)
groq_limiter      = RateLimiter(rpm=30, tpm=20_000)

# Использование
async def safe_anthropic_chat(messages: list[dict], **kwargs) -> LLMResponse:
    await anthropic_limiter.acquire(estimated_tokens=kwargs.get("max_tokens", 1000))
    return await llm._chat_anthropic(messages, **kwargs)

# Параллельные запросы с rate limiting
async def batch_requests(prompts: list[str]) -> list[str]:
    semaphore = asyncio.Semaphore(5)  # максимум 5 параллельно

    async def one(prompt: str) -> str:
        async with semaphore:
            await anthropic_limiter.acquire()
            r = await llm.chat(
                system="Ты — ассистент.",
                messages=[{"role": "user", "content": prompt}],
            )
            return r.text

    return await asyncio.gather(*[one(p) for p in prompts])

import asyncio
from datetime import datetime, date
from collections import defaultdict

class CostTracker:
    """Отслеживает расходы по провайдерам, моделям и дням."""

    # Цены на 1M токенов (USD)
    PRICES = {
        "claude-opus-4-6":   {"input": 15.0,  "output": 75.0},
        "claude-sonnet-4-6": {"input": 3.0,   "output": 15.0},
        "claude-haiku-4-5":  {"input": 0.8,   "output": 4.0},
        "gpt-4o":            {"input": 2.5,   "output": 10.0},
        "gpt-4o-mini":       {"input": 0.15,  "output": 0.6},
        "llama-3.3-70b":     {"input": 0.59,  "output": 0.79},
    }

    def __init__(self, daily_budget_usd: float = 10.0):
        self.daily_budget = daily_budget_usd
        self._daily: dict[str, float] = defaultdict(float)  # date → cost
        self._by_model: dict[str, float] = defaultdict(float)
        self._lock = asyncio.Lock()
        self._alert_callbacks: list = []

    def on_alert(self, callback):
        """Регистрация callback при превышении бюджета."""
        self._alert_callbacks.append(callback)
        return callback

    async def record(self, model: str, input_tokens: int, output_tokens: int) -> float:
        prices = self.PRICES.get(model, {"input": 0, "output": 0})
        cost = (
            input_tokens  * prices["input"]  / 1_000_000 +
            output_tokens * prices["output"] / 1_000_000
        )

        async with self._lock:
            today = str(date.today())
            self._daily[today] += cost
            self._by_model[model] += cost

            # Проверка бюджета
            if self._daily[today] >= self.daily_budget:
                for cb in self._alert_callbacks:
                    await cb(self._daily[today], self.daily_budget)

        return cost

    @property
    def today_cost(self) -> float:
        return self._daily.get(str(date.today()), 0.0)

    def report(self) -> str:
        lines = [
            f"Today: ${self.today_cost:.4f} / ${self.daily_budget:.2f}",
            "By model:",
        ]
        for model, cost in sorted(self._by_model.items(), key=lambda x: -x[1]):
            lines.append(f"  {model}: ${cost:.4f}")
        return "\n".join(lines)


# ── Интеграция с UnifiedLLMClient ──
tracker = CostTracker(daily_budget_usd=5.0)

@tracker.on_alert
async def budget_alert(spent: float, budget: float):
    print(f"⚠️  BUDGET ALERT: ${spent:.2f} / ${budget:.2f} today!")
    # В продакшне: отправить в Slack/Telegram

# Модифицируем UnifiedLLMClient.chat() для трекинга
original_chat = llm.chat

async def tracked_chat(*args, **kwargs) -> LLMResponse:
    r = await original_chat(*args, **kwargs)
    await tracker.record(r.model, r.input_tokens, r.output_tokens)
    return r

llm.chat = tracked_chat  # monkey-patch для демо

# После работы
print(tracker.report())

import anthropic
import openai
import logging

logger = logging.getLogger(__name__)

async def chat_with_fallback(
    messages: list[dict],
    system: str = "",
    primary_model: str = "claude-opus-4-6",
    fallback_chains: list[str] = ("gpt-4o", "gpt-4o-mini", "llama3.2"),
) -> LLMResponse:
    """
    Пробуем primary модель, при ошибке идём по fallback chain.
    """
    models_to_try = [primary_model, *fallback_chains]
    last_error = None

    for model in models_to_try:
        try:
            logger.info("Trying model: %s", model)
            result = await llm.chat(
                messages=messages,
                system=system,
                model=model,
            )
            if model != primary_model:
                logger.warning("Used fallback model: %s", model)
            return result

        except (
            anthropic.RateLimitError,
            anthropic.InternalServerError,
            openai.RateLimitError,
            openai.InternalServerError,
        ) as e:
            logger.warning("Model %s failed: %s. Trying fallback...", model, e)
            last_error = e
            continue

        except Exception as e:
            # Не ретраить: ошибки валидации, auth и т.д.
            raise

    raise RuntimeError(f"All models failed. Last error: {last_error}") from last_error

# Использование
response = await chat_with_fallback(
    system="Ты — ассистент.",
    messages=[{"role": "user", "content": "Привет!"}],
)
print(response.text)