Ensemble Architecture

                    Client Request
                         │
                    ┌────▼────┐
                    │  Auth   │  API key validation (SHA256 hash lookup in sync.Map)
                    └────┬────┘
                         │
                    ┌────▼────────────┐
                    │ Request Parser  │  Unified InferenceRequest from any provider format
                    └────┬────────────┘
                         │
                    ┌────▼────────────┐
                    │ Parameter       │  YAML-driven validation, provider-specific constraints
                    │ Validator       │
                    └────┬────────────┘
                         │
                    ┌────▼────────────┐
                    │    Router       │  Cache affinity → Rate limits → Cost optimization
                    └────┬────────────┘
                         │
                    ┌────▼────────────┐
                    │ Provider Client │  HTTP/2 connection pool (100 conns/host)
                    └────┬────────────┘
                         │
                    ┌────▼────────────┐
                    │ Response        │  Stream parsing → Unified EventBlocks → Cost calc
                    │ Handler         │
                    └────┬────────────┘
                         │
                    ┌────▼──────────────────┐
                    │ Persistence / Logging │  S3 response storage, async batched logs
                    └───────────────────────┘

ensemble/
├── cmd/ensemble/         # Main entry point (CLI: serve, migrate, etc.)
├── internal/
│   ├── config/           # YAML config structures, hot-reload, validation cache
│   ├── server/           # HTTP server, route handlers, middleware
│   ├── proxy/            # Provider request/response proxying
│   ├── router/           # Routing engine (cache, rate, cost)
│   ├── ratelimit/        # Local-first rate management with Redis sync
│   ├── storage/          # EmbeddedStore (SQLite + Redis), API keys, customers
│   ├── providers/        # Provider adapters (Anthropic, OpenAI, Gemini, xAI, OpenRouter)
│   ├── streaming/        # SSE/WebSocket stream handling, stall detection
│   ├── batch/            # Batch processing (Anthropic batch API)
│   ├── otel/             # OpenTelemetry tracing and metrics
│   └── async_inference/  # Response persistence (S3), status tracking
├── pkg/
│   ├── types/            # Shared types (InferenceRequest, EventBlock, ToolDefinition, etc.)
│   └── logging/          # Async batched logger (65K ring buffer)
├── client-go/            # Go client library
├── client-python/        # Python client library
├── client-typescript/    # TypeScript client library
└── config/               # Example YAML configurations

type InferenceRequest struct {
    Model           string          // Target model name
    Messages        []Message       // Conversation history
    SessionID       string          // For cache affinity routing
    Tools           []ToolDefinition // Function calling definitions
    MaxTokens       int
    Temperature     *float64
    TopP            *float64
    StopSequences   []string
    Stream          bool            // SSE streaming mode
    ProviderConfig  map[string]interface{} // Per-request provider overrides
}

type EventBlock struct {
    Type           string  // "text", "tool_call", "thinking", "error"
    Text           string  // For text blocks
    ToolCall       *ToolCall
    ThinkingContent string
}

type InferenceResponse struct {
    ID                  string
    Model               string
    Provider            string
    Blocks              []EventBlock    // The canonical representation
    InputTokens         int64
    OutputTokens        int64
    CachedPromptTokens  int64
    CacheCreationTokens int64
    ReasoningTokens     int64
    Cost                decimal.Decimal
    ProcessingTime      time.Duration
    FinishReason        string
    PerformanceMetrics  *PerformanceMetrics
    RateLimitInfo       *RateLimitInfo
}