Initial commit: Luminary AI Gateway

OpenAI-compatible AI gateway with Vue admin UI, multi-provider egress,
ingress key governance, monitoring, and security controls.

Co-authored-by: Cursor <cursoragent@cursor.com>

internal/gateway/errors.go

@@ -0,0 +1,11 @@
+package gateway
+
+import "errors"
+
+var (
+	ErrModelNotAllowed   = errors.New("model not allowed")
+	ErrBudgetExceeded    = errors.New("budget exceeded")
+	ErrRateLimitExceeded = errors.New("rate limit exceeded")
+	ErrInvalidJSON       = errors.New("invalid json")
+	ErrModelRequired     = errors.New("model is required")
+)

internal/gateway/failover.go

@@ -0,0 +1,160 @@
+package gateway
+
+import (
+	"context"
+	"fmt"
+	"sort"
+	"time"
+
+	"github.com/rose_cat707/luminary/internal/balancer"
+	"github.com/rose_cat707/luminary/internal/limiter"
+	"github.com/rose_cat707/luminary/internal/model"
+	"github.com/rose_cat707/luminary/internal/provider"
+	"github.com/rose_cat707/luminary/internal/store/cache"
+)
+
+type attempt struct {
+	provider    *model.Provider
+	key         *model.ProviderKey
+	apiKey      string
+}
+
+func (s *Service) buildAttempts(providers []model.Provider, modelName string, route *routeTarget) ([]attempt, error) {
+	var attempts []attempt
+	var decryptFailures int
+	var eligibleKeys int
+	for i := range providers {
+		p := providers[i]
+		keys := s.listEligibleKeys(p.ID, modelName, route)
+		eligibleKeys += len(keys)
+		for _, k := range keys {
+			plain, err := s.DecryptAPIKey(k.APIKeyEnc)
+			if err != nil {
+				decryptFailures++
+				continue
+			}
+			attempts = append(attempts, attempt{provider: &p, key: &k, apiKey: plain})
+		}
+	}
+	if len(attempts) == 0 {
+		if decryptFailures > 0 {
+			return nil, fmt.Errorf("no available provider keys: %d key(s) failed to decrypt (check encryption_key in system settings)", decryptFailures)
+		}
+		if eligibleKeys == 0 {
+			return nil, fmt.Errorf("no available provider keys: none match model %q or are disabled/rate-limited", modelName)
+		}
+		return nil, fmt.Errorf("no available provider keys")
+	}
+	sort.SliceStable(attempts, func(i, j int) bool {
+		return attempts[i].key.Weight > attempts[j].key.Weight
+	})
+	return attempts, nil
+}
+
+func (s *Service) listEligibleKeys(providerID uint, modelName string, route *routeTarget) []model.ProviderKey {
+	if route != nil && route.ProviderKeyID > 0 && route.ProviderID == providerID {
+		if k, ok := s.cache.GetProviderKey(route.ProviderKeyID); ok && k.Enabled && limiter.IsProviderKeyAvailable(k) {
+			return []model.ProviderKey{k}
+		}
+	}
+	var candidates []balancer.KeyCandidate
+	for _, k := range s.cache.ListProviderKeys(providerID) {
+		if !limiter.IsProviderKeyAvailable(k) {
+			continue
+		}
+		allowed := cache.ParseJSONList(k.ModelsJSON)
+		if modelName != "" && !cache.MatchesList(allowed, "*") {
+			upstream := s.resolveUpstreamModelID(providerID, modelName)
+			if !cache.MatchesList(allowed, modelName) && !cache.MatchesList(allowed, upstream) {
+				continue
+			}
+		}
+		candidates = append(candidates, balancer.KeyCandidate{Key: k, Weight: k.Weight})
+	}
+	// collect unique keys by weighted random sampling order
+	seen := map[uint]bool{}
+	var out []model.ProviderKey
+	for len(out) < len(candidates) {
+		selected := balancer.SelectWeighted(candidates)
+		if selected == nil {
+			break
+		}
+		if seen[selected.ID] {
+			// remove from candidates
+			filtered := candidates[:0]
+			for _, c := range candidates {
+				if c.Key.ID != selected.ID {
+					filtered = append(filtered, c)
+				}
+			}
+			candidates = filtered
+			if len(candidates) == 0 {
+				break
+			}
+			continue
+		}
+		seen[selected.ID] = true
+		out = append(out, *selected)
+		filtered := candidates[:0]
+		for _, c := range candidates {
+			if c.Key.ID != selected.ID {
+				filtered = append(filtered, c)
+			}
+		}
+		candidates = filtered
+	}
+	return out
+}
+
+func (s *Service) forwardWithFailover(ctx context.Context, attempts []attempt, endpoint provider.EndpointType, body []byte) (*provider.ChatResponse, *attempt, error) {
+	var lastErr error
+	var lastAtt *attempt
+	var lastResp *provider.ChatResponse
+	maxRetries := s.settings.GatewayMaxRetries()
+	if maxRetries < 1 {
+		maxRetries = 1
+	}
+	backoffMs := s.settings.GatewayRetryBackoffMs()
+	for _, att := range attempts {
+		if !s.canServeEndpoint(att.provider, endpoint) {
+			continue
+		}
+		for try := 0; try < maxRetries; try++ {
+			if try > 0 {
+				time.Sleep(time.Duration(backoffMs) * time.Millisecond)
+			}
+			resp, err := s.forward(ctx, att.provider, att.apiKey, endpoint, body)
+			cur := att
+			lastAtt = &cur
+			if resp != nil {
+				lastResp = resp
+			}
+			if err != nil {
+				lastErr = err
+				continue
+			}
+			if resp == nil {
+				lastErr = fmt.Errorf("empty upstream response")
+				continue
+			}
+			if resp.StatusCode < 400 {
+				return resp, &att, nil
+			}
+			lastErr = fmt.Errorf("%s", describeForwardError(nil, resp))
+			if passThroughUpstreamStatus(resp.StatusCode) {
+				return resp, &att, nil
+			}
+			if tryNextKeyOnUpstreamStatus(resp.StatusCode) {
+				break // try next provider key
+			}
+			if retryableUpstreamStatus(resp.StatusCode) {
+				continue // retry same key
+			}
+			return resp, &att, nil
+		}
+	}
+	if lastErr == nil {
+		lastErr = fmt.Errorf("all providers failed")
+	}
+	return lastResp, lastAtt, lastErr
+}

internal/gateway/health_check.go

@@ -0,0 +1,190 @@
+package gateway
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"time"
+
+	"github.com/rose_cat707/luminary/internal/model"
+	"github.com/rose_cat707/luminary/internal/provider"
+)
+
+func (s *Service) healthCheckEndpoints(p *model.Provider) []provider.EndpointType {
+	var out []provider.EndpointType
+	if s.providerAllowsEndpoint(p, provider.EndpointListModels) {
+		out = append(out, provider.EndpointListModels)
+	}
+	switch p.Category {
+	case model.CategoryEmbedding:
+		if s.providerAllowsEndpoint(p, provider.EndpointEmbeddings) {
+			out = append(out, provider.EndpointEmbeddings)
+		}
+	case model.CategoryRerank:
+		if s.providerAllowsEndpoint(p, provider.EndpointRerank) {
+			out = append(out, provider.EndpointRerank)
+		}
+	}
+	return out
+}
+
+func (s *Service) runHealthChecks(
+	ctx context.Context,
+	p *model.Provider,
+	keys []model.ProviderKey,
+	cfg provider.ProviderConfig,
+) (model.ProviderStatus, int64, string) {
+	endpoints := s.healthCheckEndpoints(p)
+	if len(endpoints) == 0 {
+		return model.ProviderUnhealthy, 0, "no health check endpoint enabled"
+	}
+	start := time.Now()
+	var lastErr string
+	for _, ep := range endpoints {
+		status, _, msg := s.runHealthCheck(ctx, p, keys, cfg, ep)
+		if status == model.ProviderHealthy {
+			return model.ProviderHealthy, time.Since(start).Milliseconds(), ""
+		}
+		if msg != "" {
+			lastErr = msg
+		}
+	}
+	if lastErr == "" {
+		lastErr = "health check failed"
+	}
+	return model.ProviderUnhealthy, time.Since(start).Milliseconds(), lastErr
+}
+
+func (s *Service) runHealthCheck(
+	ctx context.Context,
+	p *model.Provider,
+	keys []model.ProviderKey,
+	cfg provider.ProviderConfig,
+	endpoint provider.EndpointType,
+) (model.ProviderStatus, int64, string) {
+	start := time.Now()
+	latency := func() int64 { return time.Since(start).Milliseconds() }
+
+	switch endpoint {
+	case provider.EndpointListModels:
+		if len(keys) == 0 && p.Type == model.ProviderOllama {
+			if err := s.openai.HealthCheckWithConfig(ctx, "", cfg); err == nil {
+				return model.ProviderHealthy, latency(), ""
+			}
+			return model.ProviderUnhealthy, latency(), "health check failed"
+		}
+		for _, k := range keys {
+			if !k.Enabled {
+				continue
+			}
+			apiKey, err := s.DecryptAPIKey(k.APIKeyEnc)
+			if err != nil {
+				continue
+			}
+			var checkErr error
+			if p.Type == model.ProviderAnthropic {
+				checkErr = s.anthropic.HealthCheckWithConfig(ctx, apiKey, cfg)
+			} else {
+				checkErr = s.openai.HealthCheckWithConfig(ctx, apiKey, cfg)
+			}
+			if checkErr == nil {
+				return model.ProviderHealthy, latency(), ""
+			}
+		}
+		return model.ProviderUnhealthy, latency(), "health check failed"
+
+	case provider.EndpointEmbeddings, provider.EndpointRerank:
+		body, err := healthProbeBody(s, p, endpoint)
+		if err != nil {
+			return model.ProviderUnhealthy, latency(), err.Error()
+		}
+		if len(keys) == 0 && p.Type == model.ProviderOllama {
+			ok, msg := s.probeEndpoint(ctx, "", cfg, endpoint, body)
+			if ok {
+				return model.ProviderHealthy, latency(), ""
+			}
+			return model.ProviderUnhealthy, latency(), msg
+		}
+		var lastErr string
+		for _, k := range keys {
+			if !k.Enabled {
+				continue
+			}
+			apiKey, err := s.DecryptAPIKey(k.APIKeyEnc)
+			if err != nil {
+				continue
+			}
+			if ok, msg := s.probeEndpoint(ctx, apiKey, cfg, endpoint, body); ok {
+				return model.ProviderHealthy, latency(), ""
+			} else if msg != "" {
+				lastErr = msg
+			}
+		}
+		if lastErr == "" {
+			lastErr = "health check failed"
+		}
+		return model.ProviderUnhealthy, latency(), lastErr
+	default:
+		return model.ProviderUnhealthy, latency(), "unsupported health check endpoint"
+	}
+}
+
+func (s *Service) probeEndpoint(
+	ctx context.Context,
+	apiKey string,
+	cfg provider.ProviderConfig,
+	endpoint provider.EndpointType,
+	body []byte,
+) (bool, string) {
+	resp, err := s.openai.ForwardWithConfig(ctx, apiKey, cfg, endpoint, body)
+	if err != nil {
+		return false, err.Error()
+	}
+	if resp == nil {
+		return false, "empty response"
+	}
+	if isHealthCheckHTTPStatus(resp.StatusCode) {
+		return true, ""
+	}
+	return false, fmt.Sprintf("%s: HTTP %d", endpoint, resp.StatusCode)
+}
+
+func healthProbeBody(s *Service, p *model.Provider, endpoint provider.EndpointType) ([]byte, error) {
+	modelID := probeModelID(s, p.ID, endpoint)
+	switch endpoint {
+	case provider.EndpointEmbeddings:
+		return json.Marshal(map[string]string{"model": modelID, "input": "."})
+	case provider.EndpointRerank:
+		return json.Marshal(map[string]any{
+			"model":     modelID,
+			"query":     "ping",
+			"documents": []string{"ping"},
+		})
+	default:
+		return nil, fmt.Errorf("unsupported probe endpoint %s", endpoint)
+	}
+}
+
+func probeModelID(s *Service, providerID uint, endpoint provider.EndpointType) string {
+	for _, m := range s.cache.GetModels(providerID) {
+		if m.Enabled && m.ModelID != "" {
+			return m.ModelID
+		}
+	}
+	switch endpoint {
+	case provider.EndpointEmbeddings:
+		return "text-embedding-3-small"
+	case provider.EndpointRerank:
+		return "rerank-multilingual-v3.0"
+	default:
+		return "health-check"
+	}
+}
+
+func isHealthCheckHTTPStatus(code int) bool {
+	if code >= 200 && code < 300 {
+		return true
+	}
+	// Upstream reachable but rejected payload/model — still counts as healthy.
+	return code == 400 || code == 422
+}

internal/gateway/health_check_test.go

@@ -0,0 +1,51 @@
+package gateway
+
+import (
+	"testing"
+
+	"github.com/rose_cat707/luminary/internal/model"
+	"github.com/rose_cat707/luminary/internal/provider"
+)
+
+func TestIsHealthCheckHTTPStatus(t *testing.T) {
+	cases := map[int]bool{
+		200: true,
+		204: true,
+		400: true,
+		422: true,
+		401: false,
+		404: false,
+		500: false,
+	}
+	for code, want := range cases {
+		if got := isHealthCheckHTTPStatus(code); got != want {
+			t.Fatalf("code %d: got %v want %v", code, got, want)
+		}
+	}
+}
+
+func TestHealthCheckEndpointsEmbedding(t *testing.T) {
+	s := &Service{}
+	p := &model.Provider{
+		Category:             model.CategoryEmbedding,
+		Type:                 model.ProviderOpenAI,
+		AllowedEndpointsJSON: `["embeddings"]`,
+	}
+	eps := s.healthCheckEndpoints(p)
+	if len(eps) != 1 || eps[0] != provider.EndpointEmbeddings {
+		t.Fatalf("got %v", eps)
+	}
+}
+
+func TestHealthCheckEndpointsEmbeddingWithListModels(t *testing.T) {
+	s := &Service{}
+	p := &model.Provider{
+		Category:             model.CategoryEmbedding,
+		Type:                 model.ProviderOpenAI,
+		AllowedEndpointsJSON: `["list_models","embeddings"]`,
+	}
+	eps := s.healthCheckEndpoints(p)
+	if len(eps) != 2 || eps[0] != provider.EndpointListModels || eps[1] != provider.EndpointEmbeddings {
+		t.Fatalf("got %v", eps)
+	}
+}

internal/gateway/logbody.go

@@ -0,0 +1,90 @@
+package gateway
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+
+	"github.com/rose_cat707/luminary/internal/provider"
+)
+
+const maxLogBodyBytes = 16384
+
+func truncateLogBody(b []byte) string {
+	if len(b) == 0 {
+		return ""
+	}
+	if len(b) <= maxLogBodyBytes {
+		return string(b)
+	}
+	return string(b[:maxLogBodyBytes]) + "\n...(truncated)"
+}
+
+func extractModelName(body []byte) string {
+	if len(body) == 0 {
+		return ""
+	}
+	var payload struct {
+		Model string `json:"model"`
+	}
+	if err := json.Unmarshal(body, &payload); err != nil {
+		return ""
+	}
+	return payload.Model
+}
+
+func describeForwardError(err error, resp *provider.ChatResponse) string {
+	if err != nil {
+		return err.Error()
+	}
+	if resp == nil {
+		return "unknown error"
+	}
+	if resp.StatusCode < 400 {
+		return ""
+	}
+	if msg := extractAPIErrorMessage(resp.Body); msg != "" {
+		return fmt.Sprintf("status %d: %s", resp.StatusCode, msg)
+	}
+	if len(resp.Body) > 0 {
+		return fmt.Sprintf("status %d: %s", resp.StatusCode, truncateLogBody(resp.Body))
+	}
+	return fmt.Sprintf("status %d", resp.StatusCode)
+}
+
+func extractAPIErrorMessage(body []byte) string {
+	if len(body) == 0 {
+		return ""
+	}
+	var openAI struct {
+		Error struct {
+			Message string `json:"message"`
+			Type    string `json:"type"`
+		} `json:"error"`
+	}
+	if err := json.Unmarshal(body, &openAI); err == nil {
+		if openAI.Error.Message != "" {
+			if openAI.Error.Type != "" {
+				return openAI.Error.Type + ": " + openAI.Error.Message
+			}
+			return openAI.Error.Message
+		}
+	}
+	var simple struct {
+		Message string `json:"message"`
+		Error   string `json:"error"`
+	}
+	if err := json.Unmarshal(body, &simple); err == nil {
+		if simple.Message != "" {
+			return simple.Message
+		}
+		if simple.Error != "" {
+			return simple.Error
+		}
+	}
+	s := strings.TrimSpace(string(body))
+	if len(s) > 240 {
+		return s[:240] + "..."
+	}
+	return s
+}

internal/gateway/logbody_test.go

@@ -0,0 +1,25 @@
+package gateway
+
+import (
+	"testing"
+
+	"github.com/rose_cat707/luminary/internal/provider"
+)
+
+func TestExtractAPIErrorMessage_OpenAI(t *testing.T) {
+	body := []byte(`{"error":{"message":"Invalid API key","type":"invalid_request_error"}}`)
+	got := extractAPIErrorMessage(body)
+	if got != "invalid_request_error: Invalid API key" {
+		t.Fatalf("got %q", got)
+	}
+}
+
+func TestDescribeForwardError_WithBody(t *testing.T) {
+	got := describeForwardError(nil, &provider.ChatResponse{
+		StatusCode: 400,
+		Body:       []byte(`{"error":{"message":"model not found"}}`),
+	})
+	if got != "status 400: model not found" {
+		t.Fatalf("got %q", got)
+	}
+}

internal/gateway/model_resolve.go

@@ -0,0 +1,53 @@
+package gateway
+
+import (
+	"encoding/json"
+
+	"github.com/rose_cat707/luminary/internal/model"
+)
+
+// IngressModelID returns the model identifier exposed to ingress clients.
+func IngressModelID(m model.ModelEntry) string {
+	return m.IngressID()
+}
+
+func modelEntryMatches(m model.ModelEntry, name string) bool {
+	return m.MatchesIngressName(name)
+}
+
+func (s *Service) resolveUpstreamModelID(providerID uint, ingressModel string) string {
+	if ingressModel == "" {
+		return ""
+	}
+	for _, m := range s.cache.GetModels(providerID) {
+		if modelEntryMatches(m, ingressModel) {
+			return m.ModelID
+		}
+	}
+	return ingressModel
+}
+
+func rewriteModelInBody(body []byte, upstreamModel string) ([]byte, error) {
+	if upstreamModel == "" || len(body) == 0 {
+		return body, nil
+	}
+	var payload map[string]any
+	if err := json.Unmarshal(body, &payload); err != nil {
+		return body, nil
+	}
+	current, _ := payload["model"].(string)
+	if current == "" || current == upstreamModel {
+		return body, nil
+	}
+	payload["model"] = upstreamModel
+	return json.Marshal(payload)
+}
+
+func (s *Service) rewriteBodyForProvider(providerID uint, body []byte) ([]byte, error) {
+	ingressModel := extractModelName(body)
+	if ingressModel == "" {
+		return body, nil
+	}
+	upstream := s.resolveUpstreamModelID(providerID, ingressModel)
+	return rewriteModelInBody(body, upstream)
+}

internal/gateway/model_resolve_test.go

@@ -0,0 +1,32 @@
+package gateway
+
+import (
+	"encoding/json"
+	"testing"
+
+	"github.com/rose_cat707/luminary/internal/model"
+)
+
+func TestIngressModelID(t *testing.T) {
+	if got := IngressModelID(model.ModelEntry{ModelID: "text-embedding-3-small", Alias: "my-embed"}); got != "my-embed" {
+		t.Fatalf("got %q", got)
+	}
+	if got := (model.ModelEntry{ModelID: "gpt-4o"}).IngressID(); got != "gpt-4o" {
+		t.Fatalf("got %q", got)
+	}
+}
+
+func TestRewriteModelInBody(t *testing.T) {
+	body := []byte(`{"model":"my-embed","input":"hi"}`)
+	out, err := rewriteModelInBody(body, "text-embedding-3-small")
+	if err != nil {
+		t.Fatal(err)
+	}
+	var payload map[string]string
+	if err := json.Unmarshal(out, &payload); err != nil {
+		t.Fatal(err)
+	}
+	if payload["model"] != "text-embedding-3-small" {
+		t.Fatalf("got %q", payload["model"])
+	}
+}

internal/gateway/routing.go

@@ -0,0 +1,104 @@
+package gateway
+
+import (
+	"strings"
+
+	"github.com/rose_cat707/luminary/internal/model"
+	"github.com/rose_cat707/luminary/internal/store/cache"
+)
+
+type routeTarget struct {
+	ProviderID    uint
+	ProviderKeyID uint // 0 = auto load balance
+}
+
+func (s *Service) resolveRoute(ingressID uint, modelName string) *routeTarget {
+	rules := s.cache.ListRoutingRules(ingressID)
+	var best *model.RoutingRule
+	for i := range rules {
+		r := rules[i]
+		if !r.Enabled {
+			continue
+		}
+		if !matchPattern(r.ModelPattern, modelName) {
+			continue
+		}
+		if best == nil || r.Priority > best.Priority {
+			best = &r
+		}
+	}
+	if best == nil {
+		return nil
+	}
+	return &routeTarget{ProviderID: best.ProviderID, ProviderKeyID: best.ProviderKeyID}
+}
+
+func matchPattern(pattern, model string) bool {
+	if pattern == "" || pattern == "*" {
+		return true
+	}
+	if strings.HasSuffix(pattern, "*") {
+		return strings.HasPrefix(model, strings.TrimSuffix(pattern, "*"))
+	}
+	return pattern == model
+}
+
+func (s *Service) findProvidersForModel(modelName string, allowed []string, category model.ProviderCategory, route *routeTarget) ([]model.Provider, error) {
+	if route != nil {
+		if p, ok := s.cache.GetProvider(route.ProviderID); ok && p.Category == category && p.Enabled {
+			if cache.MatchesList(allowed, p.Name) || cache.MatchesList(allowed, string(p.Type)) || cache.MatchesList(allowed, "*") {
+				return []model.Provider{p}, nil
+			}
+		}
+	}
+	var matched []model.Provider
+	for _, p := range s.cache.ListProviders() {
+		if p.Category != category || !model.IsCategorySupported(p.Category) || !p.Enabled {
+			continue
+		}
+		if !cache.MatchesList(allowed, p.Name) && !cache.MatchesList(allowed, string(p.Type)) {
+			continue
+		}
+		if s.providerSupportsModel(&p, modelName) {
+			matched = append(matched, p)
+		}
+	}
+	if len(matched) == 0 {
+		if parts := strings.SplitN(modelName, "/", 2); len(parts) == 2 {
+			if p, ok := s.cache.GetProviderByName(parts[0]); ok && p.Category == category && p.Enabled {
+				if s.providerSupportsModel(&p, parts[1]) {
+					return []model.Provider{p}, nil
+				}
+			}
+		}
+		return nil, errNoProvider(modelName)
+	}
+	return matched, nil
+}
+
+func (s *Service) providerSupportsModel(p *model.Provider, modelName string) bool {
+	models := s.cache.GetModels(p.ID)
+	if len(models) == 0 {
+		for _, k := range s.cache.ListProviderKeys(p.ID) {
+			allowed := cache.ParseJSONList(k.ModelsJSON)
+			if cache.MatchesList(allowed, modelName) || cache.MatchesList(allowed, "*") {
+				return true
+			}
+		}
+		return false
+	}
+	for _, m := range models {
+		if m.Enabled && modelEntryMatches(m, modelName) {
+			return true
+		}
+	}
+	return false
+}
+
+func errNoProvider(model string) error {
+	return &gatewayError{msg: "no provider found for model " + model}
+}
+
+type gatewayError struct{ msg string }
+
+func (e *gatewayError) Error() string { return e.msg }

internal/gateway/service.go

@@ -0,0 +1,475 @@
+package gateway
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"time"
+
+	"github.com/rose_cat707/luminary/internal/auth"
+	"github.com/rose_cat707/luminary/internal/limiter"
+	"github.com/rose_cat707/luminary/internal/model"
+	"github.com/rose_cat707/luminary/internal/pricing"
+	"github.com/rose_cat707/luminary/internal/provider"
+	"github.com/rose_cat707/luminary/internal/provider/anthropic"
+	"github.com/rose_cat707/luminary/internal/provider/comfyui"
+	"github.com/rose_cat707/luminary/internal/provider/openai"
+	"github.com/rose_cat707/luminary/internal/settings"
+	"github.com/rose_cat707/luminary/internal/store/cache"
+)
+
+type Service struct {
+	cache        *cache.Store
+	settings     *settings.Runtime
+	openai       *openai.Client
+	anthropic    *anthropic.Client
+	streamClient *provider.HTTPClientWrapper
+}
+
+func New(c *cache.Store, rt *settings.Runtime) *Service {
+	return &Service{
+		cache:        c,
+		settings:     rt,
+		openai:       openai.New(),
+		anthropic:    anthropic.New(),
+		streamClient: &provider.HTTPClientWrapper{Client: provider.NewStreamHTTPClient()},
+	}
+}
+
+func (s *Service) ResolveIngressKey(plainKey string) (*model.IngressKey, error) {
+	k, ok := s.cache.LookupIngressKey(plainKey)
+	if !ok {
+		return nil, fmt.Errorf("invalid api key")
+	}
+	if k.ExpiresAt != nil && k.ExpiresAt.Before(time.Now()) {
+		return nil, fmt.Errorf("api key expired")
+	}
+	return k, nil
+}
+
+func (s *Service) providerConfig(p *model.Provider) provider.ProviderConfig {
+	return provider.ProviderConfig{
+		Type:          p.Type,
+		Category:      p.Category,
+		BaseURL:       p.BaseURL,
+		EndpointPaths: provider.ParseEndpointPaths(p.EndpointPathsJSON),
+	}
+}
+
+func (s *Service) DecryptAPIKey(enc string) (string, error) {
+	return auth.Decrypt(enc, s.settings.EncryptionKey())
+}
+
+func (s *Service) providerAllowsEndpoint(p *model.Provider, endpoint provider.EndpointType) bool {
+	allowed := provider.ParseAllowedEndpoints(p.AllowedEndpointsJSON, p.Category)
+	return provider.IsEndpointAllowed(allowed, endpoint)
+}
+
+func (s *Service) canServeEndpoint(p *model.Provider, endpoint provider.EndpointType) bool {
+	if s.providerAllowsEndpoint(p, endpoint) {
+		return true
+	}
+	return endpoint == provider.EndpointResponses && s.providerAllowsEndpoint(p, provider.EndpointChatCompletion)
+}
+
+func (s *Service) shouldAdaptResponses(p *model.Provider) bool {
+	return !s.providerAllowsEndpoint(p, provider.EndpointResponses) && s.providerAllowsEndpoint(p, provider.EndpointChatCompletion)
+}
+
+func (s *Service) forward(ctx context.Context, p *model.Provider, apiKey string, endpoint provider.EndpointType, body []byte) (*provider.ChatResponse, error) {
+	var err error
+	body, err = s.rewriteBodyForProvider(p.ID, body)
+	if err != nil {
+		return nil, err
+	}
+	actualEndpoint := endpoint
+	actualBody := body
+	adaptResponses := false
+
+	if endpoint == provider.EndpointResponses {
+		if s.shouldAdaptResponses(p) {
+			adapted, err := provider.ResponsesToChatCompletion(body)
+			if err != nil {
+				return nil, err
+			}
+			actualBody = adapted
+			actualEndpoint = provider.EndpointChatCompletion
+			adaptResponses = true
+		} else if !s.providerAllowsEndpoint(p, endpoint) {
+			return nil, fmt.Errorf("endpoint %s not enabled on provider %s", endpoint, p.Name)
+		}
+	} else if !s.providerAllowsEndpoint(p, endpoint) {
+		return nil, fmt.Errorf("endpoint %s not enabled on provider %s", endpoint, p.Name)
+	}
+
+	resp, err := s.forwardRaw(ctx, p, apiKey, actualEndpoint, actualBody)
+	if err != nil {
+		return nil, err
+	}
+
+	if endpoint == provider.EndpointResponses && !adaptResponses && resp != nil && resp.StatusCode == http.StatusNotFound && s.providerAllowsEndpoint(p, provider.EndpointChatCompletion) {
+		adaptedBody, adaptErr := provider.ResponsesToChatCompletion(body)
+		if adaptErr == nil {
+			if resp2, err2 := s.forwardRaw(ctx, p, apiKey, provider.EndpointChatCompletion, adaptedBody); err2 == nil && resp2 != nil {
+				resp = resp2
+				adaptResponses = true
+			}
+		}
+	}
+
+	if adaptResponses && resp != nil && resp.StatusCode < 400 {
+		converted, convErr := provider.ChatCompletionToResponses(resp.Body)
+		if convErr == nil {
+			resp.Body = converted
+		}
+	}
+	return resp, nil
+}
+
+func (s *Service) forwardRaw(ctx context.Context, p *model.Provider, apiKey string, endpoint provider.EndpointType, body []byte) (*provider.ChatResponse, error) {
+	cfg := s.providerConfig(p)
+	if p.Type == model.ProviderAnthropic && endpoint == provider.EndpointChatCompletion {
+		return s.anthropic.ForwardWithConfig(ctx, apiKey, cfg, endpoint, body)
+	}
+	return s.openai.ForwardWithConfig(ctx, apiKey, cfg, endpoint, body)
+}
+
+func (s *Service) forwardStream(ctx context.Context, p *model.Provider, apiKey string, endpoint provider.EndpointType, body []byte, w io.Writer, flusher http.Flusher) (*provider.StreamResult, error) {
+	var err error
+	body, err = s.rewriteBodyForProvider(p.ID, body)
+	if err != nil {
+		return nil, err
+	}
+	actualEndpoint := endpoint
+	actualBody := body
+	adaptResponses := false
+
+	if endpoint == provider.EndpointResponses {
+		if s.shouldAdaptResponses(p) {
+			adapted, err := provider.ResponsesToChatCompletion(body)
+			if err != nil {
+				return nil, err
+			}
+			actualBody = adapted
+			actualEndpoint = provider.EndpointChatCompletion
+			adaptResponses = true
+		} else if !s.providerAllowsEndpoint(p, endpoint) {
+			return nil, fmt.Errorf("endpoint %s not enabled", endpoint)
+		}
+	} else if !s.providerAllowsEndpoint(p, endpoint) {
+		return nil, fmt.Errorf("endpoint %s not enabled", endpoint)
+	}
+
+	cfg := s.providerConfig(p)
+	url := cfg.ResolveEndpointURL(actualEndpoint)
+
+	if adaptResponses {
+		pr, pw := io.Pipe()
+		errCh := make(chan error, 1)
+		go func() {
+			_, err := provider.ForwardStreamHTTP(ctx, s.streamClient.Client, "POST", url, apiKey, p.Type, actualBody, pw, nil)
+			_ = pw.Close()
+			errCh <- err
+		}()
+		result, err := provider.AdaptResponsesStream(body, pr, w, flusher)
+		if streamErr := <-errCh; err == nil && streamErr != nil {
+			err = streamErr
+		}
+		return result, err
+	}
+
+	return provider.ForwardStreamHTTP(ctx, s.streamClient.Client, "POST", url, apiKey, p.Type, actualBody, w, flusher)
+}
+
+type forwardParams struct {
+	ingress  *model.IngressKey
+	endpoint provider.EndpointType
+	category model.ProviderCategory
+	body     []byte
+	stream   bool
+}
+
+func (s *Service) prepareForward(params forwardParams) (modelName string, providers []model.Provider, route *routeTarget, err error) {
+	if len(params.body) > 0 {
+		var payload map[string]any
+		if err := json.Unmarshal(params.body, &payload); err != nil {
+			return "", nil, nil, fmt.Errorf("%w: %v", ErrInvalidJSON, err)
+		}
+		modelName, _ = payload["model"].(string)
+	}
+	if params.endpoint != provider.EndpointListModels && modelName == "" {
+		return "", nil, nil, ErrModelRequired
+	}
+	allowedProviders := cache.ParseJSONList(params.ingress.AllowedProvidersJSON)
+	if modelName != "" {
+		allowedModels := cache.ParseJSONList(params.ingress.AllowedModelsJSON)
+		if !cache.MatchesList(allowedModels, modelName) && !cache.MatchesList(allowedModels, "*") {
+			return "", nil, nil, ErrModelNotAllowed
+		}
+	}
+	if !limiter.CheckIngressBudget(params.ingress, 0) {
+		return "", nil, nil, ErrBudgetExceeded
+	}
+	if !limiter.CheckRateLimit(s.cache, params.ingress.ID, params.ingress.RateLimitRPM, params.ingress.RateLimitTPM, 1000) {
+		return "", nil, nil, ErrRateLimitExceeded
+	}
+	route = s.resolveRoute(params.ingress.ID, modelName)
+	providers, err = s.findProvidersForModel(modelName, allowedProviders, params.category, route)
+	return modelName, providers, route, err
+}
+
+func (s *Service) recordUsage(ingress *model.IngressKey, att *attempt, clientIP, modelName, endpoint string, stream bool, tokensIn, tokensOut int, latency int64, status, errMsg, requestBody, responseBody string) {
+	var providerKeyID, providerID uint
+	if att != nil {
+		providerKeyID = att.key.ID
+		providerID = att.provider.ID
+	}
+	cost := pricing.Estimate(modelName, tokensIn, tokensOut, endpoint)
+	s.cache.RecordUsage(cache.UsageDelta{
+		IngressKeyID:  ingress.ID,
+		ProviderKeyID: providerKeyID,
+		ProviderID:    providerID,
+		ClientIP:      clientIP,
+		Endpoint:      endpoint,
+		Model:         modelName,
+		TokensIn:      tokensIn,
+		TokensOut:     tokensOut,
+		Cost:          cost,
+		LatencyMs:     latency,
+		Status:        status,
+		Stream:        stream,
+		ErrorMsg:      errMsg,
+		RequestBody:   requestBody,
+		ResponseBody:  responseBody,
+		BudgetUsed:    cost,
+		Requests:      1,
+		Tokens:        int64(tokensIn + tokensOut),
+	})
+}
+
+func (s *Service) ForwardEndpoint(ctx context.Context, ingress *model.IngressKey, endpoint provider.EndpointType, rawBody []byte, clientIP string) (*provider.ChatResponse, error) {
+	category := categoryForEndpoint(endpoint)
+	modelName, providers, route, err := s.prepareForward(forwardParams{
+		ingress: ingress, endpoint: endpoint, category: category, body: rawBody,
+	})
+	if err != nil {
+		s.recordUsage(ingress, nil, clientIP, modelNameOrBody(rawBody, modelName), string(endpoint), false, 0, 0, 0, "error", err.Error(), truncateLogBody(rawBody), "")
+		return nil, err
+	}
+	attempts, err := s.buildAttempts(providers, modelName, route)
+	if err != nil {
+		s.recordUsage(ingress, nil, clientIP, modelName, string(endpoint), false, 0, 0, 0, "error", err.Error(), truncateLogBody(rawBody), "")
+		return nil, err
+	}
+	start := time.Now()
+	resp, att, err := s.forwardWithFailover(ctx, attempts, endpoint, rawBody)
+	latency := time.Since(start).Milliseconds()
+	status := "success"
+	if err != nil || (resp != nil && resp.StatusCode >= 400) {
+		status = "error"
+	}
+	errMsg := describeForwardError(err, resp)
+	tokensIn, tokensOut := 0, 0
+	var respBody []byte
+	if resp != nil {
+		tokensIn, tokensOut = resp.TokensIn, resp.TokensOut
+		respBody = resp.Body
+	}
+	s.recordUsage(ingress, att, clientIP, modelName, string(endpoint), false, tokensIn, tokensOut, latency, status, errMsg, truncateLogBody(rawBody), truncateLogBody(respBody))
+	if err != nil && resp != nil && resp.StatusCode >= 400 {
+		return resp, nil
+	}
+	return resp, err
+}
+
+func modelNameOrBody(body []byte, modelName string) string {
+	if modelName != "" {
+		return modelName
+	}
+	return extractModelName(body)
+}
+
+func (s *Service) ForwardStream(ctx context.Context, ingress *model.IngressKey, endpoint provider.EndpointType, rawBody []byte, clientIP string, w io.Writer, flusher http.Flusher) error {
+	category := categoryForEndpoint(endpoint)
+	modelName, providers, route, err := s.prepareForward(forwardParams{
+		ingress: ingress, endpoint: endpoint, category: category, body: rawBody, stream: true,
+	})
+	if err != nil {
+		s.recordUsage(ingress, nil, clientIP, modelNameOrBody(rawBody, modelName), string(endpoint), true, 0, 0, 0, "error", err.Error(), truncateLogBody(rawBody), "")
+		return err
+	}
+	attempts, err := s.buildAttempts(providers, modelName, route)
+	if err != nil {
+		s.recordUsage(ingress, nil, clientIP, modelName, string(endpoint), true, 0, 0, 0, "error", err.Error(), truncateLogBody(rawBody), "")
+		return err
+	}
+	var lastErr error
+	var lastAtt *attempt
+	maxRetries := s.settings.GatewayMaxRetries()
+	if maxRetries < 1 {
+		maxRetries = 1
+	}
+	backoffMs := s.settings.GatewayRetryBackoffMs()
+	for _, att := range attempts {
+		if !s.canServeEndpoint(att.provider, endpoint) {
+			continue
+		}
+		for try := 0; try < maxRetries; try++ {
+			if try > 0 {
+				time.Sleep(time.Duration(backoffMs) * time.Millisecond)
+			}
+			start := time.Now()
+			result, err := s.forwardStream(ctx, att.provider, att.apiKey, endpoint, rawBody, w, flusher)
+			latency := time.Since(start).Milliseconds()
+			lastAtt = &att
+			if err != nil {
+				lastErr = err
+				continue
+			}
+			if result == nil {
+				lastErr = fmt.Errorf("empty upstream stream response")
+				continue
+			}
+			if result.StatusCode < 400 {
+				s.recordUsage(ingress, &att, clientIP, modelName, string(endpoint), true, result.TokensIn, result.TokensOut, latency, "success", "", truncateLogBody(rawBody), "[SSE stream]")
+				return nil
+			}
+			errMsg := describeForwardError(nil, &provider.ChatResponse{StatusCode: result.StatusCode, Body: result.Body})
+			lastErr = fmt.Errorf("%s", errMsg)
+			respBody := "[SSE stream]"
+			if len(result.Body) > 0 {
+				respBody = truncateLogBody(result.Body)
+			}
+			if passThroughUpstreamStatus(result.StatusCode) {
+				s.recordUsage(ingress, &att, clientIP, modelName, string(endpoint), true, result.TokensIn, result.TokensOut, latency, "error", errMsg, truncateLogBody(rawBody), respBody)
+				return nil
+			}
+			if tryNextKeyOnUpstreamStatus(result.StatusCode) {
+				break
+			}
+			if retryableUpstreamStatus(result.StatusCode) {
+				continue
+			}
+			s.recordUsage(ingress, &att, clientIP, modelName, string(endpoint), true, result.TokensIn, result.TokensOut, latency, "error", errMsg, truncateLogBody(rawBody), respBody)
+			return nil
+		}
+	}
+	if lastErr == nil {
+		lastErr = fmt.Errorf("all providers failed")
+	}
+	s.recordUsage(ingress, lastAtt, clientIP, modelName, string(endpoint), true, 0, 0, 0, "error", lastErr.Error(), truncateLogBody(rawBody), "[SSE stream]")
+	return lastErr
+}
+
+func categoryForEndpoint(ep provider.EndpointType) model.ProviderCategory {
+	switch ep {
+	case provider.EndpointEmbeddings:
+		return model.CategoryEmbedding
+	case provider.EndpointRerank:
+		return model.CategoryRerank
+	default:
+		return model.CategoryLLM
+	}
+}
+
+func listModelsCategories() []model.ProviderCategory {
+	return []model.ProviderCategory{
+		model.CategoryLLM,
+		model.CategoryEmbedding,
+		model.CategoryRerank,
+	}
+}
+
+func providerInListModelsCategory(cat model.ProviderCategory) bool {
+	for _, c := range listModelsCategories() {
+		if c == cat {
+			return true
+		}
+	}
+	return false
+}
+
+func (s *Service) ListModels(ctx context.Context, ingress *model.IngressKey) ([]map[string]any, error) {
+	allowedProviders := cache.ParseJSONList(ingress.AllowedProvidersJSON)
+	var result []map[string]any
+	var hasLLM bool
+	for _, p := range s.cache.ListProviders() {
+		if !providerInListModelsCategory(p.Category) {
+			continue
+		}
+		if !cache.MatchesList(allowedProviders, p.Name) && !cache.MatchesList(allowedProviders, string(p.Type)) {
+			continue
+		}
+		if !s.providerAllowsEndpoint(&p, provider.EndpointListModels) {
+			continue
+		}
+		if p.Category == model.CategoryLLM {
+			hasLLM = true
+		}
+		for _, m := range s.cache.GetModels(p.ID) {
+			if m.Enabled {
+				result = append(result, map[string]any{
+					"id": IngressModelID(m), "object": "model", "owned_by": p.Name,
+				})
+			}
+		}
+	}
+	if len(result) == 0 && hasLLM {
+		result = append(result, map[string]any{"id": "gpt-4o-mini", "object": "model", "owned_by": "luminary"})
+	}
+	return result, nil
+}
+
+func (s *Service) FetchModels(ctx context.Context, providerID uint) ([]provider.ModelInfo, error) {
+	p, ok := s.cache.GetProvider(providerID)
+	if !ok {
+		return nil, fmt.Errorf("provider not found")
+	}
+	if !s.providerAllowsEndpoint(&p, provider.EndpointListModels) {
+		return nil, fmt.Errorf("list_models endpoint not enabled")
+	}
+	keys := s.cache.ListProviderKeys(providerID)
+	var firstKey string
+	for _, k := range keys {
+		if k.Enabled {
+			plain, err := s.DecryptAPIKey(k.APIKeyEnc)
+			if err == nil {
+				firstKey = plain
+				break
+			}
+		}
+	}
+	if firstKey == "" && p.Type != model.ProviderOllama {
+		return nil, fmt.Errorf("no enabled keys")
+	}
+	cfg := s.providerConfig(&p)
+	if p.Type == model.ProviderAnthropic {
+		return s.anthropic.ListModelsWithConfig(ctx, firstKey, cfg)
+	}
+	return s.openai.ListModelsWithConfig(ctx, firstKey, cfg)
+}
+
+func (s *Service) HealthCheckProvider(ctx context.Context, providerID uint) (model.ProviderStatus, int64, string) {
+	p, ok := s.cache.GetProvider(providerID)
+	if !ok {
+		return model.ProviderUnhealthy, 0, "not found"
+	}
+	if !model.IsCategorySupported(p.Category) {
+		return model.ProviderUnhealthy, 0, "category not supported"
+	}
+	if p.Type == model.ProviderComfyUI {
+		start := time.Now()
+		if err := comfyui.New(p.BaseURL).HealthCheck(ctx); err != nil {
+			return model.ProviderUnhealthy, time.Since(start).Milliseconds(), err.Error()
+		}
+		return model.ProviderHealthy, time.Since(start).Milliseconds(), ""
+	}
+	keys := s.cache.ListProviderKeys(providerID)
+	if len(keys) == 0 && p.Type != model.ProviderOllama {
+		return model.ProviderUnhealthy, 0, "no provider keys configured"
+	}
+	cfg := s.providerConfig(&p)
+	return s.runHealthChecks(ctx, &p, keys, cfg)
+}

internal/gateway/upstream.go

@@ -0,0 +1,43 @@
+package gateway
+
+import "net/http"
+
+// retryableUpstreamStatus reports HTTP codes that should trigger retry / failover.
+func retryableUpstreamStatus(code int) bool {
+	switch code {
+	case http.StatusTooManyRequests,
+		http.StatusRequestTimeout,
+		http.StatusBadGateway,
+		http.StatusServiceUnavailable,
+		http.StatusGatewayTimeout:
+		return true
+	default:
+		return code >= 500
+	}
+}
+
+// tryNextKeyOnUpstreamStatus reports auth/rate errors where another provider key may work.
+func tryNextKeyOnUpstreamStatus(code int) bool {
+	switch code {
+	case http.StatusUnauthorized,
+		http.StatusForbidden,
+		http.StatusTooManyRequests,
+		http.StatusPaymentRequired:
+		return true
+	default:
+		return retryableUpstreamStatus(code)
+	}
+}
+
+// passThroughUpstreamStatus reports client errors that should be returned as-is.
+func passThroughUpstreamStatus(code int) bool {
+	switch code {
+	case http.StatusBadRequest,
+		http.StatusNotFound,
+		http.StatusUnprocessableEntity,
+		http.StatusUnsupportedMediaType:
+		return true
+	default:
+		return false
+	}
+}

internal/gateway/upstream_test.go

@@ -0,0 +1,27 @@
+package gateway
+
+import (
+	"net/http"
+	"testing"
+)
+
+func TestRetryableUpstreamStatus(t *testing.T) {
+	if !retryableUpstreamStatus(http.StatusTooManyRequests) {
+		t.Fatal("429 should be retryable")
+	}
+	if !retryableUpstreamStatus(http.StatusServiceUnavailable) {
+		t.Fatal("503 should be retryable")
+	}
+	if retryableUpstreamStatus(http.StatusBadRequest) {
+		t.Fatal("400 should not be retryable")
+	}
+}
+
+func TestTryNextKeyOnUpstreamStatus(t *testing.T) {
+	if !tryNextKeyOnUpstreamStatus(http.StatusTooManyRequests) {
+		t.Fatal("429 should try next key")
+	}
+	if !tryNextKeyOnUpstreamStatus(http.StatusUnauthorized) {
+		t.Fatal("401 should try next key")
+	}
+}