semantic routing ui

Author: kohlivrinda

framework/configstore/complexityconfig.go

@@ -142,7 +142,7 @@ const (
 )
 
 // DefaultComplexitySemanticTimeout bounds per-request embedding generation.
-const DefaultComplexitySemanticTimeout = 100 * time.Millisecond
+const DefaultComplexitySemanticTimeout = 500 * time.Millisecond
 
 // ComplexitySemanticConfig configures the embedding-based complexity
 // classifier. A non-nil value enables semantic classification. The classifier
@@ -180,7 +180,7 @@ type ComplexitySemanticConfig struct {
 	VectorStore         string `json:"vector_store,omitempty"`
 }
 
-// UnmarshalJSON accepts Timeout as a duration string ("100ms") or a JSON number
+// UnmarshalJSON accepts Timeout as a duration string ("500ms") or a JSON number
 // (milliseconds). All other fields decode through the default path via an alias.
 func (c *ComplexitySemanticConfig) UnmarshalJSON(data []byte) error {
 	// alias suppresses ComplexitySemanticConfig's UnmarshalJSON to avoid

framework/configstore/complexityconfig_test.go

@@ -70,7 +70,7 @@ func TestComplexitySemanticConfigTimeoutMarshalRoundTrip(t *testing.T) {
 func TestComplexitySemanticConfigNormalizedDefaults(t *testing.T) {
 	normalized := testSemanticConfig().normalized()
 
-	assert.Equal(t, DefaultComplexitySemanticTimeout, normalized.Timeout)
+	assert.Equal(t, 500*time.Millisecond, normalized.Timeout)
 	assert.Equal(t, ComplexitySemanticFallbackLexical, normalized.Fallback)
 	assert.Equal(t, ComplexitySemanticVectorStoreEmbedded, normalized.VectorStore)
 	require.NoError(t, normalized.Validate())

plugins/governance/embedding.go

@@ -3,6 +3,7 @@ package governance
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"time"
 
@@ -13,6 +14,11 @@ import (
 	"github.com/maximhq/bifrost/plugins/governance/complexity"
 )
 
+// ErrEmbeddingRequestExecutorNotConfigured means the HTTP server has not
+// finished wiring the governance plugin to Bifrost's embedding request path.
+// Configuration clients can retry this transient startup state.
+var ErrEmbeddingRequestExecutorNotConfigured = errors.New("embedding request executor is not configured")
+
 // EmbeddingRequestExecutor invokes the embedding endpoint on the bifrost
 // client. The plugin calls it to embed request text for semantic complexity
 // classification. It mirrors the signature of bifrost.Client.EmbeddingRequest.
@@ -68,7 +74,7 @@ type ComplexityVectorStoreSetter interface {
 // warmupEmbeddingTimeout bounds one warmup embedding call, whether that is a
 // batch of exemplars or a single-input fallback. Warmup runs in the background
 // with no request waiting on it, so it must NOT inherit semantic.Timeout — that
-// is the hot-path budget (100ms by default), which a 32-exemplar batch cannot
+// is the hot-path budget (500ms by default), which a 32-exemplar batch cannot
 // possibly meet. It stays bounded so a hung provider cannot pin the warmup
 // worker forever.
 const warmupEmbeddingTimeout = 60 * time.Second
@@ -277,7 +283,7 @@ const probeEmbeddingText = "a"
 // budget and reported to no telemetry counter.
 func (p *GovernancePlugin) ProbeEmbeddingDimension(ctx context.Context, provider schemas.ModelProvider, model string) (int, error) {
 	if p.embeddingExecutor() == nil {
-		return 0, fmt.Errorf("embedding request executor is not configured")
+		return 0, ErrEmbeddingRequestExecutorNotConfigured
 	}
 	probeCtx := schemas.NewBifrostContext(ctx, schemas.NoDeadline)
 	defer probeCtx.Cancel()
@@ -328,7 +334,7 @@ func (p *GovernancePlugin) generateEmbedding(ctx *schemas.BifrostContext, semant
 func (p *GovernancePlugin) generateEmbeddings(ctx *schemas.BifrostContext, semantic *complexity.SemanticConfig, texts []string, timeout time.Duration) ([][]float32, int, error) {
 	executor := p.embeddingExecutor()
 	if executor == nil {
-		return nil, 0, fmt.Errorf("embedding request executor is not configured")
+		return nil, 0, ErrEmbeddingRequestExecutorNotConfigured
 	}
 	if semantic == nil || semantic.Provider == "" || semantic.EmbeddingModel == "" {
 		return nil, 0, fmt.Errorf("semantic classification is not configured")

plugins/governance/embedding_test.go

@@ -131,6 +131,7 @@ func TestProbeEmbeddingDimensionRequiresExecutorAndReportsFailures(t *testing.T)
 	unwired := &GovernancePlugin{}
 	_, err := unwired.ProbeEmbeddingDimension(t.Context(), "openai", "text-embedding-3-small")
 	require.Error(t, err)
+	assert.ErrorIs(t, err, ErrEmbeddingRequestExecutorNotConfigured)
 	assert.Contains(t, err.Error(), "executor is not configured")
 
 	plugin := &GovernancePlugin{}
@@ -210,7 +211,7 @@ func TestGenerateEmbeddingTimeoutCancelsCall(t *testing.T) {
 }
 
 // TestWarmupEmbedsDoNotInheritTheRequestTimeout is a regression guard: warmup
-// used to run through semantic.Timeout, the hot-path budget (100ms by default).
+// used to run through semantic.Timeout, the hot-path budget (500ms by default).
 // A 32-exemplar batch cannot finish in that window, so every warmup failed with
 // a 504 and semantic routing silently served its fallback forever.
 func TestWarmupEmbedsDoNotInheritTheRequestTimeout(t *testing.T) {

transports/bifrost-http/handlers/governance.go

@@ -770,7 +770,6 @@ func (h *GovernanceHandler) reconcileVKModelConfig(ctx context.Context, tx *gorm
 		}
 	}
 
-
 	// Resulting budget count: the desired set if provided, else the existing set.
 	finalBudgetCount := len(mc.Budgets)
 	if d.budgetsProvided {
@@ -1230,6 +1229,10 @@ func (h *GovernanceHandler) probeComplexityEmbeddingDimension(ctx *fasthttp.Requ
 
 	dimension, err := prober.ProbeComplexityEmbeddingDimension(ctx, payload.Provider, payload.EmbeddingModel)
 	if err != nil {
+		if errors.Is(err, governance.ErrEmbeddingRequestExecutorNotConfigured) {
+			SendError(ctx, fasthttp.StatusServiceUnavailable, "embedding service is still initializing; retry in a moment")
+			return
+		}
 		// Almost always an operator-correctable cause (unknown model, missing
 		// key, provider rejecting the request), so it is reported as a client
 		// error the form can render inline.

transports/bifrost-http/handlers/governance_test.go

@@ -271,6 +271,8 @@ type mockComplexityGovernanceManager struct {
 	validationErr  error
 	semanticStatus complexity.SemanticStatusInfo
 	semanticErr    error
+	probeDimension int
+	probeErr       error
 }
 
 func (m *mockComplexityGovernanceManager) ReloadComplexityAnalyzerConfig(_ context.Context, config *complexity.AnalyzerConfig) error {
@@ -291,6 +293,10 @@ func (m *mockComplexityGovernanceManager) GetComplexitySemanticStatus(_ context.
 	return m.semanticStatus, m.semanticErr
 }
 
+func (m *mockComplexityGovernanceManager) ProbeComplexityEmbeddingDimension(_ context.Context, _ schemas.ModelProvider, _ string) (int, error) {
+	return m.probeDimension, m.probeErr
+}
+
 func testComplexityAnalyzerPayload(t *testing.T, cfg complexity.AnalyzerConfig) string {
 	t.Helper()
 	body, err := json.Marshal(cfg)
@@ -475,6 +481,23 @@ func TestComplexitySemanticStatusReturnsRuntimeReadiness(t *testing.T) {
 	}
 }
 
+func TestProbeComplexityEmbeddingDimensionReportsExecutorStartupAsUnavailable(t *testing.T) {
+	SetLogger(&mockLogger{})
+	handler := &GovernanceHandler{governanceManager: &mockComplexityGovernanceManager{
+		probeErr: governance.ErrEmbeddingRequestExecutorNotConfigured,
+	}}
+
+	ctx := newTestRequestCtx(`{"provider":"openai","embedding_model":"text-embedding-3-small"}`)
+	handler.probeComplexityEmbeddingDimension(ctx)
+
+	if ctx.Response.StatusCode() != fasthttp.StatusServiceUnavailable {
+		t.Fatalf("expected status 503, got %d: %s", ctx.Response.StatusCode(), string(ctx.Response.Body()))
+	}
+	if !strings.Contains(string(ctx.Response.Body()), "still initializing") {
+		t.Fatalf("expected retryable startup message, got %s", string(ctx.Response.Body()))
+	}
+}
+
 func TestComplexityAnalyzerConfigGetCanonicalizesLegacyRowWithoutRewritingIt(t *testing.T) {
 	SetLogger(&mockLogger{})
 	store := setupPricingOverrideHandlerStore(t)

transports/config.schema.json

@@ -3652,7 +3652,7 @@
           "description": "Embedding vector dimension"
         },
         "timeout": {
-          "description": "Per-request embedding timeout (duration string like '100ms', or milliseconds as a number; default: 100ms). On timeout the fallback classifier is used.",
+          "description": "Per-request embedding timeout (duration string like '500ms', or milliseconds as a number; default: 500ms). On timeout the fallback classifier is used.",
           "oneOf": [
             {
               "type": "string",
@@ -3662,7 +3662,8 @@
               "type": "number",
               "exclusiveMinimum": 0
             }
-          ]
+          ],
+          "default": "500ms"
         },
         "fallback": {
           "type": "string",