diff --git a/model/perf_metric.go b/model/perf_metric.go index 1667adfb..f9c33c85 100644 --- a/model/perf_metric.go +++ b/model/perf_metric.go @@ -68,6 +68,16 @@ type PerfMetricSummary struct { GenerationMs int64 `json:"generation_ms"` } +type PerfMetricSummaryBucket struct { + ModelName string `json:"model_name"` + BucketTs int64 `json:"bucket_ts"` + RequestCount int64 `json:"request_count"` + SuccessCount int64 `json:"success_count"` + TotalLatencyMs int64 `json:"total_latency_ms"` + OutputTokens int64 `json:"output_tokens"` + GenerationMs int64 `json:"generation_ms"` +} + func GetPerfMetricsSummaryAll(startTs int64, endTs int64, groups []string) ([]PerfMetricSummary, error) { var summaries []PerfMetricSummary query := DB.Model(&PerfMetric{}). @@ -86,6 +96,25 @@ func GetPerfMetricsSummaryAll(startTs int64, endTs int64, groups []string) ([]Pe return summaries, err } +func GetPerfMetricsSummaryBucketsAll(startTs int64, endTs int64, groups []string) ([]PerfMetricSummaryBucket, error) { + var summaries []PerfMetricSummaryBucket + query := DB.Model(&PerfMetric{}). + Select("model_name, bucket_ts, SUM(request_count) as request_count, SUM(success_count) as success_count, SUM(total_latency_ms) as total_latency_ms, SUM(output_tokens) as output_tokens, SUM(generation_ms) as generation_ms"). + Where("bucket_ts >= ? AND bucket_ts <= ?", startTs, endTs) + if groups != nil { + if len(groups) == 0 { + return summaries, nil + } + query = query.Where(commonGroupCol+" IN ?", groups) + } + err := query. + Group("model_name, bucket_ts"). + Having("SUM(request_count) > 0"). + Order("bucket_ts ASC"). + Find(&summaries).Error + return summaries, err +} + func DeletePerfMetricsBefore(cutoffTs int64) error { if cutoffTs <= 0 { return nil diff --git a/pkg/perf_metrics/metrics.go b/pkg/perf_metrics/metrics.go index ab505d0d..33b79ee4 100644 --- a/pkg/perf_metrics/metrics.go +++ b/pkg/perf_metrics/metrics.go @@ -133,20 +133,23 @@ func QuerySummaryAll(hours int, groups []string) (SummaryAllResult, error) { startTs := endTs - int64(hours)*3600 allowedGroups := allowedGroupSet(groups) - rows, err := model.GetPerfMetricsSummaryAll(startTs, endTs, groups) + rows, err := model.GetPerfMetricsSummaryBucketsAll(startTs, endTs, groups) if err != nil { return SummaryAllResult{}, err } totals := map[string]counters{} + modelBuckets := map[string]map[int64]counters{} for _, row := range rows { - totals[row.ModelName] = counters{ + value := counters{ requestCount: row.RequestCount, successCount: row.SuccessCount, totalLatencyMs: row.TotalLatencyMs, outputTokens: row.OutputTokens, generationMs: row.GenerationMs, } + mergeModelTotals(totals, row.ModelName, value) + mergeModelBucket(modelBuckets, row.ModelName, row.BucketTs, value) } hotBuckets.Range(func(key, value any) bool { @@ -163,13 +166,8 @@ func QuerySummaryAll(hours int, groups []string) (SummaryAllResult, error) { if snap.requestCount == 0 { return true } - cur := totals[k.model] - cur.requestCount += snap.requestCount - cur.successCount += snap.successCount - cur.totalLatencyMs += snap.totalLatencyMs - cur.outputTokens += snap.outputTokens - cur.generationMs += snap.generationMs - totals[k.model] = cur + mergeModelTotals(totals, k.model, snap) + mergeModelBucket(modelBuckets, k.model, k.bucketTs, snap) return true }) @@ -185,11 +183,12 @@ func QuerySummaryAll(hours int, groups []string) (SummaryAllResult, error) { avgTps = float64(total.outputTokens) / (float64(total.generationMs) / 1000.0) } models = append(models, ModelSummary{ - ModelName: name, - AvgLatencyMs: avgLatency, - SuccessRate: math.Round(successRate*100) / 100, - AvgTps: math.Round(avgTps*100) / 100, - RequestCount: total.requestCount, + ModelName: name, + AvgLatencyMs: avgLatency, + SuccessRate: math.Round(successRate*100) / 100, + AvgTps: math.Round(avgTps*100) / 100, + RecentSuccessRates: recentSuccessRates(modelBuckets[name], 3), + RequestCount: total.requestCount, }) } sort.Slice(models, func(i, j int) bool { @@ -199,6 +198,60 @@ func QuerySummaryAll(hours int, groups []string) (SummaryAllResult, error) { return SummaryAllResult{Models: models}, nil } +func mergeModelTotals(totals map[string]counters, modelName string, value counters) { + if value.requestCount == 0 { + return + } + current := totals[modelName] + current.requestCount += value.requestCount + current.successCount += value.successCount + current.totalLatencyMs += value.totalLatencyMs + current.ttftSumMs += value.ttftSumMs + current.ttftCount += value.ttftCount + current.outputTokens += value.outputTokens + current.generationMs += value.generationMs + totals[modelName] = current +} + +func mergeModelBucket(modelBuckets map[string]map[int64]counters, modelName string, bucketTs int64, value counters) { + if value.requestCount == 0 { + return + } + if _, ok := modelBuckets[modelName]; !ok { + modelBuckets[modelName] = map[int64]counters{} + } + current := modelBuckets[modelName][bucketTs] + current.requestCount += value.requestCount + current.successCount += value.successCount + current.totalLatencyMs += value.totalLatencyMs + current.ttftSumMs += value.ttftSumMs + current.ttftCount += value.ttftCount + current.outputTokens += value.outputTokens + current.generationMs += value.generationMs + modelBuckets[modelName][bucketTs] = current +} + +func recentSuccessRates(buckets map[int64]counters, limit int) []float64 { + if len(buckets) == 0 || limit <= 0 { + return nil + } + timestamps := make([]int64, 0, len(buckets)) + for ts := range buckets { + timestamps = append(timestamps, ts) + } + sort.Slice(timestamps, func(i, j int) bool { + return timestamps[i] < timestamps[j] + }) + if len(timestamps) > limit { + timestamps = timestamps[len(timestamps)-limit:] + } + rates := make([]float64, 0, len(timestamps)) + for _, ts := range timestamps { + rates = append(rates, math.Round(successRate(buckets[ts])*100)/100) + } + return rates +} + func allowedGroupSet(groups []string) map[string]struct{} { if groups == nil { return nil diff --git a/pkg/perf_metrics/types.go b/pkg/perf_metrics/types.go index af68eb88..f926ddce 100644 --- a/pkg/perf_metrics/types.go +++ b/pkg/perf_metrics/types.go @@ -48,11 +48,12 @@ type QueryResult struct { } type ModelSummary struct { - ModelName string `json:"model_name"` - AvgLatencyMs int64 `json:"avg_latency_ms"` - SuccessRate float64 `json:"success_rate"` - AvgTps float64 `json:"avg_tps"` - RequestCount int64 `json:"-"` + ModelName string `json:"model_name"` + AvgLatencyMs int64 `json:"avg_latency_ms"` + SuccessRate float64 `json:"success_rate"` + AvgTps float64 `json:"avg_tps"` + RecentSuccessRates []float64 `json:"recent_success_rates,omitempty"` + RequestCount int64 `json:"-"` } type SummaryAllResult struct { diff --git a/web/default/src/features/performance-metrics/types.ts b/web/default/src/features/performance-metrics/types.ts index fcf3c298..4e4450a5 100644 --- a/web/default/src/features/performance-metrics/types.ts +++ b/web/default/src/features/performance-metrics/types.ts @@ -48,6 +48,7 @@ export type PerfModelSummary = { avg_latency_ms: number success_rate: number avg_tps: number + recent_success_rates?: number[] request_count?: number } diff --git a/web/default/src/features/pricing/components/model-perf-badge.tsx b/web/default/src/features/pricing/components/model-perf-badge.tsx index 75fa46bf..4090934c 100644 --- a/web/default/src/features/pricing/components/model-perf-badge.tsx +++ b/web/default/src/features/pricing/components/model-perf-badge.tsx @@ -29,6 +29,7 @@ export type ModelPerfBadgeData = { avg_latency_ms: number success_rate: number avg_tps: number + recent_success_rates?: number[] } export interface ModelPerfBadgeProps extends React.HTMLAttributes { @@ -50,7 +51,15 @@ export const ModelPerfBadge = memo(function ModelPerfBadge( const { avg_latency_ms, avg_tps, success_rate } = props.perf - const statusColor = getSuccessRateDotClass(success_rate) + const recentRates = + props.perf.recent_success_rates?.filter((rate) => Number.isFinite(rate)) ?? + [] + const statusRates = + recentRates.length > 0 ? recentRates.slice(-3) : [success_rate] + const statusBars = [ + ...Array(Math.max(0, 3 - statusRates.length)).fill(null), + ...statusRates, + ].slice(-3) return (
- - - + {statusBars.map((rate, index) => ( + + ))}