Jaymr/new-api
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(performance): add recent success rates to model summary and enhance metrics retrieval

Browse Source
This commit is contained in:
CaIon
2026-06-18 12:00:03 +08:00
parent a37ce3d6b5
commit 208d86e96e
5 changed files with 129 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
model/perf_metric.go
+29
View File
@@ -68,6 +68,16 @@ type PerfMetricSummary struct {
GenerationMs int64 `json:"generation_ms"`
}
type PerfMetricSummaryBucket struct {
ModelName string `json:"model_name"`
BucketTs int64 `json:"bucket_ts"`
RequestCount int64 `json:"request_count"`
SuccessCount int64 `json:"success_count"`
TotalLatencyMs int64 `json:"total_latency_ms"`
OutputTokens int64 `json:"output_tokens"`
GenerationMs int64 `json:"generation_ms"`
}
func GetPerfMetricsSummaryAll(startTs int64, endTs int64, groups []string) ([]PerfMetricSummary, error) {
var summaries []PerfMetricSummary
query := DB.Model(&PerfMetric{}).
@@ -86,6 +96,25 @@ func GetPerfMetricsSummaryAll(startTs int64, endTs int64, groups []string) ([]Pe
return summaries, err
}
func GetPerfMetricsSummaryBucketsAll(startTs int64, endTs int64, groups []string) ([]PerfMetricSummaryBucket, error) {
var summaries []PerfMetricSummaryBucket
query := DB.Model(&PerfMetric{}).
Select("model_name, bucket_ts, SUM(request_count) as request_count, SUM(success_count) as success_count, SUM(total_latency_ms) as total_latency_ms, SUM(output_tokens) as output_tokens, SUM(generation_ms) as generation_ms").
Where("bucket_ts >= ? AND bucket_ts <= ?", startTs, endTs)
if groups != nil {
if len(groups) == 0 {
return summaries, nil
}
query = query.Where(commonGroupCol+" IN ?", groups)
}
err := query.
Group("model_name, bucket_ts").
Having("SUM(request_count) > 0").
Order("bucket_ts ASC").
Find(&summaries).Error
return summaries, err
}
func DeletePerfMetricsBefore(cutoffTs int64) error {
if cutoffTs <= 0 {
return nil
pkg/perf_metrics/metrics.go
+67 -14
View File
@@ -133,20 +133,23 @@ func QuerySummaryAll(hours int, groups []string) (SummaryAllResult, error) {
startTs := endTs - int64(hours)*3600
allowedGroups := allowedGroupSet(groups)
rows, err := model.GetPerfMetricsSummaryAll(startTs, endTs, groups)
rows, err := model.GetPerfMetricsSummaryBucketsAll(startTs, endTs, groups)
if err != nil {
return SummaryAllResult{}, err
}
totals := map[string]counters{}
modelBuckets := map[string]map[int64]counters{}
for _, row := range rows {
totals[row.ModelName] = counters{
value := counters{
requestCount: row.RequestCount,
successCount: row.SuccessCount,
totalLatencyMs: row.TotalLatencyMs,
outputTokens: row.OutputTokens,
generationMs: row.GenerationMs,
}
mergeModelTotals(totals, row.ModelName, value)
mergeModelBucket(modelBuckets, row.ModelName, row.BucketTs, value)
}
hotBuckets.Range(func(key, value any) bool {
@@ -163,13 +166,8 @@ func QuerySummaryAll(hours int, groups []string) (SummaryAllResult, error) {
if snap.requestCount == 0 {
return true
}
cur := totals[k.model]
cur.requestCount += snap.requestCount
cur.successCount += snap.successCount
cur.totalLatencyMs += snap.totalLatencyMs
cur.outputTokens += snap.outputTokens
cur.generationMs += snap.generationMs
totals[k.model] = cur
mergeModelTotals(totals, k.model, snap)
mergeModelBucket(modelBuckets, k.model, k.bucketTs, snap)
return true
})
@@ -185,11 +183,12 @@ func QuerySummaryAll(hours int, groups []string) (SummaryAllResult, error) {
avgTps = float64(total.outputTokens) / (float64(total.generationMs) / 1000.0)
}
models = append(models, ModelSummary{
ModelName: name,
AvgLatencyMs: avgLatency,
SuccessRate: math.Round(successRate*100) / 100,
AvgTps: math.Round(avgTps*100) / 100,
RequestCount: total.requestCount,
ModelName: name,
AvgLatencyMs: avgLatency,
SuccessRate: math.Round(successRate*100) / 100,
AvgTps: math.Round(avgTps*100) / 100,
RecentSuccessRates: recentSuccessRates(modelBuckets[name], 3),
RequestCount: total.requestCount,
})
}
sort.Slice(models, func(i, j int) bool {
@@ -199,6 +198,60 @@ func QuerySummaryAll(hours int, groups []string) (SummaryAllResult, error) {
return SummaryAllResult{Models: models}, nil
}
func mergeModelTotals(totals map[string]counters, modelName string, value counters) {
if value.requestCount == 0 {
return
}
current := totals[modelName]
current.requestCount += value.requestCount
current.successCount += value.successCount
current.totalLatencyMs += value.totalLatencyMs
current.ttftSumMs += value.ttftSumMs
current.ttftCount += value.ttftCount
current.outputTokens += value.outputTokens
current.generationMs += value.generationMs
totals[modelName] = current
}
func mergeModelBucket(modelBuckets map[string]map[int64]counters, modelName string, bucketTs int64, value counters) {
if value.requestCount == 0 {
return
}
if _, ok := modelBuckets[modelName]; !ok {
modelBuckets[modelName] = map[int64]counters{}
}
current := modelBuckets[modelName][bucketTs]
current.requestCount += value.requestCount
current.successCount += value.successCount
current.totalLatencyMs += value.totalLatencyMs
current.ttftSumMs += value.ttftSumMs
current.ttftCount += value.ttftCount
current.outputTokens += value.outputTokens
current.generationMs += value.generationMs
modelBuckets[modelName][bucketTs] = current
}
func recentSuccessRates(buckets map[int64]counters, limit int) []float64 {
if len(buckets) == 0 || limit <= 0 {
return nil
}
timestamps := make([]int64, 0, len(buckets))
for ts := range buckets {
timestamps = append(timestamps, ts)
}
sort.Slice(timestamps, func(i, j int) bool {
return timestamps[i] < timestamps[j]
})
if len(timestamps) > limit {
timestamps = timestamps[len(timestamps)-limit:]
}
rates := make([]float64, 0, len(timestamps))
for _, ts := range timestamps {
rates = append(rates, math.Round(successRate(buckets[ts])*100)/100)
}
return rates
}
func allowedGroupSet(groups []string) map[string]struct{} {
if groups == nil {
return nil
pkg/perf_metrics/types.go
+6 -5
View File
@@ -48,11 +48,12 @@ type QueryResult struct {
}
type ModelSummary struct {
ModelName string `json:"model_name"`
AvgLatencyMs int64 `json:"avg_latency_ms"`
SuccessRate float64 `json:"success_rate"`
AvgTps float64 `json:"avg_tps"`
RequestCount int64 `json:"-"`
ModelName string `json:"model_name"`
AvgLatencyMs int64 `json:"avg_latency_ms"`
SuccessRate float64 `json:"success_rate"`
AvgTps float64 `json:"avg_tps"`
RecentSuccessRates []float64 `json:"recent_success_rates,omitempty"`
RequestCount int64 `json:"-"`
}
type SummaryAllResult struct {
web/default/src/features/performance-metrics/types.ts
Vendored
+1
View File
@@ -48,6 +48,7 @@ export type PerfModelSummary = {
avg_latency_ms: number
success_rate: number
avg_tps: number
recent_success_rates?: number[]
request_count?: number
}
web/default/src/features/pricing/components/model-perf-badge.tsx
Vendored
+26 -4
View File
@@ -29,6 +29,7 @@ export type ModelPerfBadgeData = {
avg_latency_ms: number
success_rate: number
avg_tps: number
recent_success_rates?: number[]
}
export interface ModelPerfBadgeProps extends React.HTMLAttributes<HTMLDivElement> {
@@ -50,7 +51,15 @@ export const ModelPerfBadge = memo(function ModelPerfBadge(
const { avg_latency_ms, avg_tps, success_rate } = props.perf
const statusColor = getSuccessRateDotClass(success_rate)
const recentRates =
props.perf.recent_success_rates?.filter((rate) => Number.isFinite(rate)) ??
[]
const statusRates =
recentRates.length > 0 ? recentRates.slice(-3) : [success_rate]
const statusBars = [
...Array(Math.max(0, 3 - statusRates.length)).fill(null),
...statusRates,
].slice(-3)
return (
<div
@@ -83,9 +92,22 @@ export const ModelPerfBadge = memo(function ModelPerfBadge(
{t('Status short')}
</div>
<div className='flex h-4 items-center justify-end gap-0.5'>
<span className='bg-muted-foreground/10 h-2 w-1 rounded-full' />
<span className='bg-muted-foreground/15 h-2.5 w-1 rounded-full' />
<span className={cn('h-3 w-1 rounded-full', statusColor)} />
{statusBars.map((rate, index) => (
<span
key={`${index}-${rate ?? 'empty'}`}
className={cn(
'w-1 rounded-full',
index === 0 && 'h-2',
index === 1 && 'h-2.5',
index === 2 && 'h-3',
rate == null
? index === 0
? 'bg-muted-foreground/10'
: 'bg-muted-foreground/15'
: getSuccessRateDotClass(rate)
)}
/>
))}
</div>
</div>
</div>