Calcium-Ion
bd585d78ef
* fix(aws): cancel Bedrock requests on client disconnect * fix(billing): log effective usage billing path
459 lines
14 KiB
Go
459 lines
14 KiB
Go
package aws
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"errors"
|
|
"io"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/QuantumNous/new-api/common"
|
|
relaycommon "github.com/QuantumNous/new-api/relay/common"
|
|
"github.com/QuantumNous/new-api/relaykit/dto"
|
|
relaytypes "github.com/QuantumNous/new-api/relaykit/types"
|
|
"github.com/aws/aws-sdk-go-v2/aws"
|
|
"github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream"
|
|
"github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream/eventstreamapi"
|
|
"github.com/aws/aws-sdk-go-v2/credentials"
|
|
"github.com/aws/aws-sdk-go-v2/service/bedrockruntime"
|
|
"github.com/gin-gonic/gin"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
const awsTestModel = "anthropic.claude-3-5-sonnet-20240620-v1:0"
|
|
|
|
type awsHTTPClientFunc func(*http.Request) (*http.Response, error)
|
|
|
|
func (f awsHTTPClientFunc) Do(request *http.Request) (*http.Response, error) {
|
|
return f(request)
|
|
}
|
|
|
|
type awsNotifyingResponseWriter struct {
|
|
*httptest.ResponseRecorder
|
|
notifyOn []byte
|
|
notified chan int
|
|
once sync.Once
|
|
}
|
|
|
|
func newAwsNotifyingResponseWriter(notifyOn string) *awsNotifyingResponseWriter {
|
|
return &awsNotifyingResponseWriter{
|
|
ResponseRecorder: httptest.NewRecorder(),
|
|
notifyOn: []byte(notifyOn),
|
|
notified: make(chan int, 1),
|
|
}
|
|
}
|
|
|
|
func (w *awsNotifyingResponseWriter) Write(data []byte) (int, error) {
|
|
return w.ResponseRecorder.Write(data)
|
|
}
|
|
|
|
func (w *awsNotifyingResponseWriter) Flush() {
|
|
w.ResponseRecorder.Flush()
|
|
if bytes.Contains(w.Body.Bytes(), w.notifyOn) {
|
|
w.once.Do(func() {
|
|
w.notified <- w.Body.Len()
|
|
})
|
|
}
|
|
}
|
|
|
|
func newAwsTestClient(httpClient bedrockruntime.HTTPClient) *bedrockruntime.Client {
|
|
return bedrockruntime.New(bedrockruntime.Options{
|
|
Region: "us-east-1",
|
|
BaseEndpoint: aws.String("https://bedrock.test"),
|
|
Credentials: aws.NewCredentialsCache(credentials.NewStaticCredentialsProvider(
|
|
"access-key", "secret-key", "",
|
|
)),
|
|
HTTPClient: httpClient,
|
|
Retryer: aws.NopRetryer{},
|
|
})
|
|
}
|
|
|
|
func newAwsTestContext(writer http.ResponseWriter, requestContext context.Context) *gin.Context {
|
|
c, _ := gin.CreateTestContext(writer)
|
|
c.Request = httptest.NewRequest(http.MethodPost, "/v1/chat/completions", nil).WithContext(requestContext)
|
|
return c
|
|
}
|
|
|
|
func newAwsTestRelayInfo() *relaycommon.RelayInfo {
|
|
return &relaycommon.RelayInfo{
|
|
StartTime: time.Now(),
|
|
IsStream: true,
|
|
OriginModelName: awsTestModel,
|
|
RelayFormat: relaytypes.RelayFormatOpenAI,
|
|
ShouldIncludeUsage: true,
|
|
ChannelMeta: &relaycommon.ChannelMeta{
|
|
UpstreamModelName: awsTestModel,
|
|
},
|
|
}
|
|
}
|
|
|
|
func newAwsInvokeModelInput() *bedrockruntime.InvokeModelInput {
|
|
return &bedrockruntime.InvokeModelInput{
|
|
ModelId: aws.String(awsTestModel),
|
|
Body: []byte(`{}`),
|
|
Accept: aws.String("application/json"),
|
|
ContentType: aws.String("application/json"),
|
|
}
|
|
}
|
|
|
|
func newAwsStreamInput() *bedrockruntime.InvokeModelWithResponseStreamInput {
|
|
return &bedrockruntime.InvokeModelWithResponseStreamInput{
|
|
ModelId: aws.String(awsTestModel),
|
|
Body: []byte(`{}`),
|
|
Accept: aws.String("application/json"),
|
|
ContentType: aws.String("application/json"),
|
|
}
|
|
}
|
|
|
|
func writeAwsStreamEvent(writer io.Writer, data string) error {
|
|
payload, err := common.Marshal(struct {
|
|
Bytes []byte `json:"bytes"`
|
|
}{Bytes: []byte(data)})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
return eventstream.NewEncoder().Encode(writer, eventstream.Message{
|
|
Headers: eventstream.Headers{
|
|
{Name: eventstreamapi.MessageTypeHeader, Value: eventstream.StringValue(eventstreamapi.EventMessageType)},
|
|
{Name: eventstreamapi.EventTypeHeader, Value: eventstream.StringValue("chunk")},
|
|
{Name: eventstreamapi.ContentTypeHeader, Value: eventstream.StringValue("application/json")},
|
|
},
|
|
Payload: payload,
|
|
})
|
|
}
|
|
|
|
func newAwsStreamResponse(request *http.Request, body io.ReadCloser) *http.Response {
|
|
return &http.Response{
|
|
StatusCode: http.StatusOK,
|
|
Status: "200 OK",
|
|
Header: http.Header{
|
|
"Content-Type": []string{"application/vnd.amazon.eventstream"},
|
|
"X-Amzn-Bedrock-Content-Type": []string{"application/json"},
|
|
},
|
|
Body: body,
|
|
Request: request,
|
|
}
|
|
}
|
|
|
|
func TestDoAwsClientRequest_AppliesRuntimeHeaderOverrideToAnthropicBeta(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
gin.SetMode(gin.TestMode)
|
|
recorder := httptest.NewRecorder()
|
|
ctx, _ := gin.CreateTestContext(recorder)
|
|
ctx.Request = httptest.NewRequest(http.MethodPost, "/v1/messages", nil)
|
|
|
|
info := &relaycommon.RelayInfo{
|
|
OriginModelName: "claude-3-5-sonnet-20240620",
|
|
IsStream: false,
|
|
UseRuntimeHeadersOverride: true,
|
|
RuntimeHeadersOverride: map[string]any{
|
|
"anthropic-beta": "computer-use-2025-01-24",
|
|
},
|
|
ChannelMeta: &relaycommon.ChannelMeta{
|
|
ApiKey: "access-key|secret-key|us-east-1",
|
|
UpstreamModelName: "claude-3-5-sonnet-20240620",
|
|
},
|
|
}
|
|
|
|
requestBody := bytes.NewBufferString(`{"messages":[{"role":"user","content":"hello"}],"max_tokens":128}`)
|
|
adaptor := &Adaptor{}
|
|
|
|
_, err := doAwsClientRequest(ctx, info, adaptor, requestBody)
|
|
require.NoError(t, err)
|
|
|
|
awsReq, ok := adaptor.AwsReq.(*bedrockruntime.InvokeModelInput)
|
|
require.True(t, ok)
|
|
|
|
var payload map[string]any
|
|
require.NoError(t, common.Unmarshal(awsReq.Body, &payload))
|
|
|
|
anthropicBeta, exists := payload["anthropic_beta"]
|
|
require.True(t, exists)
|
|
|
|
values, ok := anthropicBeta.([]any)
|
|
require.True(t, ok)
|
|
require.Equal(t, []any{"computer-use-2025-01-24"}, values)
|
|
}
|
|
|
|
func TestNewAwsInvokeContextInheritsParent(t *testing.T) {
|
|
originalRelayTimeout := common.RelayTimeout
|
|
t.Cleanup(func() {
|
|
common.RelayTimeout = originalRelayTimeout
|
|
})
|
|
|
|
tests := []struct {
|
|
name string
|
|
relayTimeout int
|
|
wantDeadline bool
|
|
}{
|
|
{name: "without relay timeout", relayTimeout: 0, wantDeadline: false},
|
|
{name: "with relay timeout", relayTimeout: 30, wantDeadline: true},
|
|
}
|
|
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
common.RelayTimeout = test.relayTimeout
|
|
parent, cancelParent := context.WithCancel(context.Background())
|
|
invokeContext, cancelInvoke := newAwsInvokeContext(parent)
|
|
defer cancelInvoke()
|
|
|
|
_, hasDeadline := invokeContext.Deadline()
|
|
assert.Equal(t, test.wantDeadline, hasDeadline)
|
|
|
|
cancelParent()
|
|
require.ErrorIs(t, invokeContext.Err(), context.Canceled)
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestNewAwsInvokeErrorSkipsRetryOnlyForClientCancellation(t *testing.T) {
|
|
canceledContext, cancel := context.WithCancel(context.Background())
|
|
cancel()
|
|
|
|
tests := []struct {
|
|
name string
|
|
requestContext context.Context
|
|
err error
|
|
wantSkipRetry bool
|
|
}{
|
|
{
|
|
name: "client context canceled",
|
|
requestContext: canceledContext,
|
|
err: context.Canceled,
|
|
wantSkipRetry: true,
|
|
},
|
|
{
|
|
name: "relay timeout with live client context",
|
|
requestContext: context.Background(),
|
|
err: context.DeadlineExceeded,
|
|
wantSkipRetry: false,
|
|
},
|
|
{
|
|
name: "upstream error with live client context",
|
|
requestContext: context.Background(),
|
|
err: errors.New("upstream failed"),
|
|
wantSkipRetry: false,
|
|
},
|
|
}
|
|
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
err := newAwsInvokeError(test.requestContext, test.err, "InvokeModel")
|
|
assert.Equal(t, test.wantSkipRetry, relaytypes.IsSkipRetryError(err))
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestAwsHandlersCancelSdkRequestAndSkipRetry(t *testing.T) {
|
|
originalRelayTimeout := common.RelayTimeout
|
|
common.RelayTimeout = 0
|
|
t.Cleanup(func() {
|
|
common.RelayTimeout = originalRelayTimeout
|
|
})
|
|
|
|
tests := []struct {
|
|
name string
|
|
request any
|
|
handle func(*gin.Context, *relaycommon.RelayInfo, *Adaptor) (*relaytypes.NewAPIError, *dto.Usage)
|
|
}{
|
|
{name: "non-stream", request: newAwsInvokeModelInput(), handle: awsHandler},
|
|
{name: "stream", request: newAwsStreamInput(), handle: awsStreamHandler},
|
|
{name: "nova", request: newAwsInvokeModelInput(), handle: handleNovaRequest},
|
|
}
|
|
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
requestContext, cancelRequest := context.WithCancel(context.Background())
|
|
t.Cleanup(cancelRequest)
|
|
|
|
upstreamContexts := make(chan context.Context, 1)
|
|
client := newAwsTestClient(awsHTTPClientFunc(func(request *http.Request) (*http.Response, error) {
|
|
upstreamContexts <- request.Context()
|
|
<-request.Context().Done()
|
|
return nil, request.Context().Err()
|
|
}))
|
|
adaptor := &Adaptor{AwsClient: client, AwsReq: test.request}
|
|
c := newAwsTestContext(httptest.NewRecorder(), requestContext)
|
|
info := newAwsTestRelayInfo()
|
|
|
|
type handlerResult struct {
|
|
err *relaytypes.NewAPIError
|
|
usage *dto.Usage
|
|
}
|
|
results := make(chan handlerResult, 1)
|
|
go func() {
|
|
err, usage := test.handle(c, info, adaptor)
|
|
results <- handlerResult{err: err, usage: usage}
|
|
}()
|
|
|
|
var upstreamContext context.Context
|
|
select {
|
|
case upstreamContext = <-upstreamContexts:
|
|
case result := <-results:
|
|
t.Fatalf("handler returned before issuing AWS request: %v", result.err)
|
|
case <-time.After(5 * time.Second):
|
|
t.Fatal("AWS request did not start")
|
|
}
|
|
|
|
cancelRequest()
|
|
|
|
var result handlerResult
|
|
select {
|
|
case result = <-results:
|
|
case <-time.After(5 * time.Second):
|
|
t.Fatal("handler did not stop after client cancellation")
|
|
}
|
|
|
|
require.ErrorIs(t, upstreamContext.Err(), context.Canceled)
|
|
require.NotNil(t, result.err)
|
|
assert.True(t, relaytypes.IsSkipRetryError(result.err))
|
|
assert.Nil(t, result.usage)
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestAwsStreamHandlerUsesFinalUpstreamUsage(t *testing.T) {
|
|
originalRelayTimeout := common.RelayTimeout
|
|
common.RelayTimeout = 0
|
|
t.Cleanup(func() {
|
|
common.RelayTimeout = originalRelayTimeout
|
|
})
|
|
|
|
events := []string{
|
|
`{"type":"message_start","message":{"id":"msg_test","type":"message","role":"assistant","model":"claude-test","content":[],"usage":{"input_tokens":100,"output_tokens":1}}}`,
|
|
`{"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}`,
|
|
`{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"partial"}}`,
|
|
`{"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"output_tokens":423}}`,
|
|
`{"type":"message_stop"}`,
|
|
}
|
|
client := newAwsTestClient(awsHTTPClientFunc(func(request *http.Request) (*http.Response, error) {
|
|
var body bytes.Buffer
|
|
for _, event := range events {
|
|
if err := writeAwsStreamEvent(&body, event); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
return newAwsStreamResponse(request, io.NopCloser(bytes.NewReader(body.Bytes()))), nil
|
|
}))
|
|
adaptor := &Adaptor{AwsClient: client, AwsReq: newAwsStreamInput()}
|
|
recorder := httptest.NewRecorder()
|
|
c := newAwsTestContext(recorder, context.Background())
|
|
|
|
handlerErr, usage := awsStreamHandler(c, newAwsTestRelayInfo(), adaptor)
|
|
|
|
require.Nil(t, handlerErr)
|
|
require.NotNil(t, usage)
|
|
require.NotNil(t, usage.BillingUsage)
|
|
require.NotNil(t, usage.BillingUsage.ClaudeUsage)
|
|
assert.Equal(t, 100, usage.BillingUsage.ClaudeUsage.InputTokens)
|
|
assert.Equal(t, 423, usage.BillingUsage.ClaudeUsage.OutputTokens)
|
|
assert.Contains(t, recorder.Body.String(), "[DONE]")
|
|
}
|
|
|
|
func TestAwsStreamHandlerStopsAtClientCancellationAndKeepsPartialBillingUsage(t *testing.T) {
|
|
originalRelayTimeout := common.RelayTimeout
|
|
common.RelayTimeout = 0
|
|
t.Cleanup(func() {
|
|
common.RelayTimeout = originalRelayTimeout
|
|
})
|
|
|
|
requestContext, cancelRequest := context.WithCancel(context.Background())
|
|
t.Cleanup(cancelRequest)
|
|
releaseFinal := make(chan struct{})
|
|
var releaseFinalOnce sync.Once
|
|
release := func() {
|
|
releaseFinalOnce.Do(func() {
|
|
close(releaseFinal)
|
|
})
|
|
}
|
|
t.Cleanup(release)
|
|
|
|
producerResults := make(chan error, 1)
|
|
upstreamContexts := make(chan context.Context, 1)
|
|
client := newAwsTestClient(awsHTTPClientFunc(func(request *http.Request) (*http.Response, error) {
|
|
upstreamContexts <- request.Context()
|
|
reader, writer := io.Pipe()
|
|
go func() {
|
|
defer writer.Close()
|
|
initialEvents := []string{
|
|
`{"type":"message_start","message":{"id":"msg_test","type":"message","role":"assistant","model":"claude-test","content":[],"usage":{"input_tokens":100,"output_tokens":1}}}`,
|
|
`{"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}`,
|
|
`{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"partial"}}`,
|
|
}
|
|
for _, event := range initialEvents {
|
|
if err := writeAwsStreamEvent(writer, event); err != nil {
|
|
producerResults <- err
|
|
return
|
|
}
|
|
}
|
|
|
|
<-releaseFinal
|
|
producerResults <- writeAwsStreamEvent(writer, `{"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"output_tokens":423}}`)
|
|
}()
|
|
return newAwsStreamResponse(request, reader), nil
|
|
}))
|
|
|
|
responseWriter := newAwsNotifyingResponseWriter("partial")
|
|
c := newAwsTestContext(responseWriter, requestContext)
|
|
adaptor := &Adaptor{AwsClient: client, AwsReq: newAwsStreamInput()}
|
|
|
|
type handlerResult struct {
|
|
err *relaytypes.NewAPIError
|
|
usage *dto.Usage
|
|
}
|
|
results := make(chan handlerResult, 1)
|
|
go func() {
|
|
err, usage := awsStreamHandler(c, newAwsTestRelayInfo(), adaptor)
|
|
results <- handlerResult{err: err, usage: usage}
|
|
}()
|
|
|
|
var upstreamContext context.Context
|
|
select {
|
|
case upstreamContext = <-upstreamContexts:
|
|
case <-time.After(5 * time.Second):
|
|
t.Fatal("AWS stream request did not start")
|
|
}
|
|
|
|
var bodyLengthBeforeCancel int
|
|
select {
|
|
case bodyLengthBeforeCancel = <-responseWriter.notified:
|
|
case <-time.After(5 * time.Second):
|
|
t.Fatal("partial response was not written")
|
|
}
|
|
cancelRequest()
|
|
|
|
var result handlerResult
|
|
select {
|
|
case result = <-results:
|
|
case <-time.After(5 * time.Second):
|
|
t.Fatal("stream handler did not stop after client cancellation")
|
|
}
|
|
|
|
require.ErrorIs(t, upstreamContext.Err(), context.Canceled)
|
|
require.Nil(t, result.err)
|
|
require.NotNil(t, result.usage)
|
|
require.NotNil(t, result.usage.BillingUsage)
|
|
require.NotNil(t, result.usage.BillingUsage.ClaudeUsage)
|
|
assert.Equal(t, dto.BillingUsageSourceClaudeMessages, result.usage.BillingUsage.Source)
|
|
assert.Equal(t, dto.BillingUsageSemanticAnthropic, result.usage.BillingUsage.Semantic)
|
|
assert.Equal(t, 100, result.usage.BillingUsage.ClaudeUsage.InputTokens)
|
|
assert.Equal(t, 1, result.usage.BillingUsage.ClaudeUsage.OutputTokens)
|
|
assert.Equal(t, bodyLengthBeforeCancel, responseWriter.Body.Len())
|
|
assert.NotContains(t, responseWriter.Body.String(), "[DONE]")
|
|
|
|
release()
|
|
select {
|
|
case producerErr := <-producerResults:
|
|
require.Error(t, producerErr)
|
|
case <-time.After(5 * time.Second):
|
|
t.Fatal("upstream producer did not observe the closed stream")
|
|
}
|
|
}
|