From 52858ad1e617069b708d820e1ea8a312b8077c85 Mon Sep 17 00:00:00 2001 From: Scott Date: Wed, 1 Jul 2026 21:21:41 +0800 Subject: [PATCH] feat: support Wan2.7 i2v media mapping (#4984) * feat: support Wan2.7 i2v media mapping * fix: normalize wan2.7 i2v image inputs --- relay/channel/task/ali/adaptor.go | 127 ++++++++++++++++-- relay/channel/task/ali/adaptor_test.go | 172 +++++++++++++++++++++++++ relay/channel/task/ali/constants.go | 2 + relay/common/relay_utils.go | 3 + relay/common/relay_utils_test.go | 33 +++++ 5 files changed, 323 insertions(+), 14 deletions(-) create mode 100644 relay/channel/task/ali/adaptor_test.go create mode 100644 relay/common/relay_utils_test.go diff --git a/relay/channel/task/ali/adaptor.go b/relay/channel/task/ali/adaptor.go index 5b6b01d9..f3645472 100644 --- a/relay/channel/task/ali/adaptor.go +++ b/relay/channel/task/ali/adaptor.go @@ -33,15 +33,22 @@ type AliVideoRequest struct { Parameters *AliVideoParameters `json:"parameters,omitempty"` } +// AliVideoMedia describes Wan2.7 image-to-video media inputs. +type AliVideoMedia struct { + Type string `json:"type"` + URL string `json:"url"` +} + // AliVideoInput 视频输入参数 type AliVideoInput struct { - Prompt string `json:"prompt,omitempty"` // 文本提示词 - ImgURL string `json:"img_url,omitempty"` // 首帧图像URL或Base64（图生视频） - FirstFrameURL string `json:"first_frame_url,omitempty"` // 首帧图片URL（首尾帧生视频） - LastFrameURL string `json:"last_frame_url,omitempty"` // 尾帧图片URL（首尾帧生视频） - AudioURL string `json:"audio_url,omitempty"` // 音频URL（wan2.5支持） - NegativePrompt string `json:"negative_prompt,omitempty"` // 反向提示词 - Template string `json:"template,omitempty"` // 视频特效模板 + Prompt string `json:"prompt,omitempty"` // 文本提示词 + ImgURL string `json:"img_url,omitempty"` // 首帧图像URL或Base64（图生视频） + FirstFrameURL string `json:"first_frame_url,omitempty"` // 首帧图片URL（首尾帧生视频） + LastFrameURL string `json:"last_frame_url,omitempty"` // 尾帧图片URL（首尾帧生视频） + AudioURL string `json:"audio_url,omitempty"` // 音频URL（wan2.5支持） + Media []AliVideoMedia `json:"media,omitempty"` // 媒体列表（wan2.7-i2v新协议） + NegativePrompt string `json:"negative_prompt,omitempty"` // 反向提示词 + Template string `json:"template,omitempty"` // 视频特效模板 } // AliVideoParameters 视频参数 @@ -87,12 +94,13 @@ type AliUsage struct { type AliMetadata struct { // Input 相关 - AudioURL string `json:"audio_url,omitempty"` // 音频URL - ImgURL string `json:"img_url,omitempty"` // 图片URL（图生视频） - FirstFrameURL string `json:"first_frame_url,omitempty"` // 首帧图片URL（首尾帧生视频） - LastFrameURL string `json:"last_frame_url,omitempty"` // 尾帧图片URL（首尾帧生视频） - NegativePrompt string `json:"negative_prompt,omitempty"` // 反向提示词 - Template string `json:"template,omitempty"` // 视频特效模板 + AudioURL string `json:"audio_url,omitempty"` // 音频URL + ImgURL string `json:"img_url,omitempty"` // 图片URL（图生视频） + FirstFrameURL string `json:"first_frame_url,omitempty"` // 首帧图片URL（首尾帧生视频） + LastFrameURL string `json:"last_frame_url,omitempty"` // 尾帧图片URL（首尾帧生视频） + Media []AliVideoMedia `json:"media,omitempty"` // 媒体列表（wan2.7-i2v新协议） + NegativePrompt string `json:"negative_prompt,omitempty"` // 反向提示词 + Template string `json:"template,omitempty"` // 视频特效模板 // Parameters 相关 Resolution *string `json:"resolution,omitempty"` // 分辨率: 480P/720P/1080P @@ -252,6 +260,93 @@ func ProcessAliOtherRatios(aliReq *AliVideoRequest) (map[string]float64, error) return otherRatios, nil } +func isWan27I2VModel(model string) bool { + return strings.HasPrefix(model, "wan2.7-i2v") +} + +func firstNonEmpty(values ...string) string { + for _, value := range values { + trimmed := strings.TrimSpace(value) + if trimmed != "" { + return trimmed + } + } + return "" +} + +func firstTaskImage(req relaycommon.TaskSubmitReq) string { + if image := strings.TrimSpace(req.Image); image != "" { + return image + } + for _, image := range req.Images { + if trimmed := strings.TrimSpace(image); trimmed != "" { + return trimmed + } + } + if inputReference := strings.TrimSpace(req.InputReference); inputReference != "" { + return inputReference + } + return "" +} + +func secondTaskImage(req relaycommon.TaskSubmitReq) string { + nonEmptyImages := 0 + for _, image := range req.Images { + trimmed := strings.TrimSpace(image) + if trimmed == "" { + continue + } + nonEmptyImages++ + if nonEmptyImages == 2 { + return trimmed + } + } + return "" +} + +func normalizeWan27I2VInput(aliReq *AliVideoRequest, req relaycommon.TaskSubmitReq) error { + if !isWan27I2VModel(aliReq.Model) { + return nil + } + + if len(aliReq.Input.Media) == 0 { + firstFrameURL := firstNonEmpty(aliReq.Input.FirstFrameURL, aliReq.Input.ImgURL, firstTaskImage(req)) + lastFrameURL := firstNonEmpty(aliReq.Input.LastFrameURL, secondTaskImage(req)) + audioURL := aliReq.Input.AudioURL + + if firstFrameURL != "" { + aliReq.Input.Media = append(aliReq.Input.Media, AliVideoMedia{ + Type: "first_frame", + URL: firstFrameURL, + }) + } + if lastFrameURL != "" { + aliReq.Input.Media = append(aliReq.Input.Media, AliVideoMedia{ + Type: "last_frame", + URL: lastFrameURL, + }) + } + if audioURL != "" { + aliReq.Input.Media = append(aliReq.Input.Media, AliVideoMedia{ + Type: "driving_audio", + URL: audioURL, + }) + } + } + + if len(aliReq.Input.Media) == 0 { + return fmt.Errorf("wan2.7-i2v requires image, images, input_reference, or input.media") + } + + // Wan2.7 image-to-video uses the new input.media protocol. Avoid sending + // legacy fields that belong to wan2.6 and earlier image-to-video APIs. + aliReq.Input.ImgURL = "" + aliReq.Input.FirstFrameURL = "" + aliReq.Input.LastFrameURL = "" + aliReq.Input.AudioURL = "" + return nil +} + func (a *TaskAdaptor) convertToAliRequest(info *relaycommon.RelayInfo, req relaycommon.TaskSubmitReq) (*AliVideoRequest, error) { upstreamModel := req.Model if info.IsModelMapped { @@ -261,7 +356,7 @@ func (a *TaskAdaptor) convertToAliRequest(info *relaycommon.RelayInfo, req relay Model: upstreamModel, Input: AliVideoInput{ Prompt: req.Prompt, - ImgURL: req.InputReference, + ImgURL: firstTaskImage(req), }, Parameters: &AliVideoParameters{ PromptExtend: true, // 默认开启智能改写 @@ -340,6 +435,10 @@ func (a *TaskAdaptor) convertToAliRequest(info *relaycommon.RelayInfo, req relay return nil, errors.New("can't change model with metadata") } + if err := normalizeWan27I2VInput(aliReq, req); err != nil { + return nil, err + } + return aliReq, nil } diff --git a/relay/channel/task/ali/adaptor_test.go b/relay/channel/task/ali/adaptor_test.go new file mode 100644 index 00000000..a7c414bf --- /dev/null +++ b/relay/channel/task/ali/adaptor_test.go @@ -0,0 +1,172 @@ +package ali + +import ( + "strings" + "testing" + + "github.com/QuantumNous/new-api/common" + relaycommon "github.com/QuantumNous/new-api/relay/common" + "github.com/stretchr/testify/require" +) + +func testRelayInfo() *relaycommon.RelayInfo { + return &relaycommon.RelayInfo{ + ChannelMeta: &relaycommon.ChannelMeta{}, + } +} + +func TestConvertToAliRequestWan27I2VBuildsMediaFromImage(t *testing.T) { + adaptor := &TaskAdaptor{} + req := relaycommon.TaskSubmitReq{ + Model: "wan2.7-i2v", + Prompt: "animate the first frame", + Image: "https://example.com/first.png", + Size: "720p", + Duration: 10, + } + + aliReq, err := adaptor.convertToAliRequest(testRelayInfo(), req) + + require.NoError(t, err) + require.Equal(t, "wan2.7-i2v", aliReq.Model) + require.Equal(t, "720P", aliReq.Parameters.Resolution) + require.Equal(t, 10, aliReq.Parameters.Duration) + require.Equal(t, []AliVideoMedia{ + {Type: "first_frame", URL: "https://example.com/first.png"}, + }, aliReq.Input.Media) + require.Empty(t, aliReq.Input.ImgURL) + + body, err := common.Marshal(aliReq) + require.NoError(t, err) + require.Contains(t, string(body), `"media"`) + require.NotContains(t, string(body), `"img_url"`) +} + +func TestConvertToAliRequestWan27I2VBuildsFirstAndLastFrameFromImages(t *testing.T) { + adaptor := &TaskAdaptor{} + req := relaycommon.TaskSubmitReq{ + Model: "wan2.7-i2v", + Prompt: "interpolate between frames", + Images: []string{ + "https://example.com/first.png", + "https://example.com/last.png", + }, + } + + aliReq, err := adaptor.convertToAliRequest(testRelayInfo(), req) + + require.NoError(t, err) + require.Equal(t, []AliVideoMedia{ + {Type: "first_frame", URL: "https://example.com/first.png"}, + {Type: "last_frame", URL: "https://example.com/last.png"}, + }, aliReq.Input.Media) +} + +func TestConvertToAliRequestWan27I2VPrefersImageBeforeImagesAndInputReference(t *testing.T) { + adaptor := &TaskAdaptor{} + req := relaycommon.TaskSubmitReq{ + Model: "wan2.7-i2v", + Prompt: "use the direct image", + Image: " https://example.com/direct.png ", + Images: []string{"https://example.com/images-first.png", " https://example.com/images-last.png "}, + InputReference: "https://example.com/input-reference.png", + } + + aliReq, err := adaptor.convertToAliRequest(testRelayInfo(), req) + + require.NoError(t, err) + require.Equal(t, []AliVideoMedia{ + {Type: "first_frame", URL: "https://example.com/direct.png"}, + {Type: "last_frame", URL: "https://example.com/images-last.png"}, + }, aliReq.Input.Media) +} + +func TestConvertToAliRequestWan27I2VFallsBackToFirstNonEmptyImage(t *testing.T) { + adaptor := &TaskAdaptor{} + req := relaycommon.TaskSubmitReq{ + Model: "wan2.7-i2v", + Prompt: "skip blank images", + Image: " ", + Images: []string{ + " ", + " https://example.com/first.png ", + " https://example.com/last.png ", + }, + InputReference: "https://example.com/input-reference.png", + } + + aliReq, err := adaptor.convertToAliRequest(testRelayInfo(), req) + + require.NoError(t, err) + require.Equal(t, []AliVideoMedia{ + {Type: "first_frame", URL: "https://example.com/first.png"}, + {Type: "last_frame", URL: "https://example.com/last.png"}, + }, aliReq.Input.Media) +} + +func TestConvertToAliRequestWan27I2VKeepsExplicitMetadataMedia(t *testing.T) { + adaptor := &TaskAdaptor{} + req := relaycommon.TaskSubmitReq{ + Model: "wan2.7-i2v", + Prompt: "continue the clip", + Image: "https://example.com/direct.png", + Images: []string{"https://example.com/images-first.png", "https://example.com/images-last.png"}, + InputReference: "https://example.com/input-reference.png", + Metadata: map[string]interface{}{ + "input": map[string]interface{}{ + "media": []interface{}{ + map[string]interface{}{ + "type": "first_clip", + "url": "https://example.com/input.mp4", + }, + }, + }, + }, + } + + aliReq, err := adaptor.convertToAliRequest(testRelayInfo(), req) + + require.NoError(t, err) + require.Equal(t, []AliVideoMedia{ + {Type: "first_clip", URL: "https://example.com/input.mp4"}, + }, aliReq.Input.Media) + require.Empty(t, aliReq.Input.ImgURL) + + body, err := common.Marshal(aliReq) + require.NoError(t, err) + require.Contains(t, string(body), `"media"`) + require.NotContains(t, string(body), `"img_url"`) +} + +func TestConvertToAliRequestWan27I2VRequiresMedia(t *testing.T) { + adaptor := &TaskAdaptor{} + req := relaycommon.TaskSubmitReq{ + Model: "wan2.7-i2v", + Prompt: "animate without a frame", + } + + _, err := adaptor.convertToAliRequest(testRelayInfo(), req) + + require.Error(t, err) + require.True(t, strings.Contains(err.Error(), "requires image")) +} + +func TestConvertToAliRequestWan25I2VKeepsLegacyImgURL(t *testing.T) { + adaptor := &TaskAdaptor{} + req := relaycommon.TaskSubmitReq{ + Model: "wan2.5-i2v-preview", + Prompt: "animate the first frame", + Image: "https://example.com/first.png", + } + + aliReq, err := adaptor.convertToAliRequest(testRelayInfo(), req) + + require.NoError(t, err) + require.Equal(t, "https://example.com/first.png", aliReq.Input.ImgURL) + require.Empty(t, aliReq.Input.Media) + + body, err := common.Marshal(aliReq) + require.NoError(t, err) + require.Contains(t, string(body), `"img_url"`) + require.NotContains(t, string(body), `"media"`) +} diff --git a/relay/channel/task/ali/constants.go b/relay/channel/task/ali/constants.go index 8dc64ec5..349f6560 100644 --- a/relay/channel/task/ali/constants.go +++ b/relay/channel/task/ali/constants.go @@ -1,6 +1,8 @@ package ali var ModelList = []string{ + "wan2.7-i2v", // 万相2.7图生视频（新input.media协议） + "wan2.7-t2v", // 万相2.7文生视频 "wan2.5-i2v-preview", // 万相2.5 preview（有声视频）推荐 "wan2.2-i2v-flash", // 万相2.2极速版（无声视频） "wan2.2-i2v-plus", // 万相2.2专业版（无声视频） diff --git a/relay/common/relay_utils.go b/relay/common/relay_utils.go index 18df77a6..e3bb4383 100644 --- a/relay/common/relay_utils.go +++ b/relay/common/relay_utils.go @@ -139,6 +139,9 @@ func ValidateMultipartDirect(c *gin.Context, info *RelayInfo) *dto.TaskError { } if req.InputReference != "" { req.Images = []string{req.InputReference} + } else if len(req.Images) == 0 && strings.TrimSpace(req.Image) != "" { + // 兼容单图上传 + req.Images = []string{strings.TrimSpace(req.Image)} } if strings.TrimSpace(req.Model) == "" { diff --git a/relay/common/relay_utils_test.go b/relay/common/relay_utils_test.go new file mode 100644 index 00000000..62390da0 --- /dev/null +++ b/relay/common/relay_utils_test.go @@ -0,0 +1,33 @@ +package common + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/QuantumNous/new-api/constant" + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/require" +) + +func TestValidateMultipartDirectNormalizesImageField(t *testing.T) { + gin.SetMode(gin.TestMode) + body := strings.NewReader(`{"model":"wan2.7-i2v","prompt":"animate","image":" https://example.com/first.png "}`) + request := httptest.NewRequest(http.MethodPost, "/v1/video/generations", body) + request.Header.Set("Content-Type", "application/json") + recorder := httptest.NewRecorder() + context, _ := gin.CreateTestContext(recorder) + context.Request = request + info := &RelayInfo{ + TaskRelayInfo: &TaskRelayInfo{}, + } + + taskErr := ValidateMultipartDirect(context, info) + + require.Nil(t, taskErr) + storedReq, err := GetTaskRequest(context) + require.NoError(t, err) + require.Equal(t, []string{"https://example.com/first.png"}, storedReq.Images) + require.Equal(t, constant.TaskActionGenerate, info.Action) +}