Skip to content

GitLab

Unverified Commit 8da5fac6 authored by 程序猿MT's avatar 程序猿MT Committed by GitHub
Browse files

Merge branch 'Wei-Shaw:main' into main

parents 1dd3158c 72310276
Hide whitespace changes
Inline Side-by-side
backend/ent/errorpassthroughrule.go
View file @ 8da5fac6
......@@ -44,6 +44,8 @@ type ErrorPassthroughRule struct {
PassthroughBody bool `json:"passthrough_body,omitempty"`
// CustomMessage holds the value of the "custom_message" field.
CustomMessage *string `json:"custom_message,omitempty"`
// SkipMonitoring holds the value of the "skip_monitoring" field.
SkipMonitoring bool `json:"skip_monitoring,omitempty"`
// Description holds the value of the "description" field.
Description *string `json:"description,omitempty"`
selectValues sql.SelectValues
......@@ -56,7 +58,7 @@ func (*ErrorPassthroughRule) scanValues(columns []string) ([]any, error) {
switch columns[i] {
case errorpassthroughrule.FieldErrorCodes, errorpassthroughrule.FieldKeywords, errorpassthroughrule.FieldPlatforms:
values[i] = new([]byte)
case errorpassthroughrule.FieldEnabled, errorpassthroughrule.FieldPassthroughCode, errorpassthroughrule.FieldPassthroughBody:
case errorpassthroughrule.FieldEnabled, errorpassthroughrule.FieldPassthroughCode, errorpassthroughrule.FieldPassthroughBody, errorpassthroughrule.FieldSkipMonitoring:
values[i] = new(sql.NullBool)
case errorpassthroughrule.FieldID, errorpassthroughrule.FieldPriority, errorpassthroughrule.FieldResponseCode:
values[i] = new(sql.NullInt64)
......@@ -171,6 +173,12 @@ func (_m *ErrorPassthroughRule) assignValues(columns []string, values []any) err
_m.CustomMessage = new(string)
*_m.CustomMessage = value.String
}
case errorpassthroughrule.FieldSkipMonitoring:
if value, ok := values[i].(*sql.NullBool); !ok {
return fmt.Errorf("unexpected type %T for field skip_monitoring", values[i])
} else if value.Valid {
_m.SkipMonitoring = value.Bool
}
case errorpassthroughrule.FieldDescription:
if value, ok := values[i].(*sql.NullString); !ok {
return fmt.Errorf("unexpected type %T for field description", values[i])
......@@ -257,6 +265,9 @@ func (_m *ErrorPassthroughRule) String() string {
builder.WriteString(*v)
}
builder.WriteString(", ")
builder.WriteString("skip_monitoring=")
builder.WriteString(fmt.Sprintf("%v", _m.SkipMonitoring))
builder.WriteString(", ")
if v := _m.Description; v != nil {
builder.WriteString("description=")
builder.WriteString(*v)
......
backend/ent/errorpassthroughrule/errorpassthroughrule.go
View file @ 8da5fac6
......@@ -39,6 +39,8 @@ const (
FieldPassthroughBody = "passthrough_body"
// FieldCustomMessage holds the string denoting the custom_message field in the database.
FieldCustomMessage = "custom_message"
// FieldSkipMonitoring holds the string denoting the skip_monitoring field in the database.
FieldSkipMonitoring = "skip_monitoring"
// FieldDescription holds the string denoting the description field in the database.
FieldDescription = "description"
// Table holds the table name of the errorpassthroughrule in the database.
......@@ -61,6 +63,7 @@ var Columns = []string{
FieldResponseCode,
FieldPassthroughBody,
FieldCustomMessage,
FieldSkipMonitoring,
FieldDescription,
}
......@@ -95,6 +98,8 @@ var (
DefaultPassthroughCode bool
// DefaultPassthroughBody holds the default value on creation for the "passthrough_body" field.
DefaultPassthroughBody bool
// DefaultSkipMonitoring holds the default value on creation for the "skip_monitoring" field.
DefaultSkipMonitoring bool
)
// OrderOption defines the ordering options for the ErrorPassthroughRule queries.
......@@ -155,6 +160,11 @@ func ByCustomMessage(opts ...sql.OrderTermOption) OrderOption {
return sql.OrderByField(FieldCustomMessage, opts...).ToFunc()
}
// BySkipMonitoring orders the results by the skip_monitoring field.
func BySkipMonitoring(opts ...sql.OrderTermOption) OrderOption {
return sql.OrderByField(FieldSkipMonitoring, opts...).ToFunc()
}
// ByDescription orders the results by the description field.
func ByDescription(opts ...sql.OrderTermOption) OrderOption {
return sql.OrderByField(FieldDescription, opts...).ToFunc()
......
backend/ent/errorpassthroughrule/where.go
View file @ 8da5fac6
......@@ -104,6 +104,11 @@ func CustomMessage(v string) predicate.ErrorPassthroughRule {
return predicate.ErrorPassthroughRule(sql.FieldEQ(FieldCustomMessage, v))
}
// SkipMonitoring applies equality check predicate on the "skip_monitoring" field. It's identical to SkipMonitoringEQ.
func SkipMonitoring(v bool) predicate.ErrorPassthroughRule {
return predicate.ErrorPassthroughRule(sql.FieldEQ(FieldSkipMonitoring, v))
}
// Description applies equality check predicate on the "description" field. It's identical to DescriptionEQ.
func Description(v string) predicate.ErrorPassthroughRule {
return predicate.ErrorPassthroughRule(sql.FieldEQ(FieldDescription, v))
......@@ -544,6 +549,16 @@ func CustomMessageContainsFold(v string) predicate.ErrorPassthroughRule {
return predicate.ErrorPassthroughRule(sql.FieldContainsFold(FieldCustomMessage, v))
}
// SkipMonitoringEQ applies the EQ predicate on the "skip_monitoring" field.
func SkipMonitoringEQ(v bool) predicate.ErrorPassthroughRule {
return predicate.ErrorPassthroughRule(sql.FieldEQ(FieldSkipMonitoring, v))
}
// SkipMonitoringNEQ applies the NEQ predicate on the "skip_monitoring" field.
func SkipMonitoringNEQ(v bool) predicate.ErrorPassthroughRule {
return predicate.ErrorPassthroughRule(sql.FieldNEQ(FieldSkipMonitoring, v))
}
// DescriptionEQ applies the EQ predicate on the "description" field.
func DescriptionEQ(v string) predicate.ErrorPassthroughRule {
return predicate.ErrorPassthroughRule(sql.FieldEQ(FieldDescription, v))
......
backend/ent/errorpassthroughrule_create.go
View file @ 8da5fac6
......@@ -172,6 +172,20 @@ func (_c *ErrorPassthroughRuleCreate) SetNillableCustomMessage(v *string) *Error
return _c
}
// SetSkipMonitoring sets the "skip_monitoring" field.
func (_c *ErrorPassthroughRuleCreate) SetSkipMonitoring(v bool) *ErrorPassthroughRuleCreate {
_c.mutation.SetSkipMonitoring(v)
return _c
}
// SetNillableSkipMonitoring sets the "skip_monitoring" field if the given value is not nil.
func (_c *ErrorPassthroughRuleCreate) SetNillableSkipMonitoring(v *bool) *ErrorPassthroughRuleCreate {
if v != nil {
_c.SetSkipMonitoring(*v)
}
return _c
}
// SetDescription sets the "description" field.
func (_c *ErrorPassthroughRuleCreate) SetDescription(v string) *ErrorPassthroughRuleCreate {
_c.mutation.SetDescription(v)
......@@ -249,6 +263,10 @@ func (_c *ErrorPassthroughRuleCreate) defaults() {
v := errorpassthroughrule.DefaultPassthroughBody
_c.mutation.SetPassthroughBody(v)
}
if _, ok := _c.mutation.SkipMonitoring(); !ok {
v := errorpassthroughrule.DefaultSkipMonitoring
_c.mutation.SetSkipMonitoring(v)
}
}
// check runs all checks and user-defined validators on the builder.
......@@ -287,6 +305,9 @@ func (_c *ErrorPassthroughRuleCreate) check() error {
if _, ok := _c.mutation.PassthroughBody(); !ok {
return &ValidationError{Name: "passthrough_body", err: errors.New(`ent: missing required field "ErrorPassthroughRule.passthrough_body"`)}
}
if _, ok := _c.mutation.SkipMonitoring(); !ok {
return &ValidationError{Name: "skip_monitoring", err: errors.New(`ent: missing required field "ErrorPassthroughRule.skip_monitoring"`)}
}
return nil
}
......@@ -366,6 +387,10 @@ func (_c *ErrorPassthroughRuleCreate) createSpec() (*ErrorPassthroughRule, *sqlg
_spec.SetField(errorpassthroughrule.FieldCustomMessage, field.TypeString, value)
_node.CustomMessage = &value
}
if value, ok := _c.mutation.SkipMonitoring(); ok {
_spec.SetField(errorpassthroughrule.FieldSkipMonitoring, field.TypeBool, value)
_node.SkipMonitoring = value
}
if value, ok := _c.mutation.Description(); ok {
_spec.SetField(errorpassthroughrule.FieldDescription, field.TypeString, value)
_node.Description = &value
......@@ -608,6 +633,18 @@ func (u *ErrorPassthroughRuleUpsert) ClearCustomMessage() *ErrorPassthroughRuleU
return u
}
// SetSkipMonitoring sets the "skip_monitoring" field.
func (u *ErrorPassthroughRuleUpsert) SetSkipMonitoring(v bool) *ErrorPassthroughRuleUpsert {
u.Set(errorpassthroughrule.FieldSkipMonitoring, v)
return u
}
// UpdateSkipMonitoring sets the "skip_monitoring" field to the value that was provided on create.
func (u *ErrorPassthroughRuleUpsert) UpdateSkipMonitoring() *ErrorPassthroughRuleUpsert {
u.SetExcluded(errorpassthroughrule.FieldSkipMonitoring)
return u
}
// SetDescription sets the "description" field.
func (u *ErrorPassthroughRuleUpsert) SetDescription(v string) *ErrorPassthroughRuleUpsert {
u.Set(errorpassthroughrule.FieldDescription, v)
......@@ -888,6 +925,20 @@ func (u *ErrorPassthroughRuleUpsertOne) ClearCustomMessage() *ErrorPassthroughRu
})
}
// SetSkipMonitoring sets the "skip_monitoring" field.
func (u *ErrorPassthroughRuleUpsertOne) SetSkipMonitoring(v bool) *ErrorPassthroughRuleUpsertOne {
return u.Update(func(s *ErrorPassthroughRuleUpsert) {
s.SetSkipMonitoring(v)
})
}
// UpdateSkipMonitoring sets the "skip_monitoring" field to the value that was provided on create.
func (u *ErrorPassthroughRuleUpsertOne) UpdateSkipMonitoring() *ErrorPassthroughRuleUpsertOne {
return u.Update(func(s *ErrorPassthroughRuleUpsert) {
s.UpdateSkipMonitoring()
})
}
// SetDescription sets the "description" field.
func (u *ErrorPassthroughRuleUpsertOne) SetDescription(v string) *ErrorPassthroughRuleUpsertOne {
return u.Update(func(s *ErrorPassthroughRuleUpsert) {
......@@ -1337,6 +1388,20 @@ func (u *ErrorPassthroughRuleUpsertBulk) ClearCustomMessage() *ErrorPassthroughR
})
}
// SetSkipMonitoring sets the "skip_monitoring" field.
func (u *ErrorPassthroughRuleUpsertBulk) SetSkipMonitoring(v bool) *ErrorPassthroughRuleUpsertBulk {
return u.Update(func(s *ErrorPassthroughRuleUpsert) {
s.SetSkipMonitoring(v)
})
}
// UpdateSkipMonitoring sets the "skip_monitoring" field to the value that was provided on create.
func (u *ErrorPassthroughRuleUpsertBulk) UpdateSkipMonitoring() *ErrorPassthroughRuleUpsertBulk {
return u.Update(func(s *ErrorPassthroughRuleUpsert) {
s.UpdateSkipMonitoring()
})
}
// SetDescription sets the "description" field.
func (u *ErrorPassthroughRuleUpsertBulk) SetDescription(v string) *ErrorPassthroughRuleUpsertBulk {
return u.Update(func(s *ErrorPassthroughRuleUpsert) {
......
backend/ent/errorpassthroughrule_update.go
View file @ 8da5fac6
......@@ -227,6 +227,20 @@ func (_u *ErrorPassthroughRuleUpdate) ClearCustomMessage() *ErrorPassthroughRule
return _u
}
// SetSkipMonitoring sets the "skip_monitoring" field.
func (_u *ErrorPassthroughRuleUpdate) SetSkipMonitoring(v bool) *ErrorPassthroughRuleUpdate {
_u.mutation.SetSkipMonitoring(v)
return _u
}
// SetNillableSkipMonitoring sets the "skip_monitoring" field if the given value is not nil.
func (_u *ErrorPassthroughRuleUpdate) SetNillableSkipMonitoring(v *bool) *ErrorPassthroughRuleUpdate {
if v != nil {
_u.SetSkipMonitoring(*v)
}
return _u
}
// SetDescription sets the "description" field.
func (_u *ErrorPassthroughRuleUpdate) SetDescription(v string) *ErrorPassthroughRuleUpdate {
_u.mutation.SetDescription(v)
......@@ -387,6 +401,9 @@ func (_u *ErrorPassthroughRuleUpdate) sqlSave(ctx context.Context) (_node int, e
if _u.mutation.CustomMessageCleared() {
_spec.ClearField(errorpassthroughrule.FieldCustomMessage, field.TypeString)
}
if value, ok := _u.mutation.SkipMonitoring(); ok {
_spec.SetField(errorpassthroughrule.FieldSkipMonitoring, field.TypeBool, value)
}
if value, ok := _u.mutation.Description(); ok {
_spec.SetField(errorpassthroughrule.FieldDescription, field.TypeString, value)
}
......@@ -611,6 +628,20 @@ func (_u *ErrorPassthroughRuleUpdateOne) ClearCustomMessage() *ErrorPassthroughR
return _u
}
// SetSkipMonitoring sets the "skip_monitoring" field.
func (_u *ErrorPassthroughRuleUpdateOne) SetSkipMonitoring(v bool) *ErrorPassthroughRuleUpdateOne {
_u.mutation.SetSkipMonitoring(v)
return _u
}
// SetNillableSkipMonitoring sets the "skip_monitoring" field if the given value is not nil.
func (_u *ErrorPassthroughRuleUpdateOne) SetNillableSkipMonitoring(v *bool) *ErrorPassthroughRuleUpdateOne {
if v != nil {
_u.SetSkipMonitoring(*v)
}
return _u
}
// SetDescription sets the "description" field.
func (_u *ErrorPassthroughRuleUpdateOne) SetDescription(v string) *ErrorPassthroughRuleUpdateOne {
_u.mutation.SetDescription(v)
......@@ -801,6 +832,9 @@ func (_u *ErrorPassthroughRuleUpdateOne) sqlSave(ctx context.Context) (_node *Er
if _u.mutation.CustomMessageCleared() {
_spec.ClearField(errorpassthroughrule.FieldCustomMessage, field.TypeString)
}
if value, ok := _u.mutation.SkipMonitoring(); ok {
_spec.SetField(errorpassthroughrule.FieldSkipMonitoring, field.TypeBool, value)
}
if value, ok := _u.mutation.Description(); ok {
_spec.SetField(errorpassthroughrule.FieldDescription, field.TypeString, value)
}
......
backend/ent/migrate/schema.go
View file @ 8da5fac6
......@@ -325,6 +325,7 @@ var (
{Name: "response_code", Type: field.TypeInt, Nullable: true},
{Name: "passthrough_body", Type: field.TypeBool, Default: true},
{Name: "custom_message", Type: field.TypeString, Nullable: true, Size: 2147483647},
{Name: "skip_monitoring", Type: field.TypeBool, Default: false},
{Name: "description", Type: field.TypeString, Nullable: true, Size: 2147483647},
}
// ErrorPassthroughRulesTable holds the schema information for the "error_passthrough_rules" table.
......
backend/ent/mutation.go
View file @ 8da5fac6
......@@ -5776,6 +5776,7 @@ type ErrorPassthroughRuleMutation struct {
addresponse_code *int
passthrough_body *bool
custom_message *string
skip_monitoring *bool
description *string
clearedFields map[string]struct{}
done bool
......@@ -6503,6 +6504,42 @@ func (m *ErrorPassthroughRuleMutation) ResetCustomMessage() {
delete(m.clearedFields, errorpassthroughrule.FieldCustomMessage)
}
 
// SetSkipMonitoring sets the "skip_monitoring" field.
func (m *ErrorPassthroughRuleMutation) SetSkipMonitoring(b bool) {
m.skip_monitoring = &b
}
// SkipMonitoring returns the value of the "skip_monitoring" field in the mutation.
func (m *ErrorPassthroughRuleMutation) SkipMonitoring() (r bool, exists bool) {
v := m.skip_monitoring
if v == nil {
return
}
return *v, true
}
// OldSkipMonitoring returns the old "skip_monitoring" field's value of the ErrorPassthroughRule entity.
// If the ErrorPassthroughRule object wasn't provided to the builder, the object is fetched from the database.
// An error is returned if the mutation operation is not UpdateOne, or the database query fails.
func (m *ErrorPassthroughRuleMutation) OldSkipMonitoring(ctx context.Context) (v bool, err error) {
if !m.op.Is(OpUpdateOne) {
return v, errors.New("OldSkipMonitoring is only allowed on UpdateOne operations")
}
if m.id == nil || m.oldValue == nil {
return v, errors.New("OldSkipMonitoring requires an ID field in the mutation")
}
oldValue, err := m.oldValue(ctx)
if err != nil {
return v, fmt.Errorf("querying old value for OldSkipMonitoring: %w", err)
}
return oldValue.SkipMonitoring, nil
}
// ResetSkipMonitoring resets all changes to the "skip_monitoring" field.
func (m *ErrorPassthroughRuleMutation) ResetSkipMonitoring() {
m.skip_monitoring = nil
}
// SetDescription sets the "description" field.
func (m *ErrorPassthroughRuleMutation) SetDescription(s string) {
m.description = &s
......@@ -6586,7 +6623,7 @@ func (m *ErrorPassthroughRuleMutation) Type() string {
// order to get all numeric fields that were incremented/decremented, call
// AddedFields().
func (m *ErrorPassthroughRuleMutation) Fields() []string {
fields := make([]string, 0, 14)
fields := make([]string, 0, 15)
if m.created_at != nil {
fields = append(fields, errorpassthroughrule.FieldCreatedAt)
}
......@@ -6626,6 +6663,9 @@ func (m *ErrorPassthroughRuleMutation) Fields() []string {
if m.custom_message != nil {
fields = append(fields, errorpassthroughrule.FieldCustomMessage)
}
if m.skip_monitoring != nil {
fields = append(fields, errorpassthroughrule.FieldSkipMonitoring)
}
if m.description != nil {
fields = append(fields, errorpassthroughrule.FieldDescription)
}
......@@ -6663,6 +6703,8 @@ func (m *ErrorPassthroughRuleMutation) Field(name string) (ent.Value, bool) {
return m.PassthroughBody()
case errorpassthroughrule.FieldCustomMessage:
return m.CustomMessage()
case errorpassthroughrule.FieldSkipMonitoring:
return m.SkipMonitoring()
case errorpassthroughrule.FieldDescription:
return m.Description()
}
......@@ -6700,6 +6742,8 @@ func (m *ErrorPassthroughRuleMutation) OldField(ctx context.Context, name string
return m.OldPassthroughBody(ctx)
case errorpassthroughrule.FieldCustomMessage:
return m.OldCustomMessage(ctx)
case errorpassthroughrule.FieldSkipMonitoring:
return m.OldSkipMonitoring(ctx)
case errorpassthroughrule.FieldDescription:
return m.OldDescription(ctx)
}
......@@ -6802,6 +6846,13 @@ func (m *ErrorPassthroughRuleMutation) SetField(name string, value ent.Value) er
}
m.SetCustomMessage(v)
return nil
case errorpassthroughrule.FieldSkipMonitoring:
v, ok := value.(bool)
if !ok {
return fmt.Errorf("unexpected type %T for field %s", value, name)
}
m.SetSkipMonitoring(v)
return nil
case errorpassthroughrule.FieldDescription:
v, ok := value.(string)
if !ok {
......@@ -6963,6 +7014,9 @@ func (m *ErrorPassthroughRuleMutation) ResetField(name string) error {
case errorpassthroughrule.FieldCustomMessage:
m.ResetCustomMessage()
return nil
case errorpassthroughrule.FieldSkipMonitoring:
m.ResetSkipMonitoring()
return nil
case errorpassthroughrule.FieldDescription:
m.ResetDescription()
return nil
......
backend/ent/runtime/runtime.go
View file @ 8da5fac6
......@@ -326,6 +326,10 @@ func init() {
errorpassthroughruleDescPassthroughBody := errorpassthroughruleFields[9].Descriptor()
// errorpassthroughrule.DefaultPassthroughBody holds the default value on creation for the passthrough_body field.
errorpassthroughrule.DefaultPassthroughBody = errorpassthroughruleDescPassthroughBody.Default.(bool)
// errorpassthroughruleDescSkipMonitoring is the schema descriptor for skip_monitoring field.
errorpassthroughruleDescSkipMonitoring := errorpassthroughruleFields[11].Descriptor()
// errorpassthroughrule.DefaultSkipMonitoring holds the default value on creation for the skip_monitoring field.
errorpassthroughrule.DefaultSkipMonitoring = errorpassthroughruleDescSkipMonitoring.Default.(bool)
groupMixin := schema.Group{}.Mixin()
groupMixinHooks1 := groupMixin[1].Hooks()
group.Hooks[0] = groupMixinHooks1[0]
......
backend/ent/schema/error_passthrough_rule.go
View file @ 8da5fac6
......@@ -105,6 +105,12 @@ func (ErrorPassthroughRule) Fields() []ent.Field {
Optional().
Nillable(),
// skip_monitoring: 是否跳过运维监控记录
// true: 匹配此规则的错误不会被记录到 ops_error_logs
// false: 正常记录到运维监控（默认行为）
field.Bool("skip_monitoring").
Default(false),
// description: 规则描述，用于说明规则的用途
field.Text("description").
Optional().
......
backend/internal/handler/admin/error_passthrough_handler.go
View file @ 8da5fac6
......@@ -32,6 +32,7 @@ type CreateErrorPassthroughRuleRequest struct {
ResponseCode *int `json:"response_code"`
PassthroughBody *bool `json:"passthrough_body"`
CustomMessage *string `json:"custom_message"`
SkipMonitoring *bool `json:"skip_monitoring"`
Description *string `json:"description"`
}
......@@ -48,6 +49,7 @@ type UpdateErrorPassthroughRuleRequest struct {
ResponseCode *int `json:"response_code"`
PassthroughBody *bool `json:"passthrough_body"`
CustomMessage *string `json:"custom_message"`
SkipMonitoring *bool `json:"skip_monitoring"`
Description *string `json:"description"`
}
......@@ -122,6 +124,9 @@ func (h *ErrorPassthroughHandler) Create(c *gin.Context) {
} else {
rule.PassthroughBody = true
}
if req.SkipMonitoring != nil {
rule.SkipMonitoring = *req.SkipMonitoring
}
rule.ResponseCode = req.ResponseCode
rule.CustomMessage = req.CustomMessage
rule.Description = req.Description
......@@ -190,6 +195,7 @@ func (h *ErrorPassthroughHandler) Update(c *gin.Context) {
ResponseCode: existing.ResponseCode,
PassthroughBody: existing.PassthroughBody,
CustomMessage: existing.CustomMessage,
SkipMonitoring: existing.SkipMonitoring,
Description: existing.Description,
}
......@@ -230,6 +236,9 @@ func (h *ErrorPassthroughHandler) Update(c *gin.Context) {
if req.Description != nil {
rule.Description = req.Description
}
if req.SkipMonitoring != nil {
rule.SkipMonitoring = *req.SkipMonitoring
}
// 确保切片不为 nil
if rule.ErrorCodes == nil {
......
backend/internal/handler/gateway_handler.go
View file @ 8da5fac6
......@@ -235,6 +235,7 @@ func (h *GatewayHandler) Messages(c *gin.Context) {
maxAccountSwitches := h.maxAccountSwitchesGemini
switchCount := 0
failedAccountIDs := make(map[int64]struct{})
sameAccountRetryCount := make(map[int64]int) // 同账号重试计数
var lastFailoverErr *service.UpstreamFailoverError
var forceCacheBilling bool // 粘性会话切换时的缓存计费标记
......@@ -358,11 +359,28 @@ func (h *GatewayHandler) Messages(c *gin.Context) {
if err != nil {
var failoverErr *service.UpstreamFailoverError
if errors.As(err, &failoverErr) {
failedAccountIDs[account.ID] = struct{}{}
lastFailoverErr = failoverErr
if needForceCacheBilling(hasBoundSession, failoverErr) {
forceCacheBilling = true
}
// 同账号重试：对 RetryableOnSameAccount 的临时性错误，先在同一账号上重试
if failoverErr.RetryableOnSameAccount && sameAccountRetryCount[account.ID] < maxSameAccountRetries {
sameAccountRetryCount[account.ID]++
log.Printf("Account %d: retryable error %d, same-account retry %d/%d",
account.ID, failoverErr.StatusCode, sameAccountRetryCount[account.ID], maxSameAccountRetries)
if !sleepSameAccountRetryDelay(c.Request.Context()) {
return
}
continue
}
// 同账号重试用尽，执行临时封禁并切换账号
if failoverErr.RetryableOnSameAccount {
h.gatewayService.TempUnscheduleRetryableError(c.Request.Context(), account.ID, failoverErr)
}
failedAccountIDs[account.ID] = struct{}{}
if switchCount >= maxAccountSwitches {
h.handleFailoverExhausted(c, failoverErr, service.PlatformGemini, streamStarted)
return
......@@ -426,6 +444,7 @@ func (h *GatewayHandler) Messages(c *gin.Context) {
maxAccountSwitches := h.maxAccountSwitches
switchCount := 0
failedAccountIDs := make(map[int64]struct{})
sameAccountRetryCount := make(map[int64]int) // 同账号重试计数
var lastFailoverErr *service.UpstreamFailoverError
retryWithFallback := false
var forceCacheBilling bool // 粘性会话切换时的缓存计费标记
......@@ -579,11 +598,28 @@ func (h *GatewayHandler) Messages(c *gin.Context) {
}
var failoverErr *service.UpstreamFailoverError
if errors.As(err, &failoverErr) {
failedAccountIDs[account.ID] = struct{}{}
lastFailoverErr = failoverErr
if needForceCacheBilling(hasBoundSession, failoverErr) {
forceCacheBilling = true
}
// 同账号重试：对 RetryableOnSameAccount 的临时性错误，先在同一账号上重试
if failoverErr.RetryableOnSameAccount && sameAccountRetryCount[account.ID] < maxSameAccountRetries {
sameAccountRetryCount[account.ID]++
log.Printf("Account %d: retryable error %d, same-account retry %d/%d",
account.ID, failoverErr.StatusCode, sameAccountRetryCount[account.ID], maxSameAccountRetries)
if !sleepSameAccountRetryDelay(c.Request.Context()) {
return
}
continue
}
// 同账号重试用尽，执行临时封禁并切换账号
if failoverErr.RetryableOnSameAccount {
h.gatewayService.TempUnscheduleRetryableError(c.Request.Context(), account.ID, failoverErr)
}
failedAccountIDs[account.ID] = struct{}{}
if switchCount >= maxAccountSwitches {
h.handleFailoverExhausted(c, failoverErr, account.Platform, streamStarted)
return
......@@ -863,6 +899,23 @@ func needForceCacheBilling(hasBoundSession bool, failoverErr *service.UpstreamFa
return hasBoundSession || (failoverErr != nil && failoverErr.ForceCacheBilling)
}
const (
// maxSameAccountRetries 同账号重试次数上限（针对 RetryableOnSameAccount 错误）
maxSameAccountRetries = 2
// sameAccountRetryDelay 同账号重试间隔
sameAccountRetryDelay = 500 * time.Millisecond
)
// sleepSameAccountRetryDelay 同账号重试固定延时，返回 false 表示 context 已取消。
func sleepSameAccountRetryDelay(ctx context.Context) bool {
select {
case <-ctx.Done():
return false
case <-time.After(sameAccountRetryDelay):
return true
}
}
// sleepFailoverDelay 账号切换线性递增延时：第1次0s、第2次1s、第3次2s…
// 返回 false 表示 context 已取消。
func sleepFailoverDelay(ctx context.Context, switchCount int) bool {
......@@ -918,6 +971,10 @@ func (h *GatewayHandler) handleFailoverExhausted(c *gin.Context, failoverErr *se
msg = *rule.CustomMessage
}
if rule.SkipMonitoring {
c.Set(service.OpsSkipPassthroughKey, true)
}
h.handleStreamingAwareError(c, respCode, "upstream_error", msg, streamStarted)
return
}
......
backend/internal/handler/gemini_v1beta_handler.go
View file @ 8da5fac6
......@@ -554,6 +554,10 @@ func (h *GatewayHandler) handleGeminiFailoverExhausted(c *gin.Context, failoverE
msg = *rule.CustomMessage
}
if rule.SkipMonitoring {
c.Set(service.OpsSkipPassthroughKey, true)
}
googleError(c, respCode, msg)
return
}
......
backend/internal/handler/openai_gateway_handler.go
View file @ 8da5fac6
......@@ -358,6 +358,10 @@ func (h *OpenAIGatewayHandler) handleFailoverExhausted(c *gin.Context, failoverE
msg = *rule.CustomMessage
}
if rule.SkipMonitoring {
c.Set(service.OpsSkipPassthroughKey, true)
}
h.handleStreamingAwareError(c, respCode, "upstream_error", msg, streamStarted)
return
}
......
backend/internal/handler/ops_error_logger.go
View file @ 8da5fac6
......@@ -537,6 +537,13 @@ func OpsErrorLoggerMiddleware(ops *service.OpsService) gin.HandlerFunc {
// Store request headers/body only when an upstream error occurred to keep overhead minimal.
entry.RequestHeadersJSON = extractOpsRetryRequestHeaders(c)
// Skip logging if a passthrough rule with skip_monitoring=true matched.
if v, ok := c.Get(service.OpsSkipPassthroughKey); ok {
if skip, _ := v.(bool); skip {
return
}
}
enqueueOpsErrorLog(ops, entry, requestBody)
return
}
......@@ -544,6 +551,13 @@ func OpsErrorLoggerMiddleware(ops *service.OpsService) gin.HandlerFunc {
body := w.buf.Bytes()
parsed := parseOpsErrorResponse(body)
// Skip logging if a passthrough rule with skip_monitoring=true matched.
if v, ok := c.Get(service.OpsSkipPassthroughKey); ok {
if skip, _ := v.(bool); skip {
return
}
}
// Skip logging if the error should be filtered based on settings
if shouldSkipOpsErrorLog(c.Request.Context(), ops, parsed.Message, string(body), c.Request.URL.Path) {
return
......
backend/internal/model/error_passthrough_rule.go
View file @ 8da5fac6
......@@ -18,6 +18,7 @@ type ErrorPassthroughRule struct {
ResponseCode *int `json:"response_code"` // 自定义状态码（passthrough_code=false 时使用）
PassthroughBody bool `json:"passthrough_body"` // 是否透传原始错误信息
CustomMessage *string `json:"custom_message"` // 自定义错误信息（passthrough_body=false 时使用）
SkipMonitoring bool `json:"skip_monitoring"` // 是否跳过运维监控记录
Description *string `json:"description"` // 规则描述
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
......
backend/internal/pkg/antigravity/client.go
View file @ 8da5fac6
......@@ -115,6 +115,23 @@ type LoadCodeAssistResponse struct {
IneligibleTiers []*IneligibleTier `json:"ineligibleTiers,omitempty"`
}
// OnboardUserRequest onboardUser 请求
type OnboardUserRequest struct {
TierID string `json:"tierId"`
Metadata struct {
IDEType string `json:"ideType"`
Platform string `json:"platform,omitempty"`
PluginType string `json:"pluginType,omitempty"`
} `json:"metadata"`
}
// OnboardUserResponse onboardUser 响应
type OnboardUserResponse struct {
Name string `json:"name,omitempty"`
Done bool `json:"done"`
Response map[string]any `json:"response,omitempty"`
}
// GetTier 获取账户类型
// 优先返回 paidTier（付费订阅级别），否则返回 currentTier
func (r *LoadCodeAssistResponse) GetTier() string {
......@@ -361,6 +378,117 @@ func (c *Client) LoadCodeAssist(ctx context.Context, accessToken string) (*LoadC
return nil, nil, lastErr
}
// OnboardUser 触发账号 onboarding，并返回 project_id
// 说明：
// 1) 部分账号 loadCodeAssist 不会立即返回 cloudaicompanionProject；
// 2) 这时需要调用 onboardUser 完成初始化，之后才能拿到 project_id。
func (c *Client) OnboardUser(ctx context.Context, accessToken, tierID string) (string, error) {
tierID = strings.TrimSpace(tierID)
if tierID == "" {
return "", fmt.Errorf("tier_id 为空")
}
reqBody := OnboardUserRequest{TierID: tierID}
reqBody.Metadata.IDEType = "ANTIGRAVITY"
reqBody.Metadata.Platform = "PLATFORM_UNSPECIFIED"
reqBody.Metadata.PluginType = "GEMINI"
bodyBytes, err := json.Marshal(reqBody)
if err != nil {
return "", fmt.Errorf("序列化请求失败: %w", err)
}
availableURLs := BaseURLs
var lastErr error
for urlIdx, baseURL := range availableURLs {
apiURL := baseURL + "/v1internal:onboardUser"
for attempt := 1; attempt <= 5; attempt++ {
req, err := http.NewRequestWithContext(ctx, http.MethodPost, apiURL, bytes.NewReader(bodyBytes))
if err != nil {
lastErr = fmt.Errorf("创建请求失败: %w", err)
break
}
req.Header.Set("Authorization", "Bearer "+accessToken)
req.Header.Set("Content-Type", "application/json")
req.Header.Set("User-Agent", UserAgent)
resp, err := c.httpClient.Do(req)
if err != nil {
lastErr = fmt.Errorf("onboardUser 请求失败: %w", err)
if shouldFallbackToNextURL(err, 0) && urlIdx < len(availableURLs)-1 {
log.Printf("[antigravity] onboardUser URL fallback: %s -> %s", baseURL, availableURLs[urlIdx+1])
break
}
return "", lastErr
}
respBodyBytes, err := io.ReadAll(resp.Body)
_ = resp.Body.Close()
if err != nil {
return "", fmt.Errorf("读取响应失败: %w", err)
}
if shouldFallbackToNextURL(nil, resp.StatusCode) && urlIdx < len(availableURLs)-1 {
log.Printf("[antigravity] onboardUser URL fallback (HTTP %d): %s -> %s", resp.StatusCode, baseURL, availableURLs[urlIdx+1])
break
}
if resp.StatusCode != http.StatusOK {
lastErr = fmt.Errorf("onboardUser 失败 (HTTP %d): %s", resp.StatusCode, string(respBodyBytes))
return "", lastErr
}
var onboardResp OnboardUserResponse
if err := json.Unmarshal(respBodyBytes, &onboardResp); err != nil {
lastErr = fmt.Errorf("onboardUser 响应解析失败: %w", err)
return "", lastErr
}
if onboardResp.Done {
if projectID := extractProjectIDFromOnboardResponse(onboardResp.Response); projectID != "" {
DefaultURLAvailability.MarkSuccess(baseURL)
return projectID, nil
}
lastErr = fmt.Errorf("onboardUser 完成但未返回 project_id")
return "", lastErr
}
// done=false 时等待后重试（与 CLIProxyAPI 行为一致）
select {
case <-time.After(2 * time.Second):
case <-ctx.Done():
return "", ctx.Err()
}
}
}
if lastErr != nil {
return "", lastErr
}
return "", fmt.Errorf("onboardUser 未返回 project_id")
}
func extractProjectIDFromOnboardResponse(resp map[string]any) string {
if len(resp) == 0 {
return ""
}
if v, ok := resp["cloudaicompanionProject"]; ok {
switch project := v.(type) {
case string:
return strings.TrimSpace(project)
case map[string]any:
if id, ok := project["id"].(string); ok {
return strings.TrimSpace(id)
}
}
}
return ""
}
// ModelQuotaInfo 模型配额信息
type ModelQuotaInfo struct {
RemainingFraction float64 `json:"remainingFraction"`
......
backend/internal/pkg/antigravity/client_test.go 0 → 100644
View file @ 8da5fac6
package antigravity
import (
"testing"
)
func TestExtractProjectIDFromOnboardResponse(t *testing.T) {
t.Parallel()
tests := []struct {
name string
resp map[string]any
want string
}{
{
name: "nil response",
resp: nil,
want: "",
},
{
name: "empty response",
resp: map[string]any{},
want: "",
},
{
name: "project as string",
resp: map[string]any{
"cloudaicompanionProject": "my-project-123",
},
want: "my-project-123",
},
{
name: "project as string with spaces",
resp: map[string]any{
"cloudaicompanionProject": " my-project-123 ",
},
want: "my-project-123",
},
{
name: "project as map with id",
resp: map[string]any{
"cloudaicompanionProject": map[string]any{
"id": "proj-from-map",
},
},
want: "proj-from-map",
},
{
name: "project as map without id",
resp: map[string]any{
"cloudaicompanionProject": map[string]any{
"name": "some-name",
},
},
want: "",
},
{
name: "missing cloudaicompanionProject key",
resp: map[string]any{
"otherField": "value",
},
want: "",
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
got := extractProjectIDFromOnboardResponse(tc.resp)
if got != tc.want {
t.Fatalf("extractProjectIDFromOnboardResponse() = %q, want %q", got, tc.want)
}
})
}
}
backend/internal/repository/error_passthrough_repo.go
View file @ 8da5fac6
......@@ -54,7 +54,8 @@ func (r *errorPassthroughRepository) Create(ctx context.Context, rule *model.Err
SetPriority(rule.Priority).
SetMatchMode(rule.MatchMode).
SetPassthroughCode(rule.PassthroughCode).
SetPassthroughBody(rule.PassthroughBody)
SetPassthroughBody(rule.PassthroughBody).
SetSkipMonitoring(rule.SkipMonitoring)
if len(rule.ErrorCodes) > 0 {
builder.SetErrorCodes(rule.ErrorCodes)
......@@ -90,7 +91,8 @@ func (r *errorPassthroughRepository) Update(ctx context.Context, rule *model.Err
SetPriority(rule.Priority).
SetMatchMode(rule.MatchMode).
SetPassthroughCode(rule.PassthroughCode).
SetPassthroughBody(rule.PassthroughBody)
SetPassthroughBody(rule.PassthroughBody).
SetSkipMonitoring(rule.SkipMonitoring)
// 处理可选字段
if len(rule.ErrorCodes) > 0 {
......@@ -149,6 +151,7 @@ func (r *errorPassthroughRepository) toModel(e *ent.ErrorPassthroughRule) *model
Platforms: e.Platforms,
PassthroughCode: e.PassthroughCode,
PassthroughBody: e.PassthroughBody,
SkipMonitoring: e.SkipMonitoring,
CreatedAt: e.CreatedAt,
UpdatedAt: e.UpdatedAt,
}
......
backend/internal/service/antigravity_gateway_service.go
View file @ 8da5fac6
......@@ -16,6 +16,7 @@ import (
"os"
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
......@@ -40,6 +41,12 @@ const (
antigravitySmartRetryMaxAttempts = 1 // 智能重试最大次数（仅重试 1 次，防止重复限流/长期等待）
antigravityDefaultRateLimitDuration = 30 * time.Second // 默认限流时间（无 retryDelay 时使用）
// MODEL_CAPACITY_EXHAUSTED 专用重试参数
// 模型容量不足时，所有账号共享同一容量池，切换账号无意义
// 使用固定 1s 间隔重试，最多重试 60 次
antigravityModelCapacityRetryMaxAttempts = 60
antigravityModelCapacityRetryWait = 1 * time.Second
// Google RPC 状态和类型常量
googleRPCStatusResourceExhausted = "RESOURCE_EXHAUSTED"
googleRPCStatusUnavailable = "UNAVAILABLE"
......@@ -60,6 +67,9 @@ const (
// 单账号 503 退避重试：原地重试的总累计等待时间上限
// 超过此上限将不再重试，直接返回 503
antigravitySingleAccountSmartRetryTotalMaxWait = 30 * time.Second
// MODEL_CAPACITY_EXHAUSTED 全局去重：重试全部失败后的 cooldown 时间
antigravityModelCapacityCooldown = 10 * time.Second
)
// antigravityPassthroughErrorMessages 透传给客户端的错误消息白名单（小写）
......@@ -68,8 +78,15 @@ var antigravityPassthroughErrorMessages = []string{
"prompt is too long",
}
// MODEL_CAPACITY_EXHAUSTED 全局去重：避免多个并发请求同时对同一模型进行容量耗尽重试
var (
modelCapacityExhaustedMu sync.RWMutex
modelCapacityExhaustedUntil = make(map[string]time.Time) // modelName -> cooldown until
)
const (
antigravityBillingModelEnv = "GATEWAY_ANTIGRAVITY_BILL_WITH_MAPPED_MODEL"
antigravityForwardBaseURLEnv = "GATEWAY_ANTIGRAVITY_FORWARD_BASE_URL"
antigravityFallbackSecondsEnv = "GATEWAY_ANTIGRAVITY_FALLBACK_COOLDOWN_SECONDS"
)
......@@ -131,6 +148,20 @@ type antigravityRetryLoopResult struct {
resp *http.Response
}
// resolveAntigravityForwardBaseURL 解析转发用 base URL。
// 默认使用 daily（ForwardBaseURLs 的首个地址）；当环境变量为 prod 时使用第二个地址。
func resolveAntigravityForwardBaseURL() string {
baseURLs := antigravity.ForwardBaseURLs()
if len(baseURLs) == 0 {
return ""
}
mode := strings.ToLower(strings.TrimSpace(os.Getenv(antigravityForwardBaseURLEnv)))
if mode == "prod" && len(baseURLs) > 1 {
return baseURLs[1]
}
return baseURLs[0]
}
// smartRetryAction 智能重试的处理结果
type smartRetryAction int
......@@ -158,7 +189,7 @@ func (s *AntigravityGatewayService) handleSmartRetry(p antigravityRetryLoopParam
}
// 判断是否触发智能重试
shouldSmartRetry, shouldRateLimitModel, waitDuration, modelName := shouldTriggerAntigravitySmartRetry(p.account, respBody)
shouldSmartRetry, shouldRateLimitModel, waitDuration, modelName, isModelCapacityExhausted := shouldTriggerAntigravitySmartRetry(p.account, respBody)
// 情况1: retryDelay >= 阈值，限流模型并切换账号
if shouldRateLimitModel {
......@@ -195,20 +226,48 @@ func (s *AntigravityGatewayService) handleSmartRetry(p antigravityRetryLoopParam
}
}
// 情况2: retryDelay < 阈值，智能重试（最多 antigravitySmartRetryMaxAttempts 次）
// 情况2: retryDelay < 阈值（或 MODEL_CAPACITY_EXHAUSTED），智能重试
if shouldSmartRetry {
var lastRetryResp *http.Response
var lastRetryBody []byte
for attempt := 1; attempt <= antigravitySmartRetryMaxAttempts; attempt++ {
// MODEL_CAPACITY_EXHAUSTED 使用独立的重试参数（60 次，固定 1s 间隔）
maxAttempts := antigravitySmartRetryMaxAttempts
if isModelCapacityExhausted {
maxAttempts = antigravityModelCapacityRetryMaxAttempts
waitDuration = antigravityModelCapacityRetryWait
// 全局去重：如果其他 goroutine 已在重试同一模型且尚在 cooldown 中，直接返回 503
if modelName != "" {
modelCapacityExhaustedMu.RLock()
cooldownUntil, exists := modelCapacityExhaustedUntil[modelName]
modelCapacityExhaustedMu.RUnlock()
if exists && time.Now().Before(cooldownUntil) {
log.Printf("%s status=%d model_capacity_exhausted_dedup model=%s account=%d cooldown_until=%v (skip retry)",
p.prefix, resp.StatusCode, modelName, p.account.ID, cooldownUntil.Format("15:04:05"))
return &smartRetryResult{
action: smartRetryActionBreakWithResp,
resp: &http.Response{
StatusCode: resp.StatusCode,
Header: resp.Header.Clone(),
Body: io.NopCloser(bytes.NewReader(respBody)),
},
}
}
}
}
for attempt := 1; attempt <= maxAttempts; attempt++ {
log.Printf("%s status=%d oauth_smart_retry attempt=%d/%d delay=%v model=%s account=%d",
p.prefix, resp.StatusCode, attempt, antigravitySmartRetryMaxAttempts, waitDuration, modelName, p.account.ID)
p.prefix, resp.StatusCode, attempt, maxAttempts, waitDuration, modelName, p.account.ID)
timer := time.NewTimer(waitDuration)
select {
case <-p.ctx.Done():
timer.Stop()
log.Printf("%s status=context_canceled_during_smart_retry", p.prefix)
return &smartRetryResult{action: smartRetryActionBreakWithResp, err: p.ctx.Err()}
case <-time.After(waitDuration):
case <-timer.C:
}
// 智能重试：创建新请求
......@@ -228,13 +287,19 @@ func (s *AntigravityGatewayService) handleSmartRetry(p antigravityRetryLoopParam
retryResp, retryErr := p.httpUpstream.Do(retryReq, p.proxyURL, p.account.ID, p.account.Concurrency)
if retryErr == nil && retryResp != nil && retryResp.StatusCode != http.StatusTooManyRequests && retryResp.StatusCode != http.StatusServiceUnavailable {
log.Printf("%s status=%d smart_retry_success attempt=%d/%d", p.prefix, retryResp.StatusCode, attempt, antigravitySmartRetryMaxAttempts)
log.Printf("%s status=%d smart_retry_success attempt=%d/%d", p.prefix, retryResp.StatusCode, attempt, maxAttempts)
// 重试成功，清除 MODEL_CAPACITY_EXHAUSTED cooldown
if isModelCapacityExhausted && modelName != "" {
modelCapacityExhaustedMu.Lock()
delete(modelCapacityExhaustedUntil, modelName)
modelCapacityExhaustedMu.Unlock()
}
return &smartRetryResult{action: smartRetryActionBreakWithResp, resp: retryResp}
}
// 网络错误时，继续重试
if retryErr != nil || retryResp == nil {
log.Printf("%s status=smart_retry_network_error attempt=%d/%d error=%v", p.prefix, attempt, antigravitySmartRetryMaxAttempts, retryErr)
log.Printf("%s status=smart_retry_network_error attempt=%d/%d error=%v", p.prefix, attempt, maxAttempts, retryErr)
continue
}
......@@ -244,13 +309,13 @@ func (s *AntigravityGatewayService) handleSmartRetry(p antigravityRetryLoopParam
}
lastRetryResp = retryResp
if retryResp != nil {
lastRetryBody, _ = io.ReadAll(io.LimitReader(retryResp.Body, 2<<20))
lastRetryBody, _ = io.ReadAll(io.LimitReader(retryResp.Body, 8<<10))
_ = retryResp.Body.Close()
}
// 解析新的重试信息，用于下次重试的等待时间
if attempt < antigravitySmartRetryMaxAttempts && lastRetryBody != nil {
newShouldRetry, _, newWaitDuration, _ := shouldTriggerAntigravitySmartRetry(p.account, lastRetryBody)
// 解析新的重试信息，用于下次重试的等待时间（MODEL_CAPACITY_EXHAUSTED 使用固定循环，跳过）
if !isModelCapacityExhausted && attempt < maxAttempts && lastRetryBody != nil {
newShouldRetry, _, newWaitDuration, _, _ := shouldTriggerAntigravitySmartRetry(p.account, lastRetryBody)
if newShouldRetry && newWaitDuration > 0 {
waitDuration = newWaitDuration
}
......@@ -267,6 +332,27 @@ func (s *AntigravityGatewayService) handleSmartRetry(p antigravityRetryLoopParam
retryBody = respBody
}
// MODEL_CAPACITY_EXHAUSTED：模型容量不足，切换账号无意义
// 直接返回上游错误响应，不设置模型限流，不切换账号
if isModelCapacityExhausted {
// 设置 cooldown，让后续请求快速失败，避免重复重试
if modelName != "" {
modelCapacityExhaustedMu.Lock()
modelCapacityExhaustedUntil[modelName] = time.Now().Add(antigravityModelCapacityCooldown)
modelCapacityExhaustedMu.Unlock()
}
log.Printf("%s status=%d smart_retry_exhausted_model_capacity attempts=%d model=%s account=%d body=%s (model capacity exhausted, not switching account)",
p.prefix, resp.StatusCode, maxAttempts, modelName, p.account.ID, truncateForLog(retryBody, 200))
return &smartRetryResult{
action: smartRetryActionBreakWithResp,
resp: &http.Response{
StatusCode: resp.StatusCode,
Header: resp.Header.Clone(),
Body: io.NopCloser(bytes.NewReader(retryBody)),
},
}
}
// 单账号 503 退避重试模式：智能重试耗尽后不设限流、不切换账号，
// 直接返回 503 让 Handler 层的单账号退避循环做最终处理。
if resp.StatusCode == http.StatusServiceUnavailable && isSingleAccountRetry(p.ctx) {
......@@ -283,7 +369,7 @@ func (s *AntigravityGatewayService) handleSmartRetry(p antigravityRetryLoopParam
}
log.Printf("%s status=%d smart_retry_exhausted attempts=%d model=%s account=%d upstream_retry_delay=%v body=%s (switch account)",
p.prefix, resp.StatusCode, antigravitySmartRetryMaxAttempts, modelName, p.account.ID, rateLimitDuration, truncateForLog(retryBody, 200))
p.prefix, resp.StatusCode, maxAttempts, modelName, p.account.ID, rateLimitDuration, truncateForLog(retryBody, 200))
resetAt := time.Now().Add(rateLimitDuration)
if p.accountRepo != nil && modelName != "" {
......@@ -367,11 +453,13 @@ func (s *AntigravityGatewayService) handleSingleAccountRetryInPlace(
log.Printf("%s status=%d single_account_503_retry attempt=%d/%d delay=%v total_waited=%v model=%s account=%d",
p.prefix, resp.StatusCode, attempt, antigravitySingleAccountSmartRetryMaxAttempts, waitDuration, totalWaited, modelName, p.account.ID)
timer := time.NewTimer(waitDuration)
select {
case <-p.ctx.Done():
timer.Stop()
log.Printf("%s status=context_canceled_during_single_account_retry", p.prefix)
return &smartRetryResult{action: smartRetryActionBreakWithResp, err: p.ctx.Err()}
case <-time.After(waitDuration):
case <-timer.C:
}
totalWaited += waitDuration
......@@ -405,12 +493,12 @@ func (s *AntigravityGatewayService) handleSingleAccountRetryInPlace(
_ = lastRetryResp.Body.Close()
}
lastRetryResp = retryResp
lastRetryBody, _ = io.ReadAll(io.LimitReader(retryResp.Body, 2<<20))
lastRetryBody, _ = io.ReadAll(io.LimitReader(retryResp.Body, 8<<10))
_ = retryResp.Body.Close()
// 解析新的重试信息，更新下次等待时间
if attempt < antigravitySingleAccountSmartRetryMaxAttempts && lastRetryBody != nil {
_, _, newWaitDuration, _ := shouldTriggerAntigravitySmartRetry(p.account, lastRetryBody)
_, _, newWaitDuration, _, _ := shouldTriggerAntigravitySmartRetry(p.account, lastRetryBody)
if newWaitDuration > 0 {
waitDuration = newWaitDuration
if waitDuration > antigravitySingleAccountSmartRetryMaxWait {
......@@ -466,10 +554,11 @@ func (s *AntigravityGatewayService) antigravityRetryLoop(p antigravityRetryLoopP
}
}
availableURLs := antigravity.DefaultURLAvailability.GetAvailableURLs()
if len(availableURLs) == 0 {
availableURLs = antigravity.BaseURLs
baseURL := resolveAntigravityForwardBaseURL()
if baseURL == "" {
return nil, errors.New("no antigravity forward base url configured")
}
availableURLs := []string{baseURL}
var resp *http.Response
var usedBaseURL string
......@@ -907,11 +996,11 @@ func (s *AntigravityGatewayService) TestConnection(ctx context.Context, account
proxyURL = account.Proxy.URL()
}
// URL fallback 循环
availableURLs := antigravity.DefaultURLAvailability.GetAvailableURLs()
if len(availableURLs) == 0 {
availableURLs = antigravity.BaseURLs // 所有 URL 都不可用时，重试所有
baseURL := resolveAntigravityForwardBaseURL()
if baseURL == "" {
return nil, errors.New("no antigravity forward base url configured")
}
availableURLs := []string{baseURL}
var lastErr error
for urlIdx, baseURL := range availableURLs {
......@@ -1376,7 +1465,7 @@ func (s *AntigravityGatewayService) Forward(ctx context.Context, c *gin.Context,
break
}
retryBody, _ := io.ReadAll(io.LimitReader(retryResp.Body, 2<<20))
retryBody, _ := io.ReadAll(io.LimitReader(retryResp.Body, 8<<10))
_ = retryResp.Body.Close()
if retryResp.StatusCode == http.StatusTooManyRequests {
retryBaseURL := ""
......@@ -1457,6 +1546,27 @@ func (s *AntigravityGatewayService) Forward(ctx context.Context, c *gin.Context,
s.handleUpstreamError(ctx, prefix, account, resp.StatusCode, resp.Header, respBody, originalModel, 0, "", isStickySession)
// 精确匹配服务端配置类 400 错误，触发同账号重试 + failover
if resp.StatusCode == http.StatusBadRequest {
msg := strings.ToLower(strings.TrimSpace(extractAntigravityErrorMessage(respBody)))
if isGoogleProjectConfigError(msg) {
upstreamMsg := sanitizeUpstreamErrorMessage(strings.TrimSpace(extractAntigravityErrorMessage(respBody)))
upstreamDetail := s.getUpstreamErrorDetail(respBody)
log.Printf("%s status=400 google_config_error failover=true upstream_message=%q account=%d", prefix, upstreamMsg, account.ID)
appendOpsUpstreamError(c, OpsUpstreamErrorEvent{
Platform: account.Platform,
AccountID: account.ID,
AccountName: account.Name,
UpstreamStatusCode: resp.StatusCode,
UpstreamRequestID: resp.Header.Get("x-request-id"),
Kind: "failover",
Message: upstreamMsg,
Detail: upstreamDetail,
})
return nil, &UpstreamFailoverError{StatusCode: resp.StatusCode, ResponseBody: respBody, RetryableOnSameAccount: true}
}
}
if s.shouldFailoverUpstreamError(resp.StatusCode) {
upstreamMsg := strings.TrimSpace(extractAntigravityErrorMessage(respBody))
upstreamMsg = sanitizeUpstreamErrorMessage(upstreamMsg)
......@@ -1997,6 +2107,22 @@ func (s *AntigravityGatewayService) ForwardGemini(ctx context.Context, c *gin.Co
// Always record upstream context for Ops error logs, even when we will failover.
setOpsUpstreamError(c, resp.StatusCode, upstreamMsg, upstreamDetail)
// 精确匹配服务端配置类 400 错误，触发同账号重试 + failover
if resp.StatusCode == http.StatusBadRequest && isGoogleProjectConfigError(strings.ToLower(upstreamMsg)) {
log.Printf("%s status=400 google_config_error failover=true upstream_message=%q account=%d", prefix, upstreamMsg, account.ID)
appendOpsUpstreamError(c, OpsUpstreamErrorEvent{
Platform: account.Platform,
AccountID: account.ID,
AccountName: account.Name,
UpstreamStatusCode: resp.StatusCode,
UpstreamRequestID: requestID,
Kind: "failover",
Message: upstreamMsg,
Detail: upstreamDetail,
})
return nil, &UpstreamFailoverError{StatusCode: resp.StatusCode, ResponseBody: unwrappedForOps, RetryableOnSameAccount: true}
}
if s.shouldFailoverUpstreamError(resp.StatusCode) {
appendOpsUpstreamError(c, OpsUpstreamErrorEvent{
Platform: account.Platform,
......@@ -2092,6 +2218,44 @@ func (s *AntigravityGatewayService) shouldFailoverUpstreamError(statusCode int)
}
}
// isGoogleProjectConfigError 判断（已提取的小写）错误消息是否属于 Google 服务端配置类问题。
// 只精确匹配已知的服务端侧错误，避免对客户端请求错误做无意义重试。
// 适用于所有走 Google 后端的平台（Antigravity、Gemini）。
func isGoogleProjectConfigError(lowerMsg string) bool {
// Google 间歇性 Bug：Project ID 有效但被临时识别失败
return strings.Contains(lowerMsg, "invalid project resource name")
}
// googleConfigErrorCooldown 服务端配置类 400 错误的临时封禁时长
const googleConfigErrorCooldown = 1 * time.Minute
// tempUnscheduleGoogleConfigError 对服务端配置类 400 错误触发临时封禁，
// 避免短时间内反复调度到同一个有问题的账号。
func tempUnscheduleGoogleConfigError(ctx context.Context, repo AccountRepository, accountID int64, logPrefix string) {
until := time.Now().Add(googleConfigErrorCooldown)
reason := "400: invalid project resource name (auto temp-unschedule 1m)"
if err := repo.SetTempUnschedulable(ctx, accountID, until, reason); err != nil {
log.Printf("%s temp_unschedule_failed account=%d error=%v", logPrefix, accountID, err)
} else {
log.Printf("%s temp_unscheduled account=%d until=%v reason=%q", logPrefix, accountID, until.Format("15:04:05"), reason)
}
}
// emptyResponseCooldown 空流式响应的临时封禁时长
const emptyResponseCooldown = 1 * time.Minute
// tempUnscheduleEmptyResponse 对空流式响应触发临时封禁，
// 避免短时间内反复调度到同一个返回空响应的账号。
func tempUnscheduleEmptyResponse(ctx context.Context, repo AccountRepository, accountID int64, logPrefix string) {
until := time.Now().Add(emptyResponseCooldown)
reason := "empty stream response (auto temp-unschedule 1m)"
if err := repo.SetTempUnschedulable(ctx, accountID, until, reason); err != nil {
log.Printf("%s temp_unschedule_failed account=%d error=%v", logPrefix, accountID, err)
} else {
log.Printf("%s temp_unscheduled account=%d until=%v reason=%q", logPrefix, accountID, until.Format("15:04:05"), reason)
}
}
// sleepAntigravityBackoffWithContext 带 context 取消检查的退避等待
// 返回 true 表示正常完成等待，false 表示 context 已取消
func sleepAntigravityBackoffWithContext(ctx context.Context, attempt int) bool {
......@@ -2108,10 +2272,12 @@ func sleepAntigravityBackoffWithContext(ctx context.Context, attempt int) bool {
sleepFor = 0
}
timer := time.NewTimer(sleepFor)
select {
case <-ctx.Done():
timer.Stop()
return false
case <-time.After(sleepFor):
case <-timer.C:
return true
}
}
......@@ -2156,8 +2322,9 @@ func antigravityFallbackCooldownSeconds() (time.Duration, bool) {
// antigravitySmartRetryInfo 智能重试所需的信息
type antigravitySmartRetryInfo struct {
RetryDelay time.Duration // 重试延迟时间
ModelName string // 限流的模型名称（如 "claude-sonnet-4-5"）
RetryDelay time.Duration // 重试延迟时间
ModelName string // 限流的模型名称（如 "claude-sonnet-4-5"）
IsModelCapacityExhausted bool // 是否为模型容量不足（MODEL_CAPACITY_EXHAUSTED）
}
// parseAntigravitySmartRetryInfo 解析 Google RPC RetryInfo 和 ErrorInfo 信息
......@@ -2272,31 +2439,40 @@ func parseAntigravitySmartRetryInfo(body []byte) *antigravitySmartRetryInfo {
}
return &antigravitySmartRetryInfo{
RetryDelay: retryDelay,
ModelName: modelName,
RetryDelay: retryDelay,
ModelName: modelName,
IsModelCapacityExhausted: hasModelCapacityExhausted,
}
}
// shouldTriggerAntigravitySmartRetry 判断是否应该触发智能重试
// 返回：
// - shouldRetry: 是否应该智能重试（retryDelay < antigravityRateLimitThreshold）
// - shouldRateLimitModel: 是否应该限流模型（retryDelay >= antigravityRateLimitThreshold）
// - waitDuration: 等待时间（智能重试时使用，shouldRateLimitModel=true 时为 0）
// - shouldRetry: 是否应该智能重试（retryDelay < antigravityRateLimitThreshold，或 MODEL_CAPACITY_EXHAUSTED）
// - shouldRateLimitModel: 是否应该限流模型并切换账号（仅 RATE_LIMIT_EXCEEDED 且 retryDelay >= 阈值）
// - waitDuration: 等待时间
// - modelName: 限流的模型名称
func shouldTriggerAntigravitySmartRetry(account *Account, respBody []byte) (shouldRetry bool, shouldRateLimitModel bool, waitDuration time.Duration, modelName string) {
// - isModelCapacityExhausted: 是否为模型容量不足（MODEL_CAPACITY_EXHAUSTED）
func shouldTriggerAntigravitySmartRetry(account *Account, respBody []byte) (shouldRetry bool, shouldRateLimitModel bool, waitDuration time.Duration, modelName string, isModelCapacityExhausted bool) {
if account.Platform != PlatformAntigravity {
return false, false, 0, ""
return false, false, 0, "", false
}
info := parseAntigravitySmartRetryInfo(respBody)
if info == nil {
return false, false, 0, ""
return false, false, 0, "", false
}
// MODEL_CAPACITY_EXHAUSTED（模型容量不足）：所有账号共享同一模型容量池
// 切换账号无意义，使用固定 1s 间隔重试
if info.IsModelCapacityExhausted {
return true, false, antigravityModelCapacityRetryWait, info.ModelName, true
}
// RATE_LIMIT_EXCEEDED（账号级限流）：
// retryDelay >= 阈值：直接限流模型，不重试
// 注意：如果上游未提供 retryDelay，parseAntigravitySmartRetryInfo 已设置为默认 30s
if info.RetryDelay >= antigravityRateLimitThreshold {
return false, true, info.RetryDelay, info.ModelName
return false, true, info.RetryDelay, info.ModelName, false
}
// retryDelay < 阈值：智能重试
......@@ -2305,7 +2481,7 @@ func shouldTriggerAntigravitySmartRetry(account *Account, respBody []byte) (shou
waitDuration = antigravitySmartRetryMinWait
}
return true, false, waitDuration, info.ModelName
return true, false, waitDuration, info.ModelName, false
}
// handleModelRateLimitParams 模型级限流处理参数
......@@ -2331,8 +2507,9 @@ type handleModelRateLimitResult struct {
// handleModelRateLimit 处理模型级限流（在原有逻辑之前调用）
// 仅处理 429/503，解析模型名和 retryDelay
// - retryDelay < antigravityRateLimitThreshold: 返回 ShouldRetry=true，由调用方等待后重试
// - retryDelay >= antigravityRateLimitThreshold: 设置模型限流 + 清除粘性会话 + 返回 SwitchError
// - MODEL_CAPACITY_EXHAUSTED: 返回 Handled=true（实际重试由 handleSmartRetry 处理）
// - RATE_LIMIT_EXCEEDED + retryDelay < 阈值: 返回 ShouldRetry=true，由调用方等待后重试
// - RATE_LIMIT_EXCEEDED + retryDelay >= 阈值: 设置模型限流 + 清除粘性会话 + 返回 SwitchError
func (s *AntigravityGatewayService) handleModelRateLimit(p *handleModelRateLimitParams) *handleModelRateLimitResult {
if p.statusCode != 429 && p.statusCode != 503 {
return &handleModelRateLimitResult{Handled: false}
......@@ -2343,7 +2520,17 @@ func (s *AntigravityGatewayService) handleModelRateLimit(p *handleModelRateLimit
return &handleModelRateLimitResult{Handled: false}
}
// < antigravityRateLimitThreshold: 等待后重试
// MODEL_CAPACITY_EXHAUSTED：模型容量不足，所有账号共享同一容量池
// 切换账号无意义，不设置模型限流（实际重试由 handleSmartRetry 处理）
if info.IsModelCapacityExhausted {
log.Printf("%s status=%d model_capacity_exhausted model=%s (not switching account, retry handled by smart retry)",
p.prefix, p.statusCode, info.ModelName)
return &handleModelRateLimitResult{
Handled: true,
}
}
// RATE_LIMIT_EXCEEDED: < antigravityRateLimitThreshold: 等待后重试
if info.RetryDelay < antigravityRateLimitThreshold {
log.Printf("%s status=%d model_rate_limit_wait model=%s wait=%v",
p.prefix, p.statusCode, info.ModelName, info.RetryDelay)
......@@ -2354,7 +2541,7 @@ func (s *AntigravityGatewayService) handleModelRateLimit(p *handleModelRateLimit
}
}
// >= antigravityRateLimitThreshold: 设置限流 + 清除粘性会话 + 切换账号
// RATE_LIMIT_EXCEEDED: >= antigravityRateLimitThreshold: 设置限流 + 清除粘性会话 + 切换账号
s.setModelRateLimitAndClearSession(p, info)
return &handleModelRateLimitResult{
......@@ -2906,9 +3093,14 @@ returnResponse:
// 选择最后一个有效响应
finalResponse := pickGeminiCollectResult(last, lastWithParts)
// 处理空响应情况
// 处理空响应情况 — 触发同账号重试 + failover 切换账号
if last == nil && lastWithParts == nil {
log.Printf("[antigravity-Forward] warning: empty stream response, no valid chunks received")
log.Printf("[antigravity-Forward] warning: empty stream response (gemini non-stream), triggering failover")
return nil, &UpstreamFailoverError{
StatusCode: http.StatusBadGateway,
ResponseBody: []byte(`{"error":"empty stream response from upstream"}`),
RetryableOnSameAccount: true,
}
}
// 如果收集到了图片 parts，需要合并到最终响应中
......@@ -3126,6 +3318,21 @@ func (s *AntigravityGatewayService) writeMappedClaudeError(c *gin.Context, accou
log.Printf("[antigravity-Forward] upstream_error status=%d body=%s", upstreamStatus, truncateForLog(body, maxBytes))
}
// 检查错误透传规则
if ptStatus, ptErrType, ptErrMsg, matched := applyErrorPassthroughRule(
c, account.Platform, upstreamStatus, body,
0, "", "",
); matched {
c.JSON(ptStatus, gin.H{
"type": "error",
"error": gin.H{"type": ptErrType, "message": ptErrMsg},
})
if upstreamMsg == "" {
return fmt.Errorf("upstream error: %d", upstreamStatus)
}
return fmt.Errorf("upstream error: %d message=%s", upstreamStatus, upstreamMsg)
}
var statusCode int
var errType, errMsg string
......@@ -3323,10 +3530,14 @@ returnResponse:
// 选择最后一个有效响应
finalResponse := pickGeminiCollectResult(last, lastWithParts)
// 处理空响应情况
// 处理空响应情况 — 触发同账号重试 + failover 切换账号
if last == nil && lastWithParts == nil {
log.Printf("[antigravity-Forward] warning: empty stream response, no valid chunks received")
return nil, s.writeClaudeError(c, http.StatusBadGateway, "upstream_error", "Empty response from upstream")
log.Printf("[antigravity-Forward] warning: empty stream response (claude non-stream), triggering failover")
return nil, &UpstreamFailoverError{
StatusCode: http.StatusBadGateway,
ResponseBody: []byte(`{"error":"empty stream response from upstream"}`),
RetryableOnSameAccount: true,
}
}
// 将收集的所有 parts 合并到最终响应中
......
backend/internal/service/antigravity_oauth_service.go
View file @ 8da5fac6
......@@ -273,12 +273,21 @@ func (s *AntigravityOAuthService) loadProjectIDWithRetry(ctx context.Context, ac
}
client := antigravity.NewClient(proxyURL)
loadResp, _, err := client.LoadCodeAssist(ctx, accessToken)
loadResp, loadRaw, err := client.LoadCodeAssist(ctx, accessToken)
if err == nil && loadResp != nil && loadResp.CloudAICompanionProject != "" {
return loadResp.CloudAICompanionProject, nil
}
if err == nil {
if projectID, onboardErr := tryOnboardProjectID(ctx, client, accessToken, loadRaw); onboardErr == nil && projectID != "" {
return projectID, nil
} else if onboardErr != nil {
lastErr = onboardErr
continue
}
}
// 记录错误
if err != nil {
lastErr = err
......@@ -292,6 +301,65 @@ func (s *AntigravityOAuthService) loadProjectIDWithRetry(ctx context.Context, ac
return "", fmt.Errorf("获取 project_id 失败 (重试 %d 次后): %w", maxRetries, lastErr)
}
func tryOnboardProjectID(ctx context.Context, client *antigravity.Client, accessToken string, loadRaw map[string]any) (string, error) {
tierID := resolveDefaultTierID(loadRaw)
if tierID == "" {
return "", fmt.Errorf("loadCodeAssist 未返回可用的默认 tier")
}
projectID, err := client.OnboardUser(ctx, accessToken, tierID)
if err != nil {
return "", fmt.Errorf("onboardUser 失败 (tier=%s): %w", tierID, err)
}
return projectID, nil
}
func resolveDefaultTierID(loadRaw map[string]any) string {
if len(loadRaw) == 0 {
return ""
}
rawTiers, ok := loadRaw["allowedTiers"]
if !ok {
return ""
}
tiers, ok := rawTiers.([]any)
if !ok {
return ""
}
for _, rawTier := range tiers {
tier, ok := rawTier.(map[string]any)
if !ok {
continue
}
if isDefault, _ := tier["isDefault"].(bool); !isDefault {
continue
}
if id, ok := tier["id"].(string); ok {
id = strings.TrimSpace(id)
if id != "" {
return id
}
}
}
return ""
}
// FillProjectID 仅获取 project_id，不刷新 OAuth token
func (s *AntigravityOAuthService) FillProjectID(ctx context.Context, account *Account, accessToken string) (string, error) {
var proxyURL string
if account.ProxyID != nil {
proxy, err := s.proxyRepo.GetByID(ctx, *account.ProxyID)
if err == nil && proxy != nil {
proxyURL = proxy.URL()
}
}
return s.loadProjectIDWithRetry(ctx, accessToken, proxyURL, 3)
}
// BuildAccountCredentials 构建账户凭证
func (s *AntigravityOAuthService) BuildAccountCredentials(tokenInfo *AntigravityTokenInfo) map[string]any {
creds := map[string]any{
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment