Commit e51a3288 authored by yangjianbo's avatar yangjianbo
Browse files

merge: 合并 test 分支到 test-dev,解决冲突

解决的冲突文件:
- wire_gen.go: 合并 ConcurrencyService/CRSSyncService 参数和 userAttributeHandler
- gateway_handler.go: 合并 pkg/errors 和 antigravity 导入
- gateway_service.go: 合并 validateUpstreamBaseURL 和 GetAvailableModels
- config.example.yaml: 合并 billing/turnstile 配置和额外 gateway 选项

🤖 Generated with [Claude Code](https://claude.com/claude-code

)
Co-Authored-By: default avatarClaude Opus 4.5 <noreply@anthropic.com>
parents 25e16326 8a50ca59
......@@ -13,6 +13,7 @@ import (
"log"
"net/http"
"regexp"
"sort"
"strconv"
"strings"
"time"
......@@ -82,6 +83,7 @@ type OpenAIGatewayService struct {
userSubRepo UserSubscriptionRepository
cache GatewayCache
cfg *config.Config
concurrencyService *ConcurrencyService
billingService *BillingService
rateLimitService *RateLimitService
billingCacheService *BillingCacheService
......@@ -97,6 +99,7 @@ func NewOpenAIGatewayService(
userSubRepo UserSubscriptionRepository,
cache GatewayCache,
cfg *config.Config,
concurrencyService *ConcurrencyService,
billingService *BillingService,
rateLimitService *RateLimitService,
billingCacheService *BillingCacheService,
......@@ -110,6 +113,7 @@ func NewOpenAIGatewayService(
userSubRepo: userSubRepo,
cache: cache,
cfg: cfg,
concurrencyService: concurrencyService,
billingService: billingService,
rateLimitService: rateLimitService,
billingCacheService: billingCacheService,
......@@ -128,6 +132,14 @@ func (s *OpenAIGatewayService) GenerateSessionHash(c *gin.Context) string {
return hex.EncodeToString(hash[:])
}
// BindStickySession sets session -> account binding with standard TTL.
func (s *OpenAIGatewayService) BindStickySession(ctx context.Context, sessionHash string, accountID int64) error {
if sessionHash == "" || accountID <= 0 {
return nil
}
return s.cache.SetSessionAccountID(ctx, "openai:"+sessionHash, accountID, openaiStickySessionTTL)
}
// SelectAccount selects an OpenAI account with sticky session support
func (s *OpenAIGatewayService) SelectAccount(ctx context.Context, groupID *int64, sessionHash string) (*Account, error) {
return s.SelectAccountForModel(ctx, groupID, sessionHash, "")
......@@ -220,6 +232,254 @@ func (s *OpenAIGatewayService) SelectAccountForModelWithExclusions(ctx context.C
return selected, nil
}
// SelectAccountWithLoadAwareness selects an account with load-awareness and wait plan.
func (s *OpenAIGatewayService) SelectAccountWithLoadAwareness(ctx context.Context, groupID *int64, sessionHash string, requestedModel string, excludedIDs map[int64]struct{}) (*AccountSelectionResult, error) {
cfg := s.schedulingConfig()
var stickyAccountID int64
if sessionHash != "" && s.cache != nil {
if accountID, err := s.cache.GetSessionAccountID(ctx, "openai:"+sessionHash); err == nil {
stickyAccountID = accountID
}
}
if s.concurrencyService == nil || !cfg.LoadBatchEnabled {
account, err := s.SelectAccountForModelWithExclusions(ctx, groupID, sessionHash, requestedModel, excludedIDs)
if err != nil {
return nil, err
}
result, err := s.tryAcquireAccountSlot(ctx, account.ID, account.Concurrency)
if err == nil && result.Acquired {
return &AccountSelectionResult{
Account: account,
Acquired: true,
ReleaseFunc: result.ReleaseFunc,
}, nil
}
if stickyAccountID > 0 && stickyAccountID == account.ID && s.concurrencyService != nil {
waitingCount, _ := s.concurrencyService.GetAccountWaitingCount(ctx, account.ID)
if waitingCount < cfg.StickySessionMaxWaiting {
return &AccountSelectionResult{
Account: account,
WaitPlan: &AccountWaitPlan{
AccountID: account.ID,
MaxConcurrency: account.Concurrency,
Timeout: cfg.StickySessionWaitTimeout,
MaxWaiting: cfg.StickySessionMaxWaiting,
},
}, nil
}
}
return &AccountSelectionResult{
Account: account,
WaitPlan: &AccountWaitPlan{
AccountID: account.ID,
MaxConcurrency: account.Concurrency,
Timeout: cfg.FallbackWaitTimeout,
MaxWaiting: cfg.FallbackMaxWaiting,
},
}, nil
}
accounts, err := s.listSchedulableAccounts(ctx, groupID)
if err != nil {
return nil, err
}
if len(accounts) == 0 {
return nil, errors.New("no available accounts")
}
isExcluded := func(accountID int64) bool {
if excludedIDs == nil {
return false
}
_, excluded := excludedIDs[accountID]
return excluded
}
// ============ Layer 1: Sticky session ============
if sessionHash != "" {
accountID, err := s.cache.GetSessionAccountID(ctx, "openai:"+sessionHash)
if err == nil && accountID > 0 && !isExcluded(accountID) {
account, err := s.accountRepo.GetByID(ctx, accountID)
if err == nil && account.IsSchedulable() && account.IsOpenAI() &&
(requestedModel == "" || account.IsModelSupported(requestedModel)) {
result, err := s.tryAcquireAccountSlot(ctx, accountID, account.Concurrency)
if err == nil && result.Acquired {
_ = s.cache.RefreshSessionTTL(ctx, "openai:"+sessionHash, openaiStickySessionTTL)
return &AccountSelectionResult{
Account: account,
Acquired: true,
ReleaseFunc: result.ReleaseFunc,
}, nil
}
waitingCount, _ := s.concurrencyService.GetAccountWaitingCount(ctx, accountID)
if waitingCount < cfg.StickySessionMaxWaiting {
return &AccountSelectionResult{
Account: account,
WaitPlan: &AccountWaitPlan{
AccountID: accountID,
MaxConcurrency: account.Concurrency,
Timeout: cfg.StickySessionWaitTimeout,
MaxWaiting: cfg.StickySessionMaxWaiting,
},
}, nil
}
}
}
}
// ============ Layer 2: Load-aware selection ============
candidates := make([]*Account, 0, len(accounts))
for i := range accounts {
acc := &accounts[i]
if isExcluded(acc.ID) {
continue
}
if requestedModel != "" && !acc.IsModelSupported(requestedModel) {
continue
}
candidates = append(candidates, acc)
}
if len(candidates) == 0 {
return nil, errors.New("no available accounts")
}
accountLoads := make([]AccountWithConcurrency, 0, len(candidates))
for _, acc := range candidates {
accountLoads = append(accountLoads, AccountWithConcurrency{
ID: acc.ID,
MaxConcurrency: acc.Concurrency,
})
}
loadMap, err := s.concurrencyService.GetAccountsLoadBatch(ctx, accountLoads)
if err != nil {
ordered := append([]*Account(nil), candidates...)
sortAccountsByPriorityAndLastUsed(ordered, false)
for _, acc := range ordered {
result, err := s.tryAcquireAccountSlot(ctx, acc.ID, acc.Concurrency)
if err == nil && result.Acquired {
if sessionHash != "" {
_ = s.cache.SetSessionAccountID(ctx, "openai:"+sessionHash, acc.ID, openaiStickySessionTTL)
}
return &AccountSelectionResult{
Account: acc,
Acquired: true,
ReleaseFunc: result.ReleaseFunc,
}, nil
}
}
} else {
type accountWithLoad struct {
account *Account
loadInfo *AccountLoadInfo
}
var available []accountWithLoad
for _, acc := range candidates {
loadInfo := loadMap[acc.ID]
if loadInfo == nil {
loadInfo = &AccountLoadInfo{AccountID: acc.ID}
}
if loadInfo.LoadRate < 100 {
available = append(available, accountWithLoad{
account: acc,
loadInfo: loadInfo,
})
}
}
if len(available) > 0 {
sort.SliceStable(available, func(i, j int) bool {
a, b := available[i], available[j]
if a.account.Priority != b.account.Priority {
return a.account.Priority < b.account.Priority
}
if a.loadInfo.LoadRate != b.loadInfo.LoadRate {
return a.loadInfo.LoadRate < b.loadInfo.LoadRate
}
switch {
case a.account.LastUsedAt == nil && b.account.LastUsedAt != nil:
return true
case a.account.LastUsedAt != nil && b.account.LastUsedAt == nil:
return false
case a.account.LastUsedAt == nil && b.account.LastUsedAt == nil:
return false
default:
return a.account.LastUsedAt.Before(*b.account.LastUsedAt)
}
})
for _, item := range available {
result, err := s.tryAcquireAccountSlot(ctx, item.account.ID, item.account.Concurrency)
if err == nil && result.Acquired {
if sessionHash != "" {
_ = s.cache.SetSessionAccountID(ctx, "openai:"+sessionHash, item.account.ID, openaiStickySessionTTL)
}
return &AccountSelectionResult{
Account: item.account,
Acquired: true,
ReleaseFunc: result.ReleaseFunc,
}, nil
}
}
}
}
// ============ Layer 3: Fallback wait ============
sortAccountsByPriorityAndLastUsed(candidates, false)
for _, acc := range candidates {
return &AccountSelectionResult{
Account: acc,
WaitPlan: &AccountWaitPlan{
AccountID: acc.ID,
MaxConcurrency: acc.Concurrency,
Timeout: cfg.FallbackWaitTimeout,
MaxWaiting: cfg.FallbackMaxWaiting,
},
}, nil
}
return nil, errors.New("no available accounts")
}
func (s *OpenAIGatewayService) listSchedulableAccounts(ctx context.Context, groupID *int64) ([]Account, error) {
var accounts []Account
var err error
if s.cfg != nil && s.cfg.RunMode == config.RunModeSimple {
accounts, err = s.accountRepo.ListSchedulableByPlatform(ctx, PlatformOpenAI)
} else if groupID != nil {
accounts, err = s.accountRepo.ListSchedulableByGroupIDAndPlatform(ctx, *groupID, PlatformOpenAI)
} else {
accounts, err = s.accountRepo.ListSchedulableByPlatform(ctx, PlatformOpenAI)
}
if err != nil {
return nil, fmt.Errorf("query accounts failed: %w", err)
}
return accounts, nil
}
func (s *OpenAIGatewayService) tryAcquireAccountSlot(ctx context.Context, accountID int64, maxConcurrency int) (*AcquireResult, error) {
if s.concurrencyService == nil {
return &AcquireResult{Acquired: true, ReleaseFunc: func() {}}, nil
}
return s.concurrencyService.AcquireAccountSlot(ctx, accountID, maxConcurrency)
}
func (s *OpenAIGatewayService) schedulingConfig() config.GatewaySchedulingConfig {
if s.cfg != nil {
return s.cfg.Gateway.Scheduling
}
return config.GatewaySchedulingConfig{
StickySessionMaxWaiting: 3,
StickySessionWaitTimeout: 45 * time.Second,
FallbackWaitTimeout: 30 * time.Second,
FallbackMaxWaiting: 100,
LoadBatchEnabled: true,
SlotCleanupInterval: 30 * time.Second,
}
}
// GetAccessToken gets the access token for an OpenAI account
func (s *OpenAIGatewayService) GetAccessToken(ctx context.Context, account *Account) (string, string, error) {
switch account.Type {
......
......@@ -4,7 +4,7 @@ import (
"context"
"fmt"
infraerrors "github.com/Wei-Shaw/sub2api/internal/infrastructure/errors"
infraerrors "github.com/Wei-Shaw/sub2api/internal/pkg/errors"
"github.com/Wei-Shaw/sub2api/internal/pkg/pagination"
)
......
......@@ -5,6 +5,8 @@ import (
"log"
"net/http"
"strconv"
"strings"
"sync"
"time"
"github.com/Wei-Shaw/sub2api/internal/config"
......@@ -12,15 +14,30 @@ import (
// RateLimitService 处理限流和过载状态管理
type RateLimitService struct {
accountRepo AccountRepository
cfg *config.Config
accountRepo AccountRepository
usageRepo UsageLogRepository
cfg *config.Config
geminiQuotaService *GeminiQuotaService
usageCacheMu sync.RWMutex
usageCache map[int64]*geminiUsageCacheEntry
}
type geminiUsageCacheEntry struct {
windowStart time.Time
cachedAt time.Time
totals GeminiUsageTotals
}
const geminiPrecheckCacheTTL = time.Minute
// NewRateLimitService 创建RateLimitService实例
func NewRateLimitService(accountRepo AccountRepository, cfg *config.Config) *RateLimitService {
func NewRateLimitService(accountRepo AccountRepository, usageRepo UsageLogRepository, cfg *config.Config, geminiQuotaService *GeminiQuotaService) *RateLimitService {
return &RateLimitService{
accountRepo: accountRepo,
cfg: cfg,
accountRepo: accountRepo,
usageRepo: usageRepo,
cfg: cfg,
geminiQuotaService: geminiQuotaService,
usageCache: make(map[int64]*geminiUsageCacheEntry),
}
}
......@@ -62,6 +79,106 @@ func (s *RateLimitService) HandleUpstreamError(ctx context.Context, account *Acc
}
}
// PreCheckUsage proactively checks local quota before dispatching a request.
// Returns false when the account should be skipped.
func (s *RateLimitService) PreCheckUsage(ctx context.Context, account *Account, requestedModel string) (bool, error) {
if account == nil || !account.IsGeminiCodeAssist() || strings.TrimSpace(requestedModel) == "" {
return true, nil
}
if s.usageRepo == nil || s.geminiQuotaService == nil {
return true, nil
}
quota, ok := s.geminiQuotaService.QuotaForAccount(ctx, account)
if !ok {
return true, nil
}
var limit int64
switch geminiModelClassFromName(requestedModel) {
case geminiModelFlash:
limit = quota.FlashRPD
default:
limit = quota.ProRPD
}
if limit <= 0 {
return true, nil
}
now := time.Now()
start := geminiDailyWindowStart(now)
totals, ok := s.getGeminiUsageTotals(account.ID, start, now)
if !ok {
stats, err := s.usageRepo.GetModelStatsWithFilters(ctx, start, now, 0, 0, account.ID)
if err != nil {
return true, err
}
totals = geminiAggregateUsage(stats)
s.setGeminiUsageTotals(account.ID, start, now, totals)
}
var used int64
switch geminiModelClassFromName(requestedModel) {
case geminiModelFlash:
used = totals.FlashRequests
default:
used = totals.ProRequests
}
if used >= limit {
resetAt := geminiDailyResetTime(now)
if err := s.accountRepo.SetRateLimited(ctx, account.ID, resetAt); err != nil {
log.Printf("SetRateLimited failed for account %d: %v", account.ID, err)
}
log.Printf("[Gemini PreCheck] Account %d reached daily quota (%d/%d), rate limited until %v", account.ID, used, limit, resetAt)
return false, nil
}
return true, nil
}
func (s *RateLimitService) getGeminiUsageTotals(accountID int64, windowStart, now time.Time) (GeminiUsageTotals, bool) {
s.usageCacheMu.RLock()
defer s.usageCacheMu.RUnlock()
if s.usageCache == nil {
return GeminiUsageTotals{}, false
}
entry, ok := s.usageCache[accountID]
if !ok || entry == nil {
return GeminiUsageTotals{}, false
}
if !entry.windowStart.Equal(windowStart) {
return GeminiUsageTotals{}, false
}
if now.Sub(entry.cachedAt) >= geminiPrecheckCacheTTL {
return GeminiUsageTotals{}, false
}
return entry.totals, true
}
func (s *RateLimitService) setGeminiUsageTotals(accountID int64, windowStart, now time.Time, totals GeminiUsageTotals) {
s.usageCacheMu.Lock()
defer s.usageCacheMu.Unlock()
if s.usageCache == nil {
s.usageCache = make(map[int64]*geminiUsageCacheEntry)
}
s.usageCache[accountID] = &geminiUsageCacheEntry{
windowStart: windowStart,
cachedAt: now,
totals: totals,
}
}
// GeminiCooldown returns the fallback cooldown duration for Gemini 429s based on tier.
func (s *RateLimitService) GeminiCooldown(ctx context.Context, account *Account) time.Duration {
if account == nil {
return 5 * time.Minute
}
return s.geminiQuotaService.CooldownForTier(ctx, account.GeminiTierID())
}
// handleAuthError 处理认证类错误(401/403),停止账号调度
func (s *RateLimitService) handleAuthError(ctx context.Context, account *Account, errorMsg string) {
if err := s.accountRepo.SetError(ctx, account.ID, errorMsg); err != nil {
......
......@@ -10,7 +10,7 @@ import (
"time"
dbent "github.com/Wei-Shaw/sub2api/ent"
infraerrors "github.com/Wei-Shaw/sub2api/internal/infrastructure/errors"
infraerrors "github.com/Wei-Shaw/sub2api/internal/pkg/errors"
"github.com/Wei-Shaw/sub2api/internal/pkg/pagination"
)
......
......@@ -9,7 +9,7 @@ import (
"strconv"
"github.com/Wei-Shaw/sub2api/internal/config"
infraerrors "github.com/Wei-Shaw/sub2api/internal/infrastructure/errors"
infraerrors "github.com/Wei-Shaw/sub2api/internal/pkg/errors"
)
var (
......
......@@ -6,7 +6,7 @@ import (
"log"
"time"
infraerrors "github.com/Wei-Shaw/sub2api/internal/infrastructure/errors"
infraerrors "github.com/Wei-Shaw/sub2api/internal/pkg/errors"
"github.com/Wei-Shaw/sub2api/internal/pkg/pagination"
)
......@@ -490,6 +490,7 @@ func (s *SubscriptionService) CheckAndResetWindows(ctx context.Context, sub *Use
}
// CheckUsageLimits 检查使用限额(返回错误如果超限)
// 用于中间件的快速预检查,additionalCost 通常为 0
func (s *SubscriptionService) CheckUsageLimits(ctx context.Context, sub *UserSubscription, group *Group, additionalCost float64) error {
if !sub.CheckDailyLimit(group, additionalCost) {
return ErrDailyLimitExceeded
......
......@@ -5,12 +5,13 @@ import (
"fmt"
"log"
infraerrors "github.com/Wei-Shaw/sub2api/internal/infrastructure/errors"
infraerrors "github.com/Wei-Shaw/sub2api/internal/pkg/errors"
)
var (
ErrTurnstileVerificationFailed = infraerrors.BadRequest("TURNSTILE_VERIFICATION_FAILED", "turnstile verification failed")
ErrTurnstileNotConfigured = infraerrors.ServiceUnavailable("TURNSTILE_NOT_CONFIGURED", "turnstile not configured")
ErrTurnstileInvalidSecretKey = infraerrors.BadRequest("TURNSTILE_INVALID_SECRET_KEY", "invalid turnstile secret key")
)
// TurnstileVerifier 验证 Turnstile token 的接口
......@@ -83,3 +84,22 @@ func (s *TurnstileService) VerifyToken(ctx context.Context, token string, remote
func (s *TurnstileService) IsEnabled(ctx context.Context) bool {
return s.settingService.IsTurnstileEnabled(ctx)
}
// ValidateSecretKey 验证 Turnstile Secret Key 是否有效
func (s *TurnstileService) ValidateSecretKey(ctx context.Context, secretKey string) error {
// 发送一个测试token的验证请求来检查secret_key是否有效
result, err := s.verifier.VerifyToken(ctx, secretKey, "test-validation", "")
if err != nil {
return fmt.Errorf("validate secret key: %w", err)
}
// 检查是否有 invalid-input-secret 错误
for _, code := range result.ErrorCodes {
if code == "invalid-input-secret" {
return ErrTurnstileInvalidSecretKey
}
}
// 其他错误(如 invalid-input-response)说明 secret key 是有效的
return nil
}
......@@ -5,7 +5,7 @@ import (
"fmt"
"time"
infraerrors "github.com/Wei-Shaw/sub2api/internal/infrastructure/errors"
infraerrors "github.com/Wei-Shaw/sub2api/internal/pkg/errors"
"github.com/Wei-Shaw/sub2api/internal/pkg/pagination"
"github.com/Wei-Shaw/sub2api/internal/pkg/usagestats"
)
......
......@@ -10,7 +10,6 @@ type User struct {
ID int64
Email string
Username string
Wechat string
Notes string
PasswordHash string
Role string
......
package service
import (
"context"
"time"
infraerrors "github.com/Wei-Shaw/sub2api/internal/pkg/errors"
)
// Error definitions for user attribute operations
var (
ErrAttributeDefinitionNotFound = infraerrors.NotFound("ATTRIBUTE_DEFINITION_NOT_FOUND", "attribute definition not found")
ErrAttributeKeyExists = infraerrors.Conflict("ATTRIBUTE_KEY_EXISTS", "attribute key already exists")
ErrInvalidAttributeType = infraerrors.BadRequest("INVALID_ATTRIBUTE_TYPE", "invalid attribute type")
ErrAttributeValidationFailed = infraerrors.BadRequest("ATTRIBUTE_VALIDATION_FAILED", "attribute value validation failed")
)
// UserAttributeType represents supported attribute types
type UserAttributeType string
const (
AttributeTypeText UserAttributeType = "text"
AttributeTypeTextarea UserAttributeType = "textarea"
AttributeTypeNumber UserAttributeType = "number"
AttributeTypeEmail UserAttributeType = "email"
AttributeTypeURL UserAttributeType = "url"
AttributeTypeDate UserAttributeType = "date"
AttributeTypeSelect UserAttributeType = "select"
AttributeTypeMultiSelect UserAttributeType = "multi_select"
)
// UserAttributeOption represents a select option for select/multi_select types
type UserAttributeOption struct {
Value string `json:"value"`
Label string `json:"label"`
}
// UserAttributeValidation represents validation rules for an attribute
type UserAttributeValidation struct {
MinLength *int `json:"min_length,omitempty"`
MaxLength *int `json:"max_length,omitempty"`
Min *int `json:"min,omitempty"`
Max *int `json:"max,omitempty"`
Pattern *string `json:"pattern,omitempty"`
Message *string `json:"message,omitempty"`
}
// UserAttributeDefinition represents a custom attribute definition
type UserAttributeDefinition struct {
ID int64
Key string
Name string
Description string
Type UserAttributeType
Options []UserAttributeOption
Required bool
Validation UserAttributeValidation
Placeholder string
DisplayOrder int
Enabled bool
CreatedAt time.Time
UpdatedAt time.Time
}
// UserAttributeValue represents a user's attribute value
type UserAttributeValue struct {
ID int64
UserID int64
AttributeID int64
Value string
CreatedAt time.Time
UpdatedAt time.Time
}
// CreateAttributeDefinitionInput for creating new definition
type CreateAttributeDefinitionInput struct {
Key string
Name string
Description string
Type UserAttributeType
Options []UserAttributeOption
Required bool
Validation UserAttributeValidation
Placeholder string
Enabled bool
}
// UpdateAttributeDefinitionInput for updating definition
type UpdateAttributeDefinitionInput struct {
Name *string
Description *string
Type *UserAttributeType
Options *[]UserAttributeOption
Required *bool
Validation *UserAttributeValidation
Placeholder *string
Enabled *bool
}
// UpdateUserAttributeInput for updating a single attribute value
type UpdateUserAttributeInput struct {
AttributeID int64
Value string
}
// UserAttributeDefinitionRepository interface for attribute definition persistence
type UserAttributeDefinitionRepository interface {
Create(ctx context.Context, def *UserAttributeDefinition) error
GetByID(ctx context.Context, id int64) (*UserAttributeDefinition, error)
GetByKey(ctx context.Context, key string) (*UserAttributeDefinition, error)
Update(ctx context.Context, def *UserAttributeDefinition) error
Delete(ctx context.Context, id int64) error
List(ctx context.Context, enabledOnly bool) ([]UserAttributeDefinition, error)
UpdateDisplayOrders(ctx context.Context, orders map[int64]int) error
ExistsByKey(ctx context.Context, key string) (bool, error)
}
// UserAttributeValueRepository interface for user attribute value persistence
type UserAttributeValueRepository interface {
GetByUserID(ctx context.Context, userID int64) ([]UserAttributeValue, error)
GetByUserIDs(ctx context.Context, userIDs []int64) ([]UserAttributeValue, error)
UpsertBatch(ctx context.Context, userID int64, values []UpdateUserAttributeInput) error
DeleteByAttributeID(ctx context.Context, attributeID int64) error
DeleteByUserID(ctx context.Context, userID int64) error
}
package service
import (
"context"
"encoding/json"
"fmt"
"regexp"
"strconv"
"strings"
infraerrors "github.com/Wei-Shaw/sub2api/internal/pkg/errors"
)
// UserAttributeService handles attribute management
type UserAttributeService struct {
defRepo UserAttributeDefinitionRepository
valueRepo UserAttributeValueRepository
}
// NewUserAttributeService creates a new service instance
func NewUserAttributeService(
defRepo UserAttributeDefinitionRepository,
valueRepo UserAttributeValueRepository,
) *UserAttributeService {
return &UserAttributeService{
defRepo: defRepo,
valueRepo: valueRepo,
}
}
// CreateDefinition creates a new attribute definition
func (s *UserAttributeService) CreateDefinition(ctx context.Context, input CreateAttributeDefinitionInput) (*UserAttributeDefinition, error) {
// Validate type
if !isValidAttributeType(input.Type) {
return nil, ErrInvalidAttributeType
}
// Check if key exists
exists, err := s.defRepo.ExistsByKey(ctx, input.Key)
if err != nil {
return nil, fmt.Errorf("check key exists: %w", err)
}
if exists {
return nil, ErrAttributeKeyExists
}
def := &UserAttributeDefinition{
Key: input.Key,
Name: input.Name,
Description: input.Description,
Type: input.Type,
Options: input.Options,
Required: input.Required,
Validation: input.Validation,
Placeholder: input.Placeholder,
Enabled: input.Enabled,
}
if err := s.defRepo.Create(ctx, def); err != nil {
return nil, fmt.Errorf("create definition: %w", err)
}
return def, nil
}
// GetDefinition retrieves a definition by ID
func (s *UserAttributeService) GetDefinition(ctx context.Context, id int64) (*UserAttributeDefinition, error) {
return s.defRepo.GetByID(ctx, id)
}
// ListDefinitions lists all definitions
func (s *UserAttributeService) ListDefinitions(ctx context.Context, enabledOnly bool) ([]UserAttributeDefinition, error) {
return s.defRepo.List(ctx, enabledOnly)
}
// UpdateDefinition updates an existing definition
func (s *UserAttributeService) UpdateDefinition(ctx context.Context, id int64, input UpdateAttributeDefinitionInput) (*UserAttributeDefinition, error) {
def, err := s.defRepo.GetByID(ctx, id)
if err != nil {
return nil, err
}
if input.Name != nil {
def.Name = *input.Name
}
if input.Description != nil {
def.Description = *input.Description
}
if input.Type != nil {
if !isValidAttributeType(*input.Type) {
return nil, ErrInvalidAttributeType
}
def.Type = *input.Type
}
if input.Options != nil {
def.Options = *input.Options
}
if input.Required != nil {
def.Required = *input.Required
}
if input.Validation != nil {
def.Validation = *input.Validation
}
if input.Placeholder != nil {
def.Placeholder = *input.Placeholder
}
if input.Enabled != nil {
def.Enabled = *input.Enabled
}
if err := s.defRepo.Update(ctx, def); err != nil {
return nil, fmt.Errorf("update definition: %w", err)
}
return def, nil
}
// DeleteDefinition soft-deletes a definition and hard-deletes associated values
func (s *UserAttributeService) DeleteDefinition(ctx context.Context, id int64) error {
// Check if definition exists
_, err := s.defRepo.GetByID(ctx, id)
if err != nil {
return err
}
// First delete all values (hard delete)
if err := s.valueRepo.DeleteByAttributeID(ctx, id); err != nil {
return fmt.Errorf("delete values: %w", err)
}
// Then soft-delete the definition
if err := s.defRepo.Delete(ctx, id); err != nil {
return fmt.Errorf("delete definition: %w", err)
}
return nil
}
// ReorderDefinitions updates display order for multiple definitions
func (s *UserAttributeService) ReorderDefinitions(ctx context.Context, orders map[int64]int) error {
return s.defRepo.UpdateDisplayOrders(ctx, orders)
}
// GetUserAttributes retrieves all attribute values for a user
func (s *UserAttributeService) GetUserAttributes(ctx context.Context, userID int64) ([]UserAttributeValue, error) {
return s.valueRepo.GetByUserID(ctx, userID)
}
// GetBatchUserAttributes retrieves attribute values for multiple users
// Returns a map of userID -> map of attributeID -> value
func (s *UserAttributeService) GetBatchUserAttributes(ctx context.Context, userIDs []int64) (map[int64]map[int64]string, error) {
values, err := s.valueRepo.GetByUserIDs(ctx, userIDs)
if err != nil {
return nil, err
}
result := make(map[int64]map[int64]string)
for _, v := range values {
if result[v.UserID] == nil {
result[v.UserID] = make(map[int64]string)
}
result[v.UserID][v.AttributeID] = v.Value
}
return result, nil
}
// UpdateUserAttributes batch updates attribute values for a user
func (s *UserAttributeService) UpdateUserAttributes(ctx context.Context, userID int64, inputs []UpdateUserAttributeInput) error {
// Validate all values before updating
defs, err := s.defRepo.List(ctx, true)
if err != nil {
return fmt.Errorf("list definitions: %w", err)
}
defMap := make(map[int64]*UserAttributeDefinition, len(defs))
for i := range defs {
defMap[defs[i].ID] = &defs[i]
}
for _, input := range inputs {
def, ok := defMap[input.AttributeID]
if !ok {
return ErrAttributeDefinitionNotFound
}
if err := s.validateValue(def, input.Value); err != nil {
return err
}
}
return s.valueRepo.UpsertBatch(ctx, userID, inputs)
}
// validateValue validates a value against its definition
func (s *UserAttributeService) validateValue(def *UserAttributeDefinition, value string) error {
// Skip validation for empty non-required fields
if value == "" && !def.Required {
return nil
}
// Required check
if def.Required && value == "" {
return validationError(fmt.Sprintf("%s is required", def.Name))
}
v := def.Validation
// String length validation
if v.MinLength != nil && len(value) < *v.MinLength {
return validationError(fmt.Sprintf("%s must be at least %d characters", def.Name, *v.MinLength))
}
if v.MaxLength != nil && len(value) > *v.MaxLength {
return validationError(fmt.Sprintf("%s must be at most %d characters", def.Name, *v.MaxLength))
}
// Number validation
if def.Type == AttributeTypeNumber && value != "" {
num, err := strconv.Atoi(value)
if err != nil {
return validationError(fmt.Sprintf("%s must be a number", def.Name))
}
if v.Min != nil && num < *v.Min {
return validationError(fmt.Sprintf("%s must be at least %d", def.Name, *v.Min))
}
if v.Max != nil && num > *v.Max {
return validationError(fmt.Sprintf("%s must be at most %d", def.Name, *v.Max))
}
}
// Pattern validation
if v.Pattern != nil && *v.Pattern != "" && value != "" {
re, err := regexp.Compile(*v.Pattern)
if err == nil && !re.MatchString(value) {
msg := def.Name + " format is invalid"
if v.Message != nil && *v.Message != "" {
msg = *v.Message
}
return validationError(msg)
}
}
// Select validation
if def.Type == AttributeTypeSelect && value != "" {
found := false
for _, opt := range def.Options {
if opt.Value == value {
found = true
break
}
}
if !found {
return validationError(fmt.Sprintf("%s: invalid option", def.Name))
}
}
// Multi-select validation (stored as JSON array)
if def.Type == AttributeTypeMultiSelect && value != "" {
var values []string
if err := json.Unmarshal([]byte(value), &values); err != nil {
// Try comma-separated fallback
values = strings.Split(value, ",")
}
for _, val := range values {
val = strings.TrimSpace(val)
found := false
for _, opt := range def.Options {
if opt.Value == val {
found = true
break
}
}
if !found {
return validationError(fmt.Sprintf("%s: invalid option %s", def.Name, val))
}
}
}
return nil
}
// validationError creates a validation error with a custom message
func validationError(msg string) error {
return infraerrors.BadRequest("ATTRIBUTE_VALIDATION_FAILED", msg)
}
func isValidAttributeType(t UserAttributeType) bool {
switch t {
case AttributeTypeText, AttributeTypeTextarea, AttributeTypeNumber,
AttributeTypeEmail, AttributeTypeURL, AttributeTypeDate,
AttributeTypeSelect, AttributeTypeMultiSelect:
return true
}
return false
}
......@@ -4,7 +4,7 @@ import (
"context"
"fmt"
infraerrors "github.com/Wei-Shaw/sub2api/internal/infrastructure/errors"
infraerrors "github.com/Wei-Shaw/sub2api/internal/pkg/errors"
"github.com/Wei-Shaw/sub2api/internal/pkg/pagination"
)
......@@ -14,6 +14,14 @@ var (
ErrInsufficientPerms = infraerrors.Forbidden("INSUFFICIENT_PERMISSIONS", "insufficient permissions")
)
// UserListFilters contains all filter options for listing users
type UserListFilters struct {
Status string // User status filter
Role string // User role filter
Search string // Search in email, username
Attributes map[int64]string // Custom attribute filters: attributeID -> value
}
type UserRepository interface {
Create(ctx context.Context, user *User) error
GetByID(ctx context.Context, id int64) (*User, error)
......@@ -23,7 +31,7 @@ type UserRepository interface {
Delete(ctx context.Context, id int64) error
List(ctx context.Context, params pagination.PaginationParams) ([]User, *pagination.PaginationResult, error)
ListWithFilters(ctx context.Context, params pagination.PaginationParams, status, role, search string) ([]User, *pagination.PaginationResult, error)
ListWithFilters(ctx context.Context, params pagination.PaginationParams, filters UserListFilters) ([]User, *pagination.PaginationResult, error)
UpdateBalance(ctx context.Context, id int64, amount float64) error
DeductBalance(ctx context.Context, id int64, amount float64) error
......@@ -36,7 +44,6 @@ type UserRepository interface {
type UpdateProfileRequest struct {
Email *string `json:"email"`
Username *string `json:"username"`
Wechat *string `json:"wechat"`
Concurrency *int `json:"concurrency"`
}
......@@ -100,10 +107,6 @@ func (s *UserService) UpdateProfile(ctx context.Context, userID int64, req Updat
user.Username = *req.Username
}
if req.Wechat != nil {
user.Wechat = *req.Wechat
}
if req.Concurrency != nil {
user.Concurrency = *req.Concurrency
}
......
......@@ -73,6 +73,15 @@ func ProvideDeferredService(accountRepo AccountRepository, timingWheel *TimingWh
return svc
}
// ProvideConcurrencyService creates ConcurrencyService and starts slot cleanup worker.
func ProvideConcurrencyService(cache ConcurrencyCache, accountRepo AccountRepository, cfg *config.Config) *ConcurrencyService {
svc := NewConcurrencyService(cache)
if cfg != nil {
svc.StartSlotCleanupWorker(accountRepo, cfg.Gateway.Scheduling.SlotCleanupInterval)
}
return svc
}
// ProviderSet is the Wire provider set for all services
var ProviderSet = wire.NewSet(
// Core services
......@@ -94,6 +103,7 @@ var ProviderSet = wire.NewSet(
NewOAuthService,
NewOpenAIOAuthService,
NewGeminiOAuthService,
NewGeminiQuotaService,
NewAntigravityOAuthService,
NewGeminiTokenProvider,
NewGeminiMessagesCompatService,
......@@ -107,7 +117,7 @@ var ProviderSet = wire.NewSet(
ProvideEmailQueueService,
NewTurnstileService,
NewSubscriptionService,
NewConcurrencyService,
ProvideConcurrencyService,
NewIdentityService,
NewCRSSyncService,
ProvideUpdateService,
......@@ -115,4 +125,5 @@ var ProviderSet = wire.NewSet(
ProvideTimingWheelService,
ProvideDeferredService,
ProvideAntigravityQuotaRefresher,
NewUserAttributeService,
)
......@@ -12,7 +12,7 @@ import (
"strings"
"time"
"github.com/Wei-Shaw/sub2api/internal/infrastructure"
"github.com/Wei-Shaw/sub2api/internal/repository"
"github.com/Wei-Shaw/sub2api/internal/service"
_ "github.com/lib/pq"
......@@ -269,7 +269,7 @@ func initializeDatabase(cfg *SetupConfig) error {
migrationCtx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
return infrastructure.ApplyMigrations(migrationCtx, db)
return repository.ApplyMigrations(migrationCtx, db)
}
func createAdminUser(cfg *SetupConfig) error {
......
-- Add user attribute definitions and values tables for custom user attributes.
-- User Attribute Definitions table (with soft delete support)
CREATE TABLE IF NOT EXISTS user_attribute_definitions (
id BIGSERIAL PRIMARY KEY,
key VARCHAR(100) NOT NULL,
name VARCHAR(255) NOT NULL,
description TEXT DEFAULT '',
type VARCHAR(20) NOT NULL,
options JSONB DEFAULT '[]'::jsonb,
required BOOLEAN NOT NULL DEFAULT FALSE,
validation JSONB DEFAULT '{}'::jsonb,
placeholder VARCHAR(255) DEFAULT '',
display_order INT NOT NULL DEFAULT 0,
enabled BOOLEAN NOT NULL DEFAULT TRUE,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
deleted_at TIMESTAMPTZ
);
-- Partial unique index for key (only for non-deleted records)
-- Allows reusing keys after soft delete
CREATE UNIQUE INDEX IF NOT EXISTS idx_user_attribute_definitions_key_unique
ON user_attribute_definitions(key) WHERE deleted_at IS NULL;
CREATE INDEX IF NOT EXISTS idx_user_attribute_definitions_enabled
ON user_attribute_definitions(enabled);
CREATE INDEX IF NOT EXISTS idx_user_attribute_definitions_display_order
ON user_attribute_definitions(display_order);
CREATE INDEX IF NOT EXISTS idx_user_attribute_definitions_deleted_at
ON user_attribute_definitions(deleted_at);
-- User Attribute Values table (hard delete only, no deleted_at)
CREATE TABLE IF NOT EXISTS user_attribute_values (
id BIGSERIAL PRIMARY KEY,
user_id BIGINT NOT NULL REFERENCES users(id) ON DELETE CASCADE,
attribute_id BIGINT NOT NULL REFERENCES user_attribute_definitions(id) ON DELETE CASCADE,
value TEXT DEFAULT '',
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
UNIQUE(user_id, attribute_id)
);
CREATE INDEX IF NOT EXISTS idx_user_attribute_values_user_id
ON user_attribute_values(user_id);
CREATE INDEX IF NOT EXISTS idx_user_attribute_values_attribute_id
ON user_attribute_values(attribute_id);
-- Migration: Move wechat field from users table to user_attribute_values
-- This migration:
-- 1. Creates a "wechat" attribute definition
-- 2. Migrates existing wechat data to user_attribute_values
-- 3. Does NOT drop the wechat column (for rollback safety, can be done in a later migration)
-- +goose Up
-- +goose StatementBegin
-- Step 1: Insert wechat attribute definition if not exists
INSERT INTO user_attribute_definitions (key, name, description, type, options, required, validation, placeholder, display_order, enabled, created_at, updated_at)
SELECT 'wechat', '微信', '用户微信号', 'text', '[]'::jsonb, false, '{}'::jsonb, '请输入微信号', 0, true, NOW(), NOW()
WHERE NOT EXISTS (
SELECT 1 FROM user_attribute_definitions WHERE key = 'wechat' AND deleted_at IS NULL
);
-- Step 2: Migrate existing wechat values to user_attribute_values
-- Only migrate non-empty values
INSERT INTO user_attribute_values (user_id, attribute_id, value, created_at, updated_at)
SELECT
u.id,
(SELECT id FROM user_attribute_definitions WHERE key = 'wechat' AND deleted_at IS NULL LIMIT 1),
u.wechat,
NOW(),
NOW()
FROM users u
WHERE u.wechat IS NOT NULL
AND u.wechat != ''
AND u.deleted_at IS NULL
AND NOT EXISTS (
SELECT 1 FROM user_attribute_values uav
WHERE uav.user_id = u.id
AND uav.attribute_id = (SELECT id FROM user_attribute_definitions WHERE key = 'wechat' AND deleted_at IS NULL LIMIT 1)
);
-- Step 3: Update display_order to ensure wechat appears first
UPDATE user_attribute_definitions
SET display_order = -1
WHERE key = 'wechat' AND deleted_at IS NULL;
-- Reorder all attributes starting from 0
WITH ordered AS (
SELECT id, ROW_NUMBER() OVER (ORDER BY display_order, id) - 1 as new_order
FROM user_attribute_definitions
WHERE deleted_at IS NULL
)
UPDATE user_attribute_definitions
SET display_order = ordered.new_order
FROM ordered
WHERE user_attribute_definitions.id = ordered.id;
-- Step 4: Drop the redundant wechat column from users table
ALTER TABLE users DROP COLUMN IF EXISTS wechat;
-- +goose StatementEnd
-- +goose Down
-- +goose StatementBegin
-- Restore wechat column
ALTER TABLE users ADD COLUMN IF NOT EXISTS wechat VARCHAR(100) DEFAULT '';
-- Copy attribute values back to users.wechat column
UPDATE users u
SET wechat = uav.value
FROM user_attribute_values uav
JOIN user_attribute_definitions uad ON uav.attribute_id = uad.id
WHERE uav.user_id = u.id
AND uad.key = 'wechat'
AND uad.deleted_at IS NULL;
-- Delete migrated attribute values
DELETE FROM user_attribute_values
WHERE attribute_id IN (
SELECT id FROM user_attribute_definitions WHERE key = 'wechat' AND deleted_at IS NULL
);
-- Soft-delete the wechat attribute definition
UPDATE user_attribute_definitions
SET deleted_at = NOW()
WHERE key = 'wechat' AND deleted_at IS NULL;
-- +goose StatementEnd
-- +goose Up
-- +goose StatementBegin
-- 为 Gemini Code Assist OAuth 账号添加默认 tier_id
-- 包括显式标记为 code_assist 的账号,以及 legacy 账号(oauth_type 为空但 project_id 存在)
UPDATE accounts
SET credentials = jsonb_set(
credentials,
'{tier_id}',
'"LEGACY"',
true
)
WHERE platform = 'gemini'
AND type = 'oauth'
AND jsonb_typeof(credentials) = 'object'
AND credentials->>'tier_id' IS NULL
AND (
credentials->>'oauth_type' = 'code_assist'
OR (credentials->>'oauth_type' IS NULL AND credentials->>'project_id' IS NOT NULL)
);
-- +goose StatementEnd
-- +goose Down
-- +goose StatementBegin
-- 回滚:删除 tier_id 字段
UPDATE accounts
SET credentials = credentials - 'tier_id'
WHERE platform = 'gemini'
AND type = 'oauth'
AND credentials ? 'tier_id';
-- +goose StatementEnd
# Database Migrations
## Overview
This directory contains SQL migration files for database schema changes. The migration system uses SHA256 checksums to ensure migration immutability and consistency across environments.
## Migration File Naming
Format: `NNN_description.sql`
- `NNN`: Sequential number (e.g., 001, 002, 003)
- `description`: Brief description in snake_case
Example: `017_add_gemini_tier_id.sql`
## Migration File Structure
```sql
-- +goose Up
-- +goose StatementBegin
-- Your forward migration SQL here
-- +goose StatementEnd
-- +goose Down
-- +goose StatementBegin
-- Your rollback migration SQL here
-- +goose StatementEnd
```
## Important Rules
### ⚠️ Immutability Principle
**Once a migration is applied to ANY environment (dev, staging, production), it MUST NOT be modified.**
Why?
- Each migration has a SHA256 checksum stored in the `schema_migrations` table
- Modifying an applied migration causes checksum mismatch errors
- Different environments would have inconsistent database states
- Breaks audit trail and reproducibility
### ✅ Correct Workflow
1. **Create new migration**
```bash
# Create new file with next sequential number
touch migrations/018_your_change.sql
```
2. **Write Up and Down migrations**
- Up: Apply the change
- Down: Revert the change (should be symmetric with Up)
3. **Test locally**
```bash
# Apply migration
make migrate-up
# Test rollback
make migrate-down
```
4. **Commit and deploy**
```bash
git add migrations/018_your_change.sql
git commit -m "feat(db): add your change"
```
### ❌ What NOT to Do
- ❌ Modify an already-applied migration file
- ❌ Delete migration files
- ❌ Change migration file names
- ❌ Reorder migration numbers
### 🔧 If You Accidentally Modified an Applied Migration
**Error message:**
```
migration 017_add_gemini_tier_id.sql checksum mismatch (db=abc123... file=def456...)
```
**Solution:**
```bash
# 1. Find the original version
git log --oneline -- migrations/017_add_gemini_tier_id.sql
# 2. Revert to the commit when it was first applied
git checkout <commit-hash> -- migrations/017_add_gemini_tier_id.sql
# 3. Create a NEW migration for your changes
touch migrations/018_your_new_change.sql
```
## Migration System Details
- **Checksum Algorithm**: SHA256 of trimmed file content
- **Tracking Table**: `schema_migrations` (filename, checksum, applied_at)
- **Runner**: `internal/repository/migrations_runner.go`
- **Auto-run**: Migrations run automatically on service startup
## Best Practices
1. **Keep migrations small and focused**
- One logical change per migration
- Easier to review and rollback
2. **Write reversible migrations**
- Always provide a working Down migration
- Test rollback before committing
3. **Use transactions**
- Wrap DDL statements in transactions when possible
- Ensures atomicity
4. **Add comments**
- Explain WHY the change is needed
- Document any special considerations
5. **Test in development first**
- Apply migration locally
- Verify data integrity
- Test rollback
## Example Migration
```sql
-- +goose Up
-- +goose StatementBegin
-- Add tier_id field to Gemini OAuth accounts for quota tracking
UPDATE accounts
SET credentials = jsonb_set(
credentials,
'{tier_id}',
'"LEGACY"',
true
)
WHERE platform = 'gemini'
AND type = 'oauth'
AND credentials->>'tier_id' IS NULL;
-- +goose StatementEnd
-- +goose Down
-- +goose StatementBegin
-- Remove tier_id field
UPDATE accounts
SET credentials = credentials - 'tier_id'
WHERE platform = 'gemini'
AND type = 'oauth'
AND credentials->>'tier_id' = 'LEGACY';
-- +goose StatementEnd
```
## Troubleshooting
### Checksum Mismatch
See "If You Accidentally Modified an Applied Migration" above.
### Migration Failed
```bash
# Check migration status
psql -d sub2api -c "SELECT * FROM schema_migrations ORDER BY applied_at DESC;"
# Manually rollback if needed (use with caution)
# Better to fix the migration and create a new one
```
### Need to Skip a Migration (Emergency Only)
```sql
-- DANGEROUS: Only use in development or with extreme caution
INSERT INTO schema_migrations (filename, checksum, applied_at)
VALUES ('NNN_migration.sql', 'calculated_checksum', NOW());
```
## References
- Migration runner: `internal/repository/migrations_runner.go`
- Goose syntax: https://github.com/pressly/goose
- PostgreSQL docs: https://www.postgresql.org/docs/
......@@ -86,3 +86,11 @@ GEMINI_OAUTH_CLIENT_ID=
GEMINI_OAUTH_CLIENT_SECRET=
# Optional; leave empty to auto-select scopes based on oauth_type
GEMINI_OAUTH_SCOPES=
# -----------------------------------------------------------------------------
# Gemini Quota Policy (OPTIONAL, local simulation)
# -----------------------------------------------------------------------------
# JSON overrides for local quota simulation (Code Assist only).
# Example:
# GEMINI_QUOTA_POLICY={"tiers":{"LEGACY":{"pro_rpd":50,"flash_rpd":1500,"cooldown_minutes":30},"PRO":{"pro_rpd":1500,"flash_rpd":4000,"cooldown_minutes":5},"ULTRA":{"pro_rpd":2000,"flash_rpd":0,"cooldown_minutes":5}}}
GEMINI_QUOTA_POLICY=
......@@ -123,6 +123,7 @@ docker-compose down -v
| `GEMINI_OAUTH_CLIENT_ID` | No | *(builtin)* | Google OAuth client ID (Gemini OAuth). Leave empty to use the built-in Gemini CLI client. |
| `GEMINI_OAUTH_CLIENT_SECRET` | No | *(builtin)* | Google OAuth client secret (Gemini OAuth). Leave empty to use the built-in Gemini CLI client. |
| `GEMINI_OAUTH_SCOPES` | No | *(default)* | OAuth scopes (Gemini OAuth) |
| `GEMINI_QUOTA_POLICY` | No | *(empty)* | JSON overrides for Gemini local quota simulation (Code Assist only). |
See `.env.example` for all available options.
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment