From 0e04abf4b853be3354c235511c0afd8f9008f81d Mon Sep 17 00:00:00 2001 From: malai Date: Sun, 30 Mar 2025 00:14:24 +0800 Subject: [PATCH] feat(search): add logging, rate limiting, and genre ID support Enhance the ITunesProvider by adding logging with detailed caller information, implementing rate limiting to prevent API abuse, and supporting genre IDs for more precise searches. Also, update the User-Agent string and improve error handling and logging throughout the search and charts functionality. --- go.mod | 4 +++ go.sum | 12 +++++++ main.go | 5 +-- model/genre.go | 10 ++++++ search/itunes_search.go | 71 ++++++++++++++++++++++++++++------------- 5 files changed, 78 insertions(+), 24 deletions(-) diff --git a/go.mod b/go.mod index 217a4e6..9e88de2 100644 --- a/go.mod +++ b/go.mod @@ -5,8 +5,12 @@ go 1.23 require github.com/mmcdole/gofeed v1.3.0 require ( + github.com/benbjohnson/clock v1.3.0 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect + go.uber.org/ratelimit v0.3.1 // indirect + golang.org/x/sys v0.5.0 // indirect ) require ( diff --git a/go.sum b/go.sum index 4c4f837..abac0fc 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,8 @@ github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0g github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI= github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= +github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= +github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -23,15 +25,23 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +go.uber.org/ratelimit v0.3.1 h1:K4qVE+byfv/B3tC+4nYWP7v/6SimcO7HzHekoMNBma0= +go.uber.org/ratelimit v0.3.1/go.mod h1:6euWsTB6U/Nb3X++xEUXA8ciPJvr19Q/0h1+oDcJhRk= golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.4.0 h1:Q5QPcMlvfxFTAPV0+07Xz/MpK9NTXu2VDUuy0FeMfaU= golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM= @@ -39,6 +49,8 @@ golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gorm.io/gorm v1.25.12 h1:I0u8i2hWQItBq1WfE0o2+WuL9+8L21K9e2HHSTE/0f8= diff --git a/main.go b/main.go index c106e82..deb4911 100644 --- a/main.go +++ b/main.go @@ -3,6 +3,7 @@ package main import ( "fmt" + "gitea.malaihome.work/ans/podcast-search-go/model" "gitea.malaihome.work/ans/podcast-search-go/search" ) @@ -11,12 +12,12 @@ func main() { provider := search.NewITunesProvider() // podcast, err := provider.SearchPodcasts("golang", &search.SearchOptions{Country: "us", Language: "en-us"}) - podcast, err := provider.Charts(&search.ChartsOptions{Country: "us", Language: "en-us", Limit: 10}) + podcast, err := provider.Charts(&search.ChartsOptions{Country: "us", GenreID: model.GetGenreID("en-US", "Art"), Limit: 3}) if err != nil { panic(err) } // print podcast to json - fmt.Printf("podcast: %+v\n", podcast) + fmt.Printf("podcast count: %+v\n", len(podcast.Items)) } diff --git a/model/genre.go b/model/genre.go index 08d4f75..fee62b4 100644 --- a/model/genre.go +++ b/model/genre.go @@ -108,3 +108,13 @@ func GetGenreByLanguage(language string) map[string]string { } return m } + +func GetGenreID(language string, genre string) string { + for genreId, v := range Genres { + genreOffical, ok := v[language] + if ok && genreOffical == genre { + return genreId + } + } + return "" +} diff --git a/search/itunes_search.go b/search/itunes_search.go index 27f0fa9..d32b665 100644 --- a/search/itunes_search.go +++ b/search/itunes_search.go @@ -7,10 +7,15 @@ import ( "io" "net/http" "net/url" + "path" + "runtime" "strconv" + "strings" "time" "gitea.malaihome.work/ans/podcast-search-go/model" + "github.com/sirupsen/logrus" + "go.uber.org/ratelimit" ) // SearchProvider 定义搜索提供者接口 @@ -32,16 +37,35 @@ type ITunesProvider struct { // 流派映射 GenresMap map[string]int // HTTP客户端 - Client *http.Client + Client *http.Client + Limiter ratelimit.Limiter + Logger *logrus.Logger } // NewITunesProvider 创建新的iTunes搜索提供者 func NewITunesProvider() *ITunesProvider { + // 初始化日志 + // 启用调用者信息记录 + Logger := logrus.New() + Logger.SetReportCaller(true) + + Logger.SetFormatter(&logrus.TextFormatter{ + FullTimestamp: true, + TimestampFormat: "2006-01-02 15:04:05.000", + CallerPrettyfier: func(f *runtime.Frame) (string, string) { + s := strings.Split(f.Function, ".") + funcname := s[len(s)-1] + _, filename := path.Split(f.File) + return funcname, fmt.Sprintf("%s:%d", filename, f.Line) + }, + }) return &ITunesProvider{ SearchAPIEndpoint: "https://itunes.apple.com/search", FeedAPIEndpoint: "https://itunes.apple.com", GenresMap: initITunesGenres(), Client: &http.Client{Timeout: 20 * time.Second}, + Limiter: ratelimit.New(20, ratelimit.Per(time.Minute)), // 20 RPM + Logger: Logger, } } @@ -76,7 +100,7 @@ type SearchOptions struct { // 国家 Country string // 流派 - Genre string + GenreID string // 属性 Attribute string // 限制结果数量 @@ -98,7 +122,7 @@ type ChartsOptions struct { // 是否包含限制内容 Explicit bool // 流派 - Genre string + GenreID string // 额外的查询参数 QueryParams map[string]string } @@ -120,10 +144,8 @@ func (p *ITunesProvider) SearchPodcasts(term string, options *SearchOptions) (*m params.Add("country", options.Country) } - if options.Genre != "" { - if genreID, ok := p.GenresMap[options.Genre]; ok && genreID != -1 { - params.Add("genreId", strconv.Itoa(genreID)) - } + if len(options.GenreID) > 0 { + params.Add("genreId", options.GenreID) } if options.Attribute != "" { @@ -139,15 +161,15 @@ func (p *ITunesProvider) SearchPodcasts(term string, options *SearchOptions) (*m } if options.Explicit { - params.Add("explicit", "Yes") + params.Add("explicit", "true") } else { - params.Add("explicit", "No") + params.Add("explicit", "false") } } // 构建URL url := fmt.Sprintf("%s?%s", p.SearchAPIEndpoint, params.Encode()) - // fmt.Printf("url: %s\n", url) + p.Logger.Infof("url: %s", url) // 发送请求 req, err := http.NewRequest("GET", url, nil) @@ -156,10 +178,12 @@ func (p *ITunesProvider) SearchPodcasts(term string, options *SearchOptions) (*m } // 设置User-Agent - req.Header.Set("User-Agent", "PodcastSearch-Go/0.1") + req.Header.Set("User-Agent", "podcast-search-go/0.1") // 发送请求 + p.Limiter.Take() resp, err := p.Client.Do(req) + p.Logger.Infof("response code: %d", resp.StatusCode) if err != nil { return nil, fmt.Errorf("发送请求失败: %w", err) } @@ -175,7 +199,6 @@ func (p *ITunesProvider) SearchPodcasts(term string, options *SearchOptions) (*m if err != nil { return nil, fmt.Errorf("读取响应失败: %w", err) } - // fmt.Printf("body: %s\n", string(body)) // 创建搜索结果 result := &model.SearchResult2{ @@ -186,6 +209,7 @@ func (p *ITunesProvider) SearchPodcasts(term string, options *SearchOptions) (*m if err := json.Unmarshal(body, &result); err != nil { return nil, fmt.Errorf("解析响应失败: %w", err) } + p.Logger.Infof("search result, got %d items", len(result.Items)) result.ResultCount = len(result.Items) result.Successful = true @@ -199,7 +223,7 @@ func (p *ITunesProvider) SearchPodcasts(term string, options *SearchOptions) (*m func (p *ITunesProvider) Charts(options *ChartsOptions) (*model.SearchResult2, error) { // 构建URL url := p.buildChartsURL(options) - // fmt.Printf("url: %s\n", url) + p.Logger.Infof("url: %s", url) // 发送请求 req, err := http.NewRequest("GET", url, nil) @@ -208,10 +232,13 @@ func (p *ITunesProvider) Charts(options *ChartsOptions) (*model.SearchResult2, e } // 设置User-Agent - req.Header.Set("User-Agent", "PodcastSearch-Go/0.1") + req.Header.Set("User-Agent", "podcast-search-go/0.1") // 发送请求 + p.Limiter.Take() resp, err := p.Client.Do(req) + p.Logger.Infof("response code: %d", resp.StatusCode) + if err != nil { return nil, fmt.Errorf("发送请求失败: %w", err) } @@ -227,13 +254,12 @@ func (p *ITunesProvider) Charts(options *ChartsOptions) (*model.SearchResult2, e if err != nil { return nil, fmt.Errorf("读取响应失败: %w", err) } - // fmt.Printf("body: %s\n", string(body)[:3000]) - // fmt.Printf("body: %s\n", string(body)) var chartResult model.ChartResult if err := json.Unmarshal(body, &chartResult); err != nil { return nil, fmt.Errorf("解析响应失败: %w", err) } + p.Logger.Infof("chart result, got %d items", len(chartResult.Feed.Entry)) // 创建搜索结果 result := &model.SearchResult2{ @@ -252,13 +278,16 @@ func (p *ITunesProvider) Charts(options *ChartsOptions) (*model.SearchResult2, e // 为每个播客获取详细信息 lookupURL := fmt.Sprintf("%s/lookup?id=%s", p.FeedAPIEndpoint, id) + p.Logger.Infof("lookup url: %s", lookupURL) lookupReq, err := http.NewRequest("GET", lookupURL, nil) if err != nil { continue } - lookupReq.Header.Set("User-Agent", "PodcastSearch-Go/0.1") + req.Header.Set("User-Agent", "podcast-search-go/0.1") + p.Limiter.Take() lookupResp, err := p.Client.Do(lookupReq) + p.Logger.Infof("lookup response code: %d", lookupResp.StatusCode) if err != nil { continue } @@ -273,12 +302,12 @@ func (p *ITunesProvider) Charts(options *ChartsOptions) (*model.SearchResult2, e if err != nil { continue } - // fmt.Printf("lookupBody: %s\n", string(lookupBody)) var lookupResult model.LookupResult if err := json.Unmarshal(lookupBody, &lookupResult); err != nil { continue } + p.Logger.Infof("lookup result, collection id: %d, collection name: %s", lookupResult.Items[0].CollectionID, lookupResult.Items[0].CollectionName) if lookupResult.ResultCount > 0 && len(lookupResult.Items) > 0 { lookupResult.Items[0].Summary = entry.Summary.Label @@ -323,10 +352,8 @@ func (p *ITunesProvider) buildChartsURL(options *ChartsOptions) string { } // 添加流派 - if options.Genre != "" { - if genreID, ok := p.GenresMap[options.Genre]; ok && genreID != -1 { - buf += "/genre=" + strconv.Itoa(genreID) - } + if len(options.GenreID) > 0 { + buf += "/genre=" + options.GenreID } // 添加限制内容设置