feat(search): add logging, rate limiting, and genre ID support

Enhance the ITunesProvider by adding logging with detailed caller information, implementing rate limiting to prevent API abuse, and supporting genre IDs for more precise searches. Also, update the User-Agent string and improve error handling and logging throughout the search and charts functionality.

go.mod

@@ -5,8 +5,12 @@ go 1.23
 require github.com/mmcdole/gofeed v1.3.0
 
 require (
+	github.com/benbjohnson/clock v1.3.0 // indirect
 	github.com/jinzhu/inflection v1.0.0 // indirect
 	github.com/jinzhu/now v1.1.5 // indirect
+	github.com/sirupsen/logrus v1.9.3 // indirect
+	go.uber.org/ratelimit v0.3.1 // indirect
+	golang.org/x/sys v0.5.0 // indirect
 )
 
 require (

go.sum

@@ -2,6 +2,8 @@ github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0g
 github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
 github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
 github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
+github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A=
+github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -23,15 +25,23 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
+github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+go.uber.org/ratelimit v0.3.1 h1:K4qVE+byfv/B3tC+4nYWP7v/6SimcO7HzHekoMNBma0=
+go.uber.org/ratelimit v0.3.1/go.mod h1:6euWsTB6U/Nb3X++xEUXA8ciPJvr19Q/0h1+oDcJhRk=
 golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.4.0 h1:Q5QPcMlvfxFTAPV0+07Xz/MpK9NTXu2VDUuy0FeMfaU=
 golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM=
@@ -39,6 +49,8 @@ golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gorm.io/gorm v1.25.12 h1:I0u8i2hWQItBq1WfE0o2+WuL9+8L21K9e2HHSTE/0f8=

main.go

@@ -3,6 +3,7 @@ package main
 import (
 	"fmt"
 
+	"gitea.malaihome.work/ans/podcast-search-go/model"
 	"gitea.malaihome.work/ans/podcast-search-go/search"
 )
 
@@ -11,12 +12,12 @@ func main() {
 	provider := search.NewITunesProvider()
 
 	// podcast, err := provider.SearchPodcasts("golang", &search.SearchOptions{Country: "us", Language: "en-us"})
-	podcast, err := provider.Charts(&search.ChartsOptions{Country: "us", Language: "en-us", Limit: 10})
+	podcast, err := provider.Charts(&search.ChartsOptions{Country: "us", GenreID: model.GetGenreID("en-US", "Art"), Limit: 3})
 	if err != nil {
 		panic(err)
 	}
 
 	// print podcast to json
-	fmt.Printf("podcast: %+v\n", podcast)
+	fmt.Printf("podcast count: %+v\n", len(podcast.Items))
 
 }

model/genre.go

@@ -108,3 +108,13 @@ func GetGenreByLanguage(language string) map[string]string {
 	}
 	return m
 }
+
+func GetGenreID(language string, genre string) string {
+	for genreId, v := range Genres {
+		genreOffical, ok := v[language]
+		if ok && genreOffical == genre {
+			return genreId
+		}
+	}
+	return ""
+}

search/itunes_search.go

@@ -7,10 +7,15 @@ import (
 	"io"
 	"net/http"
 	"net/url"
+	"path"
+	"runtime"
 	"strconv"
+	"strings"
 	"time"
 
 	"gitea.malaihome.work/ans/podcast-search-go/model"
+	"github.com/sirupsen/logrus"
+	"go.uber.org/ratelimit"
 )
 
 // SearchProvider 定义搜索提供者接口
@@ -33,15 +38,34 @@ type ITunesProvider struct {
 	GenresMap map[string]int
 	// HTTP客户端
 	Client  *http.Client
+	Limiter ratelimit.Limiter
+	Logger  *logrus.Logger
 }
 
 // NewITunesProvider 创建新的iTunes搜索提供者
 func NewITunesProvider() *ITunesProvider {
+	// 初始化日志
+	// 启用调用者信息记录
+	Logger := logrus.New()
+	Logger.SetReportCaller(true)
+
+	Logger.SetFormatter(&logrus.TextFormatter{
+		FullTimestamp:   true,
+		TimestampFormat: "2006-01-02 15:04:05.000",
+		CallerPrettyfier: func(f *runtime.Frame) (string, string) {
+			s := strings.Split(f.Function, ".")
+			funcname := s[len(s)-1]
+			_, filename := path.Split(f.File)
+			return funcname, fmt.Sprintf("%s:%d", filename, f.Line)
+		},
+	})
 	return &ITunesProvider{
 		SearchAPIEndpoint: "https://itunes.apple.com/search",
 		FeedAPIEndpoint:   "https://itunes.apple.com",
 		GenresMap:         initITunesGenres(),
 		Client:            &http.Client{Timeout: 20 * time.Second},
+		Limiter:           ratelimit.New(20, ratelimit.Per(time.Minute)), // 20 RPM
+		Logger:            Logger,
 	}
 }
 
@@ -76,7 +100,7 @@ type SearchOptions struct {
 	// 国家
 	Country string
 	// 流派
-	Genre string
+	GenreID string
 	// 属性
 	Attribute string
 	// 限制结果数量
@@ -98,7 +122,7 @@ type ChartsOptions struct {
 	// 是否包含限制内容
 	Explicit bool
 	// 流派
-	Genre string
+	GenreID string
 	// 额外的查询参数
 	QueryParams map[string]string
 }
@@ -120,10 +144,8 @@ func (p *ITunesProvider) SearchPodcasts(term string, options *SearchOptions) (*m
 			params.Add("country", options.Country)
 		}
 
-		if options.Genre != "" {
-			if genreID, ok := p.GenresMap[options.Genre]; ok && genreID != -1 {
-				params.Add("genreId", strconv.Itoa(genreID))
-			}
+		if len(options.GenreID) > 0 {
+			params.Add("genreId", options.GenreID)
 		}
 
 		if options.Attribute != "" {
@@ -139,15 +161,15 @@ func (p *ITunesProvider) SearchPodcasts(term string, options *SearchOptions) (*m
 		}
 
 		if options.Explicit {
-			params.Add("explicit", "Yes")
+			params.Add("explicit", "true")
 		} else {
-			params.Add("explicit", "No")
+			params.Add("explicit", "false")
 		}
 	}
 
 	// 构建URL
 	url := fmt.Sprintf("%s?%s", p.SearchAPIEndpoint, params.Encode())
-	// fmt.Printf("url: %s\n", url)
+	p.Logger.Infof("url: %s", url)
 
 	// 发送请求
 	req, err := http.NewRequest("GET", url, nil)
@@ -156,10 +178,12 @@ func (p *ITunesProvider) SearchPodcasts(term string, options *SearchOptions) (*m
 	}
 
 	// 设置User-Agent
-	req.Header.Set("User-Agent", "PodcastSearch-Go/0.1")
+	req.Header.Set("User-Agent", "podcast-search-go/0.1")
 
 	// 发送请求
+	p.Limiter.Take()
 	resp, err := p.Client.Do(req)
+	p.Logger.Infof("response code: %d", resp.StatusCode)
 	if err != nil {
 		return nil, fmt.Errorf("发送请求失败: %w", err)
 	}
@@ -175,7 +199,6 @@ func (p *ITunesProvider) SearchPodcasts(term string, options *SearchOptions) (*m
 	if err != nil {
 		return nil, fmt.Errorf("读取响应失败: %w", err)
 	}
-	// fmt.Printf("body: %s\n", string(body))
 
 	// 创建搜索结果
 	result := &model.SearchResult2{
@@ -186,6 +209,7 @@ func (p *ITunesProvider) SearchPodcasts(term string, options *SearchOptions) (*m
 	if err := json.Unmarshal(body, &result); err != nil {
 		return nil, fmt.Errorf("解析响应失败: %w", err)
 	}
+	p.Logger.Infof("search result, got %d items", len(result.Items))
 	result.ResultCount = len(result.Items)
 	result.Successful = true
 
@@ -199,7 +223,7 @@ func (p *ITunesProvider) SearchPodcasts(term string, options *SearchOptions) (*m
 func (p *ITunesProvider) Charts(options *ChartsOptions) (*model.SearchResult2, error) {
 	// 构建URL
 	url := p.buildChartsURL(options)
-	// fmt.Printf("url: %s\n", url)
+	p.Logger.Infof("url: %s", url)
 
 	// 发送请求
 	req, err := http.NewRequest("GET", url, nil)
@@ -208,10 +232,13 @@ func (p *ITunesProvider) Charts(options *ChartsOptions) (*model.SearchResult2, e
 	}
 
 	// 设置User-Agent
-	req.Header.Set("User-Agent", "PodcastSearch-Go/0.1")
+	req.Header.Set("User-Agent", "podcast-search-go/0.1")
 
 	// 发送请求
+	p.Limiter.Take()
 	resp, err := p.Client.Do(req)
+	p.Logger.Infof("response code: %d", resp.StatusCode)
+
 	if err != nil {
 		return nil, fmt.Errorf("发送请求失败: %w", err)
 	}
@@ -227,13 +254,12 @@ func (p *ITunesProvider) Charts(options *ChartsOptions) (*model.SearchResult2, e
 	if err != nil {
 		return nil, fmt.Errorf("读取响应失败: %w", err)
 	}
-	// fmt.Printf("body: %s\n", string(body)[:3000])
-	// fmt.Printf("body: %s\n", string(body))
 
 	var chartResult model.ChartResult
 	if err := json.Unmarshal(body, &chartResult); err != nil {
 		return nil, fmt.Errorf("解析响应失败: %w", err)
 	}
+	p.Logger.Infof("chart result, got %d items", len(chartResult.Feed.Entry))
 
 	// 创建搜索结果
 	result := &model.SearchResult2{
@@ -252,13 +278,16 @@ func (p *ITunesProvider) Charts(options *ChartsOptions) (*model.SearchResult2, e
 
 			// 为每个播客获取详细信息
 			lookupURL := fmt.Sprintf("%s/lookup?id=%s", p.FeedAPIEndpoint, id)
+			p.Logger.Infof("lookup url: %s", lookupURL)
 			lookupReq, err := http.NewRequest("GET", lookupURL, nil)
 			if err != nil {
 				continue
 			}
 
-			lookupReq.Header.Set("User-Agent", "PodcastSearch-Go/0.1")
+			req.Header.Set("User-Agent", "podcast-search-go/0.1")
+			p.Limiter.Take()
 			lookupResp, err := p.Client.Do(lookupReq)
+			p.Logger.Infof("lookup response code: %d", lookupResp.StatusCode)
 			if err != nil {
 				continue
 			}
@@ -273,12 +302,12 @@ func (p *ITunesProvider) Charts(options *ChartsOptions) (*model.SearchResult2, e
 			if err != nil {
 				continue
 			}
-			// fmt.Printf("lookupBody: %s\n", string(lookupBody))
 
 			var lookupResult model.LookupResult
 			if err := json.Unmarshal(lookupBody, &lookupResult); err != nil {
 				continue
 			}
+			p.Logger.Infof("lookup result, collection id: %d, collection name: %s", lookupResult.Items[0].CollectionID, lookupResult.Items[0].CollectionName)
 
 			if lookupResult.ResultCount > 0 && len(lookupResult.Items) > 0 {
 				lookupResult.Items[0].Summary = entry.Summary.Label
@@ -323,10 +352,8 @@ func (p *ITunesProvider) buildChartsURL(options *ChartsOptions) string {
 	}
 
 	// 添加流派
-	if options.Genre != "" {
-		if genreID, ok := p.GenresMap[options.Genre]; ok && genreID != -1 {
-			buf += "/genre=" + strconv.Itoa(genreID)
-		}
+	if len(options.GenreID) > 0 {
+		buf += "/genre=" + options.GenreID
 	}
 
 	// 添加限制内容设置