diff --git a/go.mod b/go.mod index 217a4e6..9e88de2 100644 --- a/go.mod +++ b/go.mod @@ -5,8 +5,12 @@ go 1.23 require github.com/mmcdole/gofeed v1.3.0 require ( + github.com/benbjohnson/clock v1.3.0 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect + go.uber.org/ratelimit v0.3.1 // indirect + golang.org/x/sys v0.5.0 // indirect ) require ( diff --git a/go.sum b/go.sum index 4c4f837..abac0fc 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,8 @@ github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0g github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI= github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= +github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= +github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -23,15 +25,23 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +go.uber.org/ratelimit v0.3.1 h1:K4qVE+byfv/B3tC+4nYWP7v/6SimcO7HzHekoMNBma0= +go.uber.org/ratelimit v0.3.1/go.mod h1:6euWsTB6U/Nb3X++xEUXA8ciPJvr19Q/0h1+oDcJhRk= golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.4.0 h1:Q5QPcMlvfxFTAPV0+07Xz/MpK9NTXu2VDUuy0FeMfaU= golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM= @@ -39,6 +49,8 @@ golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gorm.io/gorm v1.25.12 h1:I0u8i2hWQItBq1WfE0o2+WuL9+8L21K9e2HHSTE/0f8= diff --git a/main.go b/main.go index c106e82..deb4911 100644 --- a/main.go +++ b/main.go @@ -3,6 +3,7 @@ package main import ( "fmt" + "gitea.malaihome.work/ans/podcast-search-go/model" "gitea.malaihome.work/ans/podcast-search-go/search" ) @@ -11,12 +12,12 @@ func main() { provider := search.NewITunesProvider() // podcast, err := provider.SearchPodcasts("golang", &search.SearchOptions{Country: "us", Language: "en-us"}) - podcast, err := provider.Charts(&search.ChartsOptions{Country: "us", Language: "en-us", Limit: 10}) + podcast, err := provider.Charts(&search.ChartsOptions{Country: "us", GenreID: model.GetGenreID("en-US", "Art"), Limit: 3}) if err != nil { panic(err) } // print podcast to json - fmt.Printf("podcast: %+v\n", podcast) + fmt.Printf("podcast count: %+v\n", len(podcast.Items)) } diff --git a/model/genre.go b/model/genre.go index 08d4f75..fee62b4 100644 --- a/model/genre.go +++ b/model/genre.go @@ -108,3 +108,13 @@ func GetGenreByLanguage(language string) map[string]string { } return m } + +func GetGenreID(language string, genre string) string { + for genreId, v := range Genres { + genreOffical, ok := v[language] + if ok && genreOffical == genre { + return genreId + } + } + return "" +} diff --git a/search/itunes_search.go b/search/itunes_search.go index 27f0fa9..d32b665 100644 --- a/search/itunes_search.go +++ b/search/itunes_search.go @@ -7,10 +7,15 @@ import ( "io" "net/http" "net/url" + "path" + "runtime" "strconv" + "strings" "time" "gitea.malaihome.work/ans/podcast-search-go/model" + "github.com/sirupsen/logrus" + "go.uber.org/ratelimit" ) // SearchProvider 定义搜索提供者接口 @@ -32,16 +37,35 @@ type ITunesProvider struct { // 流派映射 GenresMap map[string]int // HTTP客户端 - Client *http.Client + Client *http.Client + Limiter ratelimit.Limiter + Logger *logrus.Logger } // NewITunesProvider 创建新的iTunes搜索提供者 func NewITunesProvider() *ITunesProvider { + // 初始化日志 + // 启用调用者信息记录 + Logger := logrus.New() + Logger.SetReportCaller(true) + + Logger.SetFormatter(&logrus.TextFormatter{ + FullTimestamp: true, + TimestampFormat: "2006-01-02 15:04:05.000", + CallerPrettyfier: func(f *runtime.Frame) (string, string) { + s := strings.Split(f.Function, ".") + funcname := s[len(s)-1] + _, filename := path.Split(f.File) + return funcname, fmt.Sprintf("%s:%d", filename, f.Line) + }, + }) return &ITunesProvider{ SearchAPIEndpoint: "https://itunes.apple.com/search", FeedAPIEndpoint: "https://itunes.apple.com", GenresMap: initITunesGenres(), Client: &http.Client{Timeout: 20 * time.Second}, + Limiter: ratelimit.New(20, ratelimit.Per(time.Minute)), // 20 RPM + Logger: Logger, } } @@ -76,7 +100,7 @@ type SearchOptions struct { // 国家 Country string // 流派 - Genre string + GenreID string // 属性 Attribute string // 限制结果数量 @@ -98,7 +122,7 @@ type ChartsOptions struct { // 是否包含限制内容 Explicit bool // 流派 - Genre string + GenreID string // 额外的查询参数 QueryParams map[string]string } @@ -120,10 +144,8 @@ func (p *ITunesProvider) SearchPodcasts(term string, options *SearchOptions) (*m params.Add("country", options.Country) } - if options.Genre != "" { - if genreID, ok := p.GenresMap[options.Genre]; ok && genreID != -1 { - params.Add("genreId", strconv.Itoa(genreID)) - } + if len(options.GenreID) > 0 { + params.Add("genreId", options.GenreID) } if options.Attribute != "" { @@ -139,15 +161,15 @@ func (p *ITunesProvider) SearchPodcasts(term string, options *SearchOptions) (*m } if options.Explicit { - params.Add("explicit", "Yes") + params.Add("explicit", "true") } else { - params.Add("explicit", "No") + params.Add("explicit", "false") } } // 构建URL url := fmt.Sprintf("%s?%s", p.SearchAPIEndpoint, params.Encode()) - // fmt.Printf("url: %s\n", url) + p.Logger.Infof("url: %s", url) // 发送请求 req, err := http.NewRequest("GET", url, nil) @@ -156,10 +178,12 @@ func (p *ITunesProvider) SearchPodcasts(term string, options *SearchOptions) (*m } // 设置User-Agent - req.Header.Set("User-Agent", "PodcastSearch-Go/0.1") + req.Header.Set("User-Agent", "podcast-search-go/0.1") // 发送请求 + p.Limiter.Take() resp, err := p.Client.Do(req) + p.Logger.Infof("response code: %d", resp.StatusCode) if err != nil { return nil, fmt.Errorf("发送请求失败: %w", err) } @@ -175,7 +199,6 @@ func (p *ITunesProvider) SearchPodcasts(term string, options *SearchOptions) (*m if err != nil { return nil, fmt.Errorf("读取响应失败: %w", err) } - // fmt.Printf("body: %s\n", string(body)) // 创建搜索结果 result := &model.SearchResult2{ @@ -186,6 +209,7 @@ func (p *ITunesProvider) SearchPodcasts(term string, options *SearchOptions) (*m if err := json.Unmarshal(body, &result); err != nil { return nil, fmt.Errorf("解析响应失败: %w", err) } + p.Logger.Infof("search result, got %d items", len(result.Items)) result.ResultCount = len(result.Items) result.Successful = true @@ -199,7 +223,7 @@ func (p *ITunesProvider) SearchPodcasts(term string, options *SearchOptions) (*m func (p *ITunesProvider) Charts(options *ChartsOptions) (*model.SearchResult2, error) { // 构建URL url := p.buildChartsURL(options) - // fmt.Printf("url: %s\n", url) + p.Logger.Infof("url: %s", url) // 发送请求 req, err := http.NewRequest("GET", url, nil) @@ -208,10 +232,13 @@ func (p *ITunesProvider) Charts(options *ChartsOptions) (*model.SearchResult2, e } // 设置User-Agent - req.Header.Set("User-Agent", "PodcastSearch-Go/0.1") + req.Header.Set("User-Agent", "podcast-search-go/0.1") // 发送请求 + p.Limiter.Take() resp, err := p.Client.Do(req) + p.Logger.Infof("response code: %d", resp.StatusCode) + if err != nil { return nil, fmt.Errorf("发送请求失败: %w", err) } @@ -227,13 +254,12 @@ func (p *ITunesProvider) Charts(options *ChartsOptions) (*model.SearchResult2, e if err != nil { return nil, fmt.Errorf("读取响应失败: %w", err) } - // fmt.Printf("body: %s\n", string(body)[:3000]) - // fmt.Printf("body: %s\n", string(body)) var chartResult model.ChartResult if err := json.Unmarshal(body, &chartResult); err != nil { return nil, fmt.Errorf("解析响应失败: %w", err) } + p.Logger.Infof("chart result, got %d items", len(chartResult.Feed.Entry)) // 创建搜索结果 result := &model.SearchResult2{ @@ -252,13 +278,16 @@ func (p *ITunesProvider) Charts(options *ChartsOptions) (*model.SearchResult2, e // 为每个播客获取详细信息 lookupURL := fmt.Sprintf("%s/lookup?id=%s", p.FeedAPIEndpoint, id) + p.Logger.Infof("lookup url: %s", lookupURL) lookupReq, err := http.NewRequest("GET", lookupURL, nil) if err != nil { continue } - lookupReq.Header.Set("User-Agent", "PodcastSearch-Go/0.1") + req.Header.Set("User-Agent", "podcast-search-go/0.1") + p.Limiter.Take() lookupResp, err := p.Client.Do(lookupReq) + p.Logger.Infof("lookup response code: %d", lookupResp.StatusCode) if err != nil { continue } @@ -273,12 +302,12 @@ func (p *ITunesProvider) Charts(options *ChartsOptions) (*model.SearchResult2, e if err != nil { continue } - // fmt.Printf("lookupBody: %s\n", string(lookupBody)) var lookupResult model.LookupResult if err := json.Unmarshal(lookupBody, &lookupResult); err != nil { continue } + p.Logger.Infof("lookup result, collection id: %d, collection name: %s", lookupResult.Items[0].CollectionID, lookupResult.Items[0].CollectionName) if lookupResult.ResultCount > 0 && len(lookupResult.Items) > 0 { lookupResult.Items[0].Summary = entry.Summary.Label @@ -323,10 +352,8 @@ func (p *ITunesProvider) buildChartsURL(options *ChartsOptions) string { } // 添加流派 - if options.Genre != "" { - if genreID, ok := p.GenresMap[options.Genre]; ok && genreID != -1 { - buf += "/genre=" + strconv.Itoa(genreID) - } + if len(options.GenreID) > 0 { + buf += "/genre=" + options.GenreID } // 添加限制内容设置