Changes instagram parser to use their pagination
Limited to max 5 pages to avoid endless scanning on first start. If the current page contains at least one media element not in our database it will check the next page (max. 5 pages).
This commit is contained in:
parent
cf135a9af5
commit
a97bfc7c14
|
@ -69,9 +69,15 @@ type InstagramPostResponse struct {
|
|||
} `json:"media"`
|
||||
}
|
||||
|
||||
func LoadList() (*InstagramListResponse, error) {
|
||||
func LoadList(token string) (*InstagramListResponse, error) {
|
||||
var listResponse InstagramListResponse
|
||||
res, err := GetHTTPResource("https://www.instagram.com/explore/tags/" + config.C.Instagram.Tag + "/?__a=1")
|
||||
var url string
|
||||
if token == "" {
|
||||
url = "https://www.instagram.com/explore/tags/" + config.C.Instagram.Tag + "/?__a=1"
|
||||
} else {
|
||||
url = "https://www.instagram.com/explore/tags/" + config.C.Instagram.Tag + "/?__a=1&max_id=" + token
|
||||
}
|
||||
res, err := GetHTTPResource(url)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
|
@ -8,16 +8,29 @@ import (
|
|||
|
||||
func LoadNewInstas() {
|
||||
log.Printf("Loading new Instas...")
|
||||
list, err := LoadList()
|
||||
if err != nil {
|
||||
log.Printf("Can't load Instagram feed: %+v", err)
|
||||
return
|
||||
token := ""
|
||||
for i := 0; i < 5; i++ {
|
||||
list, err := LoadList(token)
|
||||
if err != nil {
|
||||
log.Printf("Can't load Instagram feed: %+v", err)
|
||||
return
|
||||
}
|
||||
token = iterList(list)
|
||||
if token == "" {
|
||||
break
|
||||
}
|
||||
}
|
||||
log.Printf("Finished looking for new Instas.")
|
||||
}
|
||||
|
||||
func iterList(list *InstagramListResponse) string {
|
||||
var token string
|
||||
for _, image := range list.Tag.Media.Nodes {
|
||||
var count int
|
||||
if database.Db.Model(database.Item{}).Where("original_id = ?", image.Code).Count(&count); count > 0 {
|
||||
continue
|
||||
}
|
||||
token = list.Tag.Media.PageInfo.EndCursor
|
||||
post, err := LoadPost(image.Code)
|
||||
if err != nil {
|
||||
log.Printf("Can't load Instagram post %s: %+v", image.Code, err)
|
||||
|
@ -90,5 +103,5 @@ func LoadNewInstas() {
|
|||
database.Db.Create(&IT)
|
||||
}
|
||||
}
|
||||
log.Printf("Finished looking for new Instas.")
|
||||
return token
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue