Changes instagram parser to use their pagination
Limited to max 5 pages to avoid endless scanning on first start. If the current page contains at least one media element not in our database it will check the next page (max. 5 pages).
This commit is contained in:
parent
cf135a9af5
commit
a97bfc7c14
|
@ -69,9 +69,15 @@ type InstagramPostResponse struct {
|
||||||
} `json:"media"`
|
} `json:"media"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func LoadList() (*InstagramListResponse, error) {
|
func LoadList(token string) (*InstagramListResponse, error) {
|
||||||
var listResponse InstagramListResponse
|
var listResponse InstagramListResponse
|
||||||
res, err := GetHTTPResource("https://www.instagram.com/explore/tags/" + config.C.Instagram.Tag + "/?__a=1")
|
var url string
|
||||||
|
if token == "" {
|
||||||
|
url = "https://www.instagram.com/explore/tags/" + config.C.Instagram.Tag + "/?__a=1"
|
||||||
|
} else {
|
||||||
|
url = "https://www.instagram.com/explore/tags/" + config.C.Instagram.Tag + "/?__a=1&max_id=" + token
|
||||||
|
}
|
||||||
|
res, err := GetHTTPResource(url)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
|
@ -8,16 +8,29 @@ import (
|
||||||
|
|
||||||
func LoadNewInstas() {
|
func LoadNewInstas() {
|
||||||
log.Printf("Loading new Instas...")
|
log.Printf("Loading new Instas...")
|
||||||
list, err := LoadList()
|
token := ""
|
||||||
|
for i := 0; i < 5; i++ {
|
||||||
|
list, err := LoadList(token)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Printf("Can't load Instagram feed: %+v", err)
|
log.Printf("Can't load Instagram feed: %+v", err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
token = iterList(list)
|
||||||
|
if token == "" {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
log.Printf("Finished looking for new Instas.")
|
||||||
|
}
|
||||||
|
|
||||||
|
func iterList(list *InstagramListResponse) string {
|
||||||
|
var token string
|
||||||
for _, image := range list.Tag.Media.Nodes {
|
for _, image := range list.Tag.Media.Nodes {
|
||||||
var count int
|
var count int
|
||||||
if database.Db.Model(database.Item{}).Where("original_id = ?", image.Code).Count(&count); count > 0 {
|
if database.Db.Model(database.Item{}).Where("original_id = ?", image.Code).Count(&count); count > 0 {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
token = list.Tag.Media.PageInfo.EndCursor
|
||||||
post, err := LoadPost(image.Code)
|
post, err := LoadPost(image.Code)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Printf("Can't load Instagram post %s: %+v", image.Code, err)
|
log.Printf("Can't load Instagram post %s: %+v", image.Code, err)
|
||||||
|
@ -90,5 +103,5 @@ func LoadNewInstas() {
|
||||||
database.Db.Create(&IT)
|
database.Db.Create(&IT)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
log.Printf("Finished looking for new Instas.")
|
return token
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in a new issue