package main import ( "log" "net/url" "strings" "path" "github.com/PuerkitoBio/goquery" "git.1750studios.com/gronkhDE/gogronkh/config" "git.1750studios.com/gronkhDE/gogronkh/database" "git.1750studios.com/gronkhDE/gogronkh/image" ) // Parse new Lets Plays func ParseLetsPlays() { res, err := GetHTTPResource(config.C.GronkhUrl + "/lets-play") if err != nil { log.Printf("ERR LP Page: Request failed (%+v)", err) return } doc, err := goquery.NewDocumentFromResponse(res) if err != nil { log.Printf("ERR LP Page: Document failure (%+v)", err) return } doc.Find(".postpadding > a").Each(func(i int, s *goquery.Selection) { u, _ := s.Attr("href") ur, _ := url.Parse(u) slug := path.Base(ur.Path) var count int if database.Db.Model(database.LetsPlay{}).Unscoped().Where("slug = ?", slug).Count(&count); count > 0 { return } var LP database.LetsPlay LP.Slug.String = slug wg.Add(1) go ParseLPPage(LP.Slug.String, &LP) }) wg.Wait() } func ParseLPPage(gslug string, LP *database.LetsPlay) { defer wg.Done() res, err := GetHTTPResource(config.C.GronkhUrl + "/lets-play/" + gslug) if err != nil { log.Printf("ERR LP %s: Request failed (%+v)", gslug, err) return } doc, err := goquery.NewDocumentFromResponse(res) if err != nil { log.Printf("ERR LP %s: Document failure (%+v)", gslug, err) return } LP.Name.String = doc.Find("h1 > a > strong").First().Text() LP.Name.String = lpRenameRegex.ReplaceAllString(LP.Name.String, "") if LP.Slug.String == "secret-of-mana" { LP.Name.String = "Secret of Mana" } else if LP.Slug.String == "prince-of-persia-2008" { LP.Name.String = "Prince Of Persia (2008)" } else if LP.Slug.String == "half-life-source" { LP.Name.String = "Half-Life: Source" } pos, _ := doc.Find(".lightbox").Attr("href") LP.PosterS.String, LP.PosterB.String, err = image.ResizeCover(pos) if err != nil { log.Printf("WAR LP %s: Error resizing poster: %+v", gslug, err) } au, _ := doc.Find(".author > a.avatar").Attr("href") au = strings.TrimSuffix(au, "/") aur, _ := url.Parse(au) aus := path.Base(aur.Path) if aus == "" || aus == "." { log.Printf("ERR LP %s: No author found", gslug) return } var AT database.Author database.Db.Where("slug = ?", aus).First(&AT) LP.AuthorID = AT.ID if err := database.Db.Create(&LP).Error; err != nil { log.Printf("ERR LP %s: Could not be added to database (%+v)", gslug, err) return } } // Update Lets Plays func UpdateLetsPlays() { var LPs []database.LetsPlay var count int database.Db.Find(&LPs).Count(&count) for _, LP := range LPs { wg.Add(1) go UpdateLP(LP) } wg.Wait() } func UpdateLP(LP database.LetsPlay) { defer wg.Done() if LP.PosterB.Valid == false { res, err := GetHTTPResource(config.C.GronkhUrl + "/lets-play/" + LP.Slug.String) if err != nil { log.Printf("ERR LP %s: Request failed (%+v)", LP.Slug.String, err) return } doc, err := goquery.NewDocumentFromResponse(res) if err != nil { log.Printf("ERR LP %s: Document failure (%+v)", LP.Slug.String, err) return } pos, _ := doc.Find(".lightbox").Attr("href") LP.PosterS.String, LP.PosterB.String, err = image.ResizeCover(pos) if err != nil { log.Printf("WAR LP %s: Error resizing poster: %+v", LP.Slug.String, err) } if err := database.Db.Save(&LP).Error; err != nil { log.Printf("ERR LP %s: Could not be updated in database (%+v)", LP.Slug.String, err) return } } }