package main import ( "log" "net/url" "strings" "path" "github.com/PuerkitoBio/goquery" "github.com/cheggaaa/pb" "git.1750studios.com/gronkhDE/gogronkh/config" "git.1750studios.com/gronkhDE/gogronkh/database" "git.1750studios.com/gronkhDE/gogronkh/image" ) // Parse new Lets Plays func ParseLetsPlays() { bar := pb.StartNew(0) res, err := GetHTTPResource(config.C.GronkhUrl + "/lets-play") if err != nil { log.Printf("ERR LP Page: Request failed (%+v)", err) return } doc, err := goquery.NewDocumentFromResponse(res) if err != nil { log.Printf("ERR LP Page: Document failure (%+v)", err) return } doc.Find(".postpadding > a").Each(func(i int, s *goquery.Selection) { u, _ := s.Attr("href") ur, _ := url.Parse(u) slug := path.Base(ur.Path) var count int if database.Db.Model(database.LetsPlay{}).Where("slug = ?", slug).Count(&count); count > 0 { return } var LP database.LetsPlay LP.Slug.String = slug LP.Name.String, _ = s.Attr("title") wg.Add(1) go ParseLPPage(LP.Slug.String, &LP, bar) }) wg.Wait() bar.FinishPrint("Parsed lets plays") } func ParseLPPage(gslug string, LP *database.LetsPlay, bar *pb.ProgressBar) { defer wg.Done() if bar != nil { defer bar.Increment() } res, err := GetHTTPResource(config.C.GronkhUrl + "/lets-play/" + gslug) if err != nil { log.Printf("ERR LP %s: Request failed (%+v)", gslug, err) return } doc, err := goquery.NewDocumentFromResponse(res) if err != nil { log.Printf("ERR LP %s: Document failure (%+v)", gslug, err) return } pos, _ := doc.Find(".lightbox").Attr("href") LP.PosterS.String, LP.PosterB.String, err = image.ResizeCover(pos) if err != nil { log.Printf("WAR LP %s: Error resizing poster: %+v", gslug, err) } au, _ := doc.Find(".author > a.avatar").Attr("href") au = strings.TrimSuffix(au, "/") aur, _ := url.Parse(au) aus := path.Base(aur.Path) if aus == "" || aus == "." { log.Printf("ERR LP %s: No author found", gslug) return } var AT database.Author database.Db.Where("slug = ?", aus).First(&AT) LP.AuthorID = AT.ID if err := database.Db.Create(&LP).Error; err != nil { log.Printf("ERR LP %s: Could not be added to database (%+v)", gslug, err) return } else { log.Printf("SUC LP %s: Added to database", gslug) } } // Update Lets Plays func UpdateLetsPlays() { var LPs []database.LetsPlay var count int database.Db.Find(&LPs).Count(&count) bar := pb.StartNew(count) for _, LP := range LPs { wg.Add(1) go UpdateLP(LP, bar) } wg.Wait() bar.FinishPrint("Updated lets plays") } func UpdateLP(LP database.LetsPlay, bar *pb.ProgressBar) { defer wg.Done() if bar != nil { defer bar.Increment() } if LP.PosterB.Valid == false { res, err := GetHTTPResource(config.C.GronkhUrl + "/lets-play/" + LP.Slug.String) if err != nil { log.Printf("ERR LP %s: Request failed (%+v)", LP.Slug.String, err) return } doc, err := goquery.NewDocumentFromResponse(res) if err != nil { log.Printf("ERR LP %s: Document failure (%+v)", LP.Slug.String, err) return } pos, _ := doc.Find(".lightbox").Attr("href") LP.PosterS.String, LP.PosterB.String, err = image.ResizeCover(pos) if err != nil { log.Printf("WAR LP %s: Error resizing poster: %+v", LP.Slug.String, err) } if err := database.Db.Save(&LP).Error; err != nil { log.Printf("ERR LP %s: Could not be updated in database (%+v)", LP.Slug.String, err) return } else { log.Printf("SUC LP %s: Updated in database", LP.Slug.String) } } }