package main import ( "log" "net/url" "strconv" "strings" "time" "path" "github.com/ChannelMeter/iso8601duration" "github.com/PuerkitoBio/goquery" "git.1750studios.com/gronkhDE/gogronkh/config" "git.1750studios.com/gronkhDE/gogronkh/database" "git.1750studios.com/gronkhDE/gogronkh/image" "git.1750studios.com/gronkhDE/gogronkh/youtube" ) // Parse new Lets Tests func ParseLetsTests() { ParseLTPage(1) wg.Wait() } func ParseLT(i int, s *goquery.Selection) { defer wg.Done() u, _ := s.Find("h1 > a").Attr("href") ur, _ := url.Parse(u) slug := path.Base(ur.Path) var count int if database.Db.Model(database.LetsTest{}).Unscoped().Where("slug = ?", slug).Count(&count); count > 0 { return } var LT database.LetsTest LT.Slug.String = slug LT.Name.String = s.Find("h1 > a").First().Text() res, err := GetHTTPResource(u) if err != nil { log.Printf("ERR LT %s: Request failed (%+v)", slug, err) return } doc, err := goquery.NewDocumentFromResponse(res) if err != nil { log.Printf("ERR LT %s: Document failure (%+v)", slug, err) return } pos, _ := doc.Find("div#game-cover > a.lightbox").Attr("href") LT.PosterS.String, LT.PosterB.String, err = image.ResizeCover(pos) if err != nil { log.Printf("WAR LT %s: Error resizing cover: %+v", slug, err) } au, _ := doc.Find(".author > a.avatar").Attr("href") aur, _ := url.Parse(au) if path.Base(aur.Path) == "" || path.Base(aur.Path) == "." { log.Printf("ERR LT %s: No author found", slug) return } var AT database.Author database.Db.Where("slug = ?", path.Base(aur.Path)).First(&AT) LT.AuthorID = AT.ID doc.Find(".article > p").Each(func(i int, s *goquery.Selection) { LT.Descr.String += s.Text() + "\n" }) LT.Descr.String = strings.Trim(LT.Descr.String, "\n ") yt, _ := doc.Find(".youtube > iframe").Attr("src") ytpath, _ := url.Parse(yt) LT.Youtube.String = path.Base(ytpath.Path) if LT.Youtube.String == "watch" { LT.Youtube.String = ytpath.Query().Get("v") if idx := strings.LastIndex(LT.Youtube.String, "?"); idx != -1 { LT.Youtube.String = LT.Youtube.String[:idx] } } ytres, err := youtube.GetVideos([]string {"snippet", "statistics", "status", "contentDetails"}, []string {LT.Youtube.String}, config.C.YoutubeKey) if err != nil || len(ytres.Items) == 0 || ytres.Items[0].Status.UploadStatus != "processed" { log.Printf("ERR LT %s: Video %s is private (%+v)", slug, LT.Youtube.String, err) return } if LT.Descr.String == "" { LT.Descr.String = ytres.Items[0].Snippet.Description } if thumb, ok := ytres.Items[0].Snippet.Thumbnails["maxres"]; ok { LT.ThumbS.String, LT.ThumbB.String, err = image.ResizeThumb(thumb.Url) } else if thumb, ok := ytres.Items[0].Snippet.Thumbnails["high"]; ok { LT.ThumbS.String, LT.ThumbB.String, err = image.ResizeThumb(thumb.Url) } if err != nil { log.Printf("WAR LT %s: Error resizing thumbnail: %+v", slug, err) } LT.Aired, err = time.Parse(time.RFC3339Nano, ytres.Items[0].Snippet.PublishedAt) if err != nil { log.Printf("ERR LT %s: Failed to parse aired", slug) return } dur, err := duration.FromString(ytres.Items[0].ContentDetails.Duration) if err == nil { LT.Duration.Int64 = int64(dur.ToDuration().Seconds()) } else { log.Printf("ERR LT %s: Failed to parse duration", slug) return } LT.Rating.Float64, LT.Votes.Int64 = youtube.GetRatingAndVotesWithRes(ytres) if err := database.Db.Create(<).Error; err != nil { log.Printf("ERR LT %s: Could not be added to databse (%+v)", slug, err) } } func ParseLTPage(page int) { res, err := GetHTTPResource(config.C.GronkhUrl + "/testet/page/" + strconv.Itoa(page)) if err != nil { log.Printf("ERR LT Page: Request failed (%+v)", err) return } doc, err := goquery.NewDocumentFromResponse(res) if err != nil { log.Printf("ERR LT Page: Document failure (%+v)", err) return } // Parse the Episodes doc.Find(".entry.entry-letsplay").Each(func(i int, s *goquery.Selection) { wg.Add(1) go ParseLT(i, s) }) _, found := doc.Find(".next").Attr("href") if found { ParseLTPage(page+1) } } // Update Lets Tests func UpdateLetsTests() { var LTs []database.LetsTest var count int database.Db.Find(<s).Count(&count) for _, LT := range LTs { wg.Add(1) go UpdateLT(LT) } wg.Wait() } func UpdateLT(LT database.LetsTest) { defer wg.Done() var thumb youtube.Thumb var err error LT.Rating.Float64, LT.Votes.Int64, _, thumb = youtube.GetRatingAndVotesWithId(LT.Youtube.String, config.C.YoutubeKey) LT.ThumbS.String, LT.ThumbB.String, err = image.ResizeThumb(thumb.Url) if err != nil { log.Printf("WAR EP %s: Error resizing thumbnail: %+v", LT.Slug.String, err) } if LT.PosterB.Valid == false { res, err := GetHTTPResource(config.C.GronkhUrl + "/testet/" + LT.Slug.String) if err != nil { log.Printf("ERR LT %s: Request failed (%+v)", LT.Slug.String, err) return } doc, err := goquery.NewDocumentFromResponse(res) if err != nil { log.Printf("ERR LT %s: Document failure (%+v)", LT.Slug.String, err) return } pos, _ := doc.Find("div#game-cover > a.lightbox").Attr("href") LT.PosterS.String, LT.PosterB.String, err = image.ResizeCover(pos) if err != nil { log.Printf("WAR LT %s: Error resizing cover: %+v", LT.Slug.String, err) } } if err := database.Db.Save(<).Error; err != nil { log.Printf("ERR LT %s: Could not be updated in databse (%+v)", LT.Slug.String, err) } }