package main import ( "log" "net/url" "strconv" "strings" "time" "path" "github.com/ChannelMeter/iso8601duration" "github.com/PuerkitoBio/goquery" "github.com/cheggaaa/pb" "git.1750studios.com/gronkhDE/gogronkh/config" "git.1750studios.com/gronkhDE/gogronkh/database" "git.1750studios.com/gronkhDE/gogronkh/image" "git.1750studios.com/gronkhDE/gogronkh/youtube" ) // Parse new episodes func ParseEpisodes() { bar := pb.StartNew(0) var LPs []database.LetsPlay database.Db.Find(&LPs) for _, LP := range LPs { wg.Add(1) go ParseLPEpisodes(LP.Slug.String, 1, bar) } wg.Wait() bar.FinishPrint("Parsed episodes") } func ParseLPEpisodes(gslug string, page int, bar *pb.ProgressBar) { defer wg.Done() res, err := GetHTTPResource(config.C.GronkhUrl + "/lets-play/" + gslug + "/page/" + strconv.Itoa(page)) if err != nil { log.Printf("ERR LP %s: Request failed (%+v)", gslug, err) return } doc, err := goquery.NewDocumentFromResponse(res) if err != nil { log.Printf("ERR LP %s: Document failure (%+v)", gslug, err) return } // Parse the Episodes doc.Find(".entry.entry-letsplay").Each(func(i int, s *goquery.Selection) { wg.Add(1) go ParseEpisode(i, s, bar) }) _, found := doc.Find(".next").Attr("href") if found { wg.Add(1) go ParseLPEpisodes(gslug, page+1, bar) } } func ParseEpisode(i int, s *goquery.Selection, bar *pb.ProgressBar) { defer wg.Done() if bar != nil { defer bar.Increment() } gu, _ := s.Find("h2 > a").Attr("href") ur, _ := url.Parse(gu) slug := path.Base(ur.Path) var count int if database.Db.Model(database.Episode{}).Where("slug = ?", slug).Count(&count); count > 0 { return } var EP database.Episode EP.Slug.String = slug u, _ := s.Find("h1 > a").Attr("href") ur, _ = url.Parse(u) var LP database.LetsPlay database.Db.Where("slug = ?", path.Base(ur.Path)).First(&LP) EP.LetsPlayID = LP.ID res, err := GetHTTPResource(gu) if err != nil { log.Printf("ERR EP %s: Request failed (%+v)", slug, err) return } doc, err := goquery.NewDocumentFromResponse(res) if err != nil { log.Printf("ERR EP %s: Document failure (%+v)", slug, err) return } EP.Name.String = doc.Find("div.article > h2").First().Text() if EP.Name.String == "" { log.Printf("ERR EP %s: Episode name empty", slug) return } if match := episodeRegex.FindStringSubmatch(EP.Name.String); len(match) > 0 { num, err := strconv.Atoi(match[1]) if err != nil { log.Printf("SUC EP %s: Integer conversion not possible", slug) EP.Episode.Int64 = 0 } EP.Episode.Int64 = int64(num) } else { log.Printf("SUC EP %s: Name does not match RegEx", slug) EP.Episode.Int64 = 0 } doc.Find(".article > p").Each(func(i int, s *goquery.Selection) { EP.Descr.String += s.Text() + "\n" }) EP.Descr.String = strings.Trim(EP.Descr.String, "\n ") au, _ := doc.Find(".author > a.avatar").Attr("href") au = strings.TrimSuffix(au, "/") aur, _ := url.Parse(au) if path.Base(aur.Path) == "" || path.Base(aur.Path) == "." { log.Printf("ERR EP %s: No author found", slug) return } if path.Base(aur.Path) == "tobinator612" { // Don't ask… just… don't ask EP.AuthorID = LP.AuthorID } else { var AT database.Author database.Db.Where("slug = ?", path.Base(aur.Path)).First(&AT) EP.AuthorID = AT.ID } yt, _ := doc.Find(".youtube > iframe").Attr("src") ytpath, _ := url.Parse(yt) EP.Youtube.String = path.Base(ytpath.Path) if EP.Youtube.String == "watch" { EP.Youtube.String = ytpath.Query().Get("v") if idx := strings.LastIndex(EP.Youtube.String, "?"); idx != -1 { EP.Youtube.String = EP.Youtube.String[:idx] } } ytres, err := youtube.GetVideos([]string {"snippet", "statistics", "status", "contentDetails"}, []string {EP.Youtube.String}, config.C.YoutubeKey) if err != nil || len(ytres.Items) == 0 || ytres.Items[0].Status.UploadStatus != "processed" { log.Printf("WAR EP %s: Video %s is private (%+v)", slug, EP.Youtube.String, err) return } if EP.Descr.String == "" { EP.Descr.String = ytres.Items[0].Snippet.Description } if thumb, ok := ytres.Items[0].Snippet.Thumbnails["maxres"]; ok { EP.ThumbS.String, EP.ThumbB.String, err = image.ResizeThumb(thumb.Url) } else if thumb, ok := ytres.Items[0].Snippet.Thumbnails["high"]; ok { EP.ThumbS.String, EP.ThumbB.String, err = image.ResizeThumb(thumb.Url) } if err != nil { log.Printf("WAR EP %s: Error resizing thumbnail: %+v", slug, err) } EP.Aired, err = time.Parse(time.RFC3339Nano, ytres.Items[0].Snippet.PublishedAt) if err != nil { log.Printf("ERR EP %s: Could not parse aired", slug) return } dur, err := duration.FromString(ytres.Items[0].ContentDetails.Duration) if err == nil { EP.Duration.Int64 = int64(dur.ToDuration().Seconds()) } else { log.Printf("SUC EP %s: Could not parse duration", slug) EP.Duration.Int64 = 0 } EP.Rating.Float64, EP.Votes.Int64 = youtube.GetRatingAndVotesWithRes(ytres) if LP.Aired.IsZero() || LP.Aired.After(EP.Aired) { LP.Aired = EP.Aired database.Db.Save(&LP) } if err := database.Db.Create(&EP).Error; err != nil { log.Printf("ERR EP %s: Could not be added to databse (%+v)", slug, err) } else { log.Printf("SUC EP %s: Added to database", slug) } } // Update episodes func UpdateEpisodes() { var EPs []database.Episode var count int database.Db.Find(&EPs).Count(&count) bar := pb.StartNew(count) for _, EP := range EPs { wg.Add(1) go UpdateEpisode(EP, bar) } wg.Wait() bar.FinishPrint("Updated episodes") } func UpdateEpisode(EP database.Episode, bar *pb.ProgressBar) { defer wg.Done() if bar != nil { defer bar.Increment() } var status string var thumb youtube.Thumb EP.Rating.Float64, EP.Votes.Int64, status, thumb = youtube.GetRatingAndVotesWithId(EP.Youtube.String, config.C.YoutubeKey) var LP database.LetsPlay database.Db.Where("ID = ?", EP.LetsPlayID).First(&LP) if LP.Aired.IsZero() || LP.Aired.After(EP.Aired) { LP.Aired = EP.Aired database.Db.Save(&LP) } if EP.ThumbB.Valid == false { var err error EP.ThumbS.String, EP.ThumbB.String, err = image.ResizeThumb(thumb.Url) if err != nil { log.Printf("WAR EP %s: Error resizing thumbnail: %+v", EP.Slug.String, err) } } if err := database.Db.Save(&EP).Error; err != nil { log.Printf("ERR EP %s: Could not be updated in databse (%+v)", EP.Slug.String, err) } else { log.Printf("SUC EP %s: Updated in database", EP.Slug.String) } }