GoGronkh/gparser/ltparser.go

package main

import (
    "log"
    "net/url"
    "strconv"
    "strings"
    "time"
    "path"

    "github.com/ChannelMeter/iso8601duration"
    "github.com/PuerkitoBio/goquery"
    "github.com/cheggaaa/pb"

    "git.1750studios.com/gronkhDE/gogronkh/config"
    "git.1750studios.com/gronkhDE/gogronkh/database"
    "git.1750studios.com/gronkhDE/gogronkh/image"
    "git.1750studios.com/gronkhDE/gogronkh/youtube"
)

// Parse new Lets Tests
func ParseLetsTests() {
    bar := pb.StartNew(0)
    ParseLTPage(1, bar)
    wg.Wait()
    bar.FinishPrint("Parsed lets tests")
}

func ParseLT(i int, s *goquery.Selection, bar *pb.ProgressBar) {
    defer wg.Done()
    if bar != nil {
        defer bar.Increment()
    }
    u, _ := s.Find("h1 > a").Attr("href")
    ur, _ := url.Parse(u)
    slug := path.Base(ur.Path)
    var count int
    if database.Db.Model(database.LetsTest{}).Where("slug = ?", slug).Count(&count); count > 0 {
        return
    }

    var LT database.LetsTest
    LT.Slug.String = slug
    LT.Name.String = s.Find("h1 > a").First().Text()
    res, err := GetHTTPResource(u)
    if err != nil {
        log.Printf("ERR LT %s: Request failed (%+v)", slug, err)
        return
    }
    doc, err := goquery.NewDocumentFromResponse(res)
    if err != nil {
        log.Printf("ERR LT %s: Document failure (%+v)", slug, err)
        return
    }

    pos, _ := doc.Find("div#game-cover > a.lightbox").Attr("href")
    LT.PosterS.String, LT.PosterB.String, err = image.ResizeCover(pos)
    if err != nil {
        log.Printf("WAR LT %s: Error resizing cover: %+v", slug, err)
    }
    au, _ := doc.Find(".author > a.avatar").Attr("href")
    aur, _ := url.Parse(au)
    if path.Base(aur.Path) == "" || path.Base(aur.Path) == "." {
        log.Printf("ERR LT %s: No author found", slug)
        return
    }
    var AT database.Author
    database.Db.Where("slug = ?", path.Base(aur.Path)).First(&AT)
    LT.AuthorID = AT.ID
    doc.Find(".article > p").Each(func(i int, s *goquery.Selection) {
        LT.Descr.String += s.Text() + "\n"
    })
    LT.Descr.String = strings.Trim(LT.Descr.String, "\n ")

    yt, _ := doc.Find(".youtube > iframe").Attr("src")
    ytpath, _ := url.Parse(yt)
    LT.Youtube.String = path.Base(ytpath.Path)
    if LT.Youtube.String == "watch" {
        LT.Youtube.String = ytpath.Query().Get("v")
        if idx := strings.LastIndex(LT.Youtube.String, "?"); idx != -1 {
            LT.Youtube.String = LT.Youtube.String[:idx]
        }
    }
    ytres, err := youtube.GetVideos([]string {"snippet", "statistics", "status", "contentDetails"}, []string {LT.Youtube.String}, config.C.YoutubeKey)
    if err != nil || len(ytres.Items) == 0 || ytres.Items[0].Status.UploadStatus != "processed" {
        log.Printf("WAR LT %s: Video %s is private (%+v)", slug, LT.Youtube.String, err)
        return
    }
    if LT.Descr.String == "" {
        LT.Descr.String = ytres.Items[0].Snippet.Description
    }
    if thumb, ok := ytres.Items[0].Snippet.Thumbnails["maxres"]; ok {
        LT.ThumbS.String, LT.ThumbB.String, err = image.ResizeThumb(thumb.Url)
    } else if thumb, ok := ytres.Items[0].Snippet.Thumbnails["high"]; ok {
        LT.ThumbS.String, LT.ThumbB.String, err = image.ResizeThumb(thumb.Url)
    }
    if err != nil {
        log.Printf("WAR LT %s: Error resizing thumbnail: %+v", slug, err)
    }
    LT.Aired, err = time.Parse(time.RFC3339Nano, ytres.Items[0].Snippet.PublishedAt)
    if err != nil {
        log.Printf("ERR LT %s: Failed to parse aired", slug)
        return
    }
    dur, err := duration.FromString(ytres.Items[0].ContentDetails.Duration)
    if err == nil {
        LT.Duration.Int64 = int64(dur.ToDuration().Seconds())
    } else {
        log.Printf("ERR LT %s: Failed to parse duration", slug)
        return
    }
    LT.Rating.Float64, LT.Votes.Int64 = youtube.GetRatingAndVotesWithRes(ytres)
    if err := database.Db.Create(&LT).Error; err != nil {
        log.Printf("ERR LT %s: Could not be added to databse (%+v)", slug, err)
    } else {
        log.Printf("SUC LT %s: Added to database", slug)
    }
}

func ParseLTPage(page int, bar *pb.ProgressBar) {
    res, err := GetHTTPResource(config.C.GronkhUrl + "/testet/page/" + strconv.Itoa(page))
    if err != nil {
        log.Printf("ERR LT Page: Request failed (%+v)", err)
        return
    }
    doc, err := goquery.NewDocumentFromResponse(res)
    if err != nil {
        log.Printf("ERR LT Page: Document failure (%+v)", err)
        return
    }
    // Parse the Episodes
    doc.Find(".entry.entry-letsplay").Each(func(i int, s *goquery.Selection) {
        wg.Add(1)
        go ParseLT(i, s, bar)
    })

    _, found := doc.Find(".next").Attr("href")
    if found {
        ParseLTPage(page+1, bar)
    }
}

// Update Lets Tests
func UpdateLetsTests() {
    var LTs []database.LetsTest
    var count int
    database.Db.Find(&LTs).Count(&count)
    bar := pb.StartNew(count)
    for _, LT := range LTs {
        wg.Add(1)
        go UpdateLT(LT, bar)
    }
    wg.Wait()
    bar.FinishPrint("Updated lets tests")
}

func UpdateLT(LT database.LetsTest, bar *pb.ProgressBar) {
    defer wg.Done()
    if bar != nil {
        defer bar.Increment()
    }
    var status string
    var thumb youtube.Thumb
    var err error
    LT.Rating.Float64, LT.Votes.Int64, status, thumb = youtube.GetRatingAndVotesWithId(LT.Youtube.String, config.C.YoutubeKey)
    if status != "private" {
        LT.ThumbS.String, LT.ThumbB.String, err = image.ResizeThumb(thumb.Url)
        if err != nil {
            log.Printf("WAR EP %s: Error resizing thumbnail: %+v", LT.Slug.String, err)
        }
        if LT.PosterB.Valid == false {
            res, err := GetHTTPResource(config.C.GronkhUrl + "/testet/" + LT.Slug.String)
            if err != nil {
                log.Printf("ERR LT %s: Request failed (%+v)", LT.Slug.String, err)
                return
            }
            doc, err := goquery.NewDocumentFromResponse(res)
            if err != nil {
                log.Printf("ERR LT %s: Document failure (%+v)", LT.Slug.String, err)
                return
            }
            pos, _ := doc.Find("div#game-cover > a.lightbox").Attr("href")
            LT.PosterS.String, LT.PosterB.String, err = image.ResizeCover(pos)
            if err != nil {
                log.Printf("WAR LT %s: Error resizing cover: %+v", LT.Slug.String, err)
            }
        }
        if err := database.Db.Save(&LT).Error; err != nil {
            log.Printf("ERR LT %s: Could not be updated in databse (%+v)", LT.Slug.String, err)
        } else {
            log.Printf("SUC LT %s: Updated in database", LT.Slug.String)
        }
    } else {
        database.Db.Delete(&LT)
        log.Printf("SUC LT %s: Removed from database", LT.Slug.String)
    }
}