GoGronkh/gparser/lpparser.go

package main

import (
    "log"
    "net/url"
    "strings"
    "path"

    "github.com/PuerkitoBio/goquery"
    "github.com/cheggaaa/pb"

    "git.1750studios.com/gronkhDE/gogronkh/config"
    "git.1750studios.com/gronkhDE/gogronkh/database"
    "git.1750studios.com/gronkhDE/gogronkh/image"
)

// Parse new Lets Plays
func ParseLetsPlays() {
    bar := pb.StartNew(0)
    res, err := GetHTTPResource(config.C.GronkhUrl + "/lets-play")
    if err != nil {
        log.Printf("ERR LP Page: Request failed (%+v)", err)
        return
    }
    doc, err := goquery.NewDocumentFromResponse(res)
    if err != nil {
        log.Printf("ERR LP Page: Document failure (%+v)", err)
        return
    }

    doc.Find(".postpadding > a").Each(func(i int, s *goquery.Selection) {
        u, _ := s.Attr("href")
        ur, _ := url.Parse(u)
        slug := path.Base(ur.Path)
        var count int
        if database.Db.Model(database.LetsPlay{}).Where("slug = ?", slug).Count(&count); count > 0 {
            return
        }

        var LP database.LetsPlay
        LP.Slug.String = slug
        wg.Add(1)
        go ParseLPPage(LP.Slug.String, &LP, bar)
    })
    wg.Wait()
    bar.FinishPrint("Parsed lets plays")
}

func ParseLPPage(gslug string, LP *database.LetsPlay, bar *pb.ProgressBar) {
    defer wg.Done()
    if bar != nil {
        defer bar.Increment()
    }
    res, err := GetHTTPResource(config.C.GronkhUrl + "/lets-play/" + gslug)
    if err != nil {
        log.Printf("ERR LP %s: Request failed (%+v)", gslug, err)
        return
    }
    doc, err := goquery.NewDocumentFromResponse(res)
    if err != nil {
        log.Printf("ERR LP %s: Document failure (%+v)", gslug, err)
        return
    }

    LP.Name.String = doc.Find("h1 > a > strong").First().Text()

    pos, _ := doc.Find(".lightbox").Attr("href")
    LP.PosterS.String, LP.PosterB.String, err = image.ResizeCover(pos)
    if err != nil {
        log.Printf("WAR LP %s: Error resizing poster: %+v", gslug, err)
    }

    au, _ := doc.Find(".author > a.avatar").Attr("href")
    au = strings.TrimSuffix(au, "/")
    aur, _ := url.Parse(au)
    aus := path.Base(aur.Path)
    if aus == "" || aus == "." {
        log.Printf("ERR LP %s: No author found", gslug)
        return
    }
    var AT database.Author
    database.Db.Where("slug = ?", aus).First(&AT)
    LP.AuthorID = AT.ID

    if err := database.Db.Create(&LP).Error; err != nil {
        log.Printf("ERR LP %s: Could not be added to database (%+v)", gslug, err)
        return
    } else {
        log.Printf("SUC LP %s: Added to database", gslug)
    }
}

// Update Lets Plays
func UpdateLetsPlays() {
    var LPs []database.LetsPlay
    var count int
    database.Db.Find(&LPs).Count(&count)
    bar := pb.StartNew(count)
    for _, LP := range LPs {
        wg.Add(1)
        go UpdateLP(LP, bar)
    }
    wg.Wait()
    bar.FinishPrint("Updated lets plays")
}

func UpdateLP(LP database.LetsPlay, bar *pb.ProgressBar) {
    defer wg.Done()
    if bar != nil {
        defer bar.Increment()
    }
    if LP.PosterB.Valid == false {
        res, err := GetHTTPResource(config.C.GronkhUrl + "/lets-play/" + LP.Slug.String)
        if err != nil {
            log.Printf("ERR LP %s: Request failed (%+v)", LP.Slug.String, err)
            return
        }
        doc, err := goquery.NewDocumentFromResponse(res)
        if err != nil {
            log.Printf("ERR LP %s: Document failure (%+v)", LP.Slug.String, err)
            return
        }

        pos, _ := doc.Find(".lightbox").Attr("href")
        LP.PosterS.String, LP.PosterB.String, err = image.ResizeCover(pos)
        if err != nil {
            log.Printf("WAR LP %s: Error resizing poster: %+v", LP.Slug.String, err)
        }

        if err := database.Db.Save(&LP).Error; err != nil {
            log.Printf("ERR LP %s: Could not be updated in database (%+v)", LP.Slug.String, err)
            return
        } else {
            log.Printf("SUC LP %s: Updated in database", LP.Slug.String)
        }
    }
}

func DeleteEmptyLPs() {
    var LPs []database.LetsPlay
    database.Db.Find(&LPs)
    for _, LP := range LPs {
        var count int
        database.Db.Model(database.Episode{}).Where("lets_play_id = ?", LP.ID).Count(&count)
        if count == 0 {
            database.Db.Delete(&LP)
        }
    }
}