Removing ProgressBar
This commit is contained in:
parent
62c2481bd3
commit
8faf206b11
|
@ -16,7 +16,6 @@ build:
|
|||
- go get github.com/lib/pq
|
||||
- go get github.com/ChannelMeter/iso8601duration
|
||||
- go get github.com/PuerkitoBio/goquery
|
||||
- go get github.com/cheggaaa/pb
|
||||
- go get github.com/robfig/cron
|
||||
- go get github.com/gin-gonic/gin
|
||||
- go get github.com/jinzhu/copier
|
||||
|
|
|
@ -6,7 +6,6 @@ import (
|
|||
"path"
|
||||
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
"github.com/cheggaaa/pb"
|
||||
|
||||
"git.1750studios.com/gronkhDE/gogronkh/config"
|
||||
"git.1750studios.com/gronkhDE/gogronkh/database"
|
||||
|
@ -16,7 +15,6 @@ import (
|
|||
|
||||
// Parse new authors
|
||||
func ParseAuthors() {
|
||||
bar := pb.StartNew(0)
|
||||
res, err := GetHTTPResource(config.C.GronkhUrl + "/lets-player")
|
||||
if err != nil {
|
||||
log.Printf("ERR AT Page: Request failed (%+v)", err)
|
||||
|
@ -30,17 +28,13 @@ func ParseAuthors() {
|
|||
// Parse the Episodes
|
||||
doc.Find("div.column > a").Each(func(i int, s *goquery.Selection) {
|
||||
wg.Add(1)
|
||||
go ParseAuthor(i, s, bar)
|
||||
go ParseAuthor(i, s)
|
||||
})
|
||||
wg.Wait()
|
||||
bar.FinishPrint("Parsed authors")
|
||||
}
|
||||
|
||||
func ParseAuthor(i int, s *goquery.Selection, bar *pb.ProgressBar) {
|
||||
func ParseAuthor(i int, s *goquery.Selection) {
|
||||
defer wg.Done()
|
||||
if bar != nil {
|
||||
defer bar.Increment()
|
||||
}
|
||||
u, _ := s.Attr("href")
|
||||
ur, _ := url.Parse(u)
|
||||
slug := path.Base(ur.Path)
|
||||
|
@ -96,20 +90,15 @@ func UpdateAuthors() {
|
|||
var ATs []database.Author
|
||||
var count int
|
||||
database.Db.Find(&ATs).Count(&count)
|
||||
bar := pb.StartNew(count)
|
||||
for _, AT := range ATs {
|
||||
wg.Add(1)
|
||||
go UpdateAuthor(AT, bar)
|
||||
go UpdateAuthor(AT)
|
||||
}
|
||||
wg.Wait()
|
||||
bar.FinishPrint("Updated authors")
|
||||
}
|
||||
|
||||
func UpdateAuthor(AT database.Author, bar *pb.ProgressBar) {
|
||||
func UpdateAuthor(AT database.Author) {
|
||||
defer wg.Done()
|
||||
if bar != nil {
|
||||
defer bar.Increment()
|
||||
}
|
||||
ytres, err := youtube.GetChannel([]string {"snippet", "brandingSettings"}, AT.Youtube.String, config.C.YoutubeKey)
|
||||
AT.AvatarS.String, AT.AvatarB.String, err = image.ResizeAvatar(ytres.Items[0].Snippet.Thumbnails["high"].Url)
|
||||
if err != nil {
|
||||
|
|
|
@ -10,7 +10,6 @@ import (
|
|||
|
||||
"github.com/ChannelMeter/iso8601duration"
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
"github.com/cheggaaa/pb"
|
||||
|
||||
"git.1750studios.com/gronkhDE/gogronkh/config"
|
||||
"git.1750studios.com/gronkhDE/gogronkh/database"
|
||||
|
@ -20,18 +19,16 @@ import (
|
|||
|
||||
// Parse new episodes
|
||||
func ParseEpisodes() {
|
||||
bar := pb.StartNew(0)
|
||||
var LPs []database.LetsPlay
|
||||
database.Db.Find(&LPs)
|
||||
for _, LP := range LPs {
|
||||
wg.Add(1)
|
||||
go ParseLPEpisodes(LP.Slug.String, 1, bar)
|
||||
go ParseLPEpisodes(LP.Slug.String, 1)
|
||||
}
|
||||
wg.Wait()
|
||||
bar.FinishPrint("Parsed episodes")
|
||||
}
|
||||
|
||||
func ParseLPEpisodes(gslug string, page int, bar *pb.ProgressBar) {
|
||||
func ParseLPEpisodes(gslug string, page int) {
|
||||
defer wg.Done()
|
||||
res, err := GetHTTPResource(config.C.GronkhUrl + "/lets-play/" + gslug + "/page/" + strconv.Itoa(page))
|
||||
if err != nil {
|
||||
|
@ -46,21 +43,18 @@ func ParseLPEpisodes(gslug string, page int, bar *pb.ProgressBar) {
|
|||
// Parse the Episodes
|
||||
doc.Find(".entry.entry-letsplay").Each(func(i int, s *goquery.Selection) {
|
||||
wg.Add(1)
|
||||
go ParseEpisode(i, s, bar)
|
||||
go ParseEpisode(i, s)
|
||||
})
|
||||
|
||||
_, found := doc.Find(".next").Attr("href")
|
||||
if found {
|
||||
wg.Add(1)
|
||||
go ParseLPEpisodes(gslug, page+1, bar)
|
||||
go ParseLPEpisodes(gslug, page+1)
|
||||
}
|
||||
}
|
||||
|
||||
func ParseEpisode(i int, s *goquery.Selection, bar *pb.ProgressBar) {
|
||||
func ParseEpisode(i int, s *goquery.Selection) {
|
||||
defer wg.Done()
|
||||
if bar != nil {
|
||||
defer bar.Increment()
|
||||
}
|
||||
gu, _ := s.Find("h2 > a").Attr("href")
|
||||
ur, _ := url.Parse(gu)
|
||||
slug := path.Base(ur.Path)
|
||||
|
@ -162,20 +156,15 @@ func UpdateEpisodes() {
|
|||
var EPs []database.Episode
|
||||
var count int
|
||||
database.Db.Find(&EPs).Count(&count)
|
||||
bar := pb.StartNew(count)
|
||||
for _, EP := range EPs {
|
||||
wg.Add(1)
|
||||
go UpdateEpisode(EP, bar)
|
||||
go UpdateEpisode(EP)
|
||||
}
|
||||
wg.Wait()
|
||||
bar.FinishPrint("Updated episodes")
|
||||
}
|
||||
|
||||
func UpdateEpisode(EP database.Episode, bar *pb.ProgressBar) {
|
||||
func UpdateEpisode(EP database.Episode) {
|
||||
defer wg.Done()
|
||||
if bar != nil {
|
||||
defer bar.Increment()
|
||||
}
|
||||
var thumb youtube.Thumb
|
||||
EP.Rating.Float64, EP.Votes.Int64, _, thumb = youtube.GetRatingAndVotesWithId(EP.Youtube.String, config.C.YoutubeKey)
|
||||
var LP database.LetsPlay
|
||||
|
|
|
@ -12,7 +12,6 @@ import (
|
|||
|
||||
"github.com/ChannelMeter/iso8601duration"
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
"github.com/cheggaaa/pb"
|
||||
|
||||
"git.1750studios.com/gronkhDE/gogronkh/config"
|
||||
"git.1750studios.com/gronkhDE/gogronkh/database"
|
||||
|
@ -49,7 +48,6 @@ func ParseRSSFeeds() {
|
|||
log.Printf("ERR RS %s: XML failure (%+v)", AT.Slug, err)
|
||||
continue
|
||||
}
|
||||
bar := pb.StartNew(len(Feed.Items))
|
||||
for _, item := range Feed.Items {
|
||||
ur, _ := url.Parse(item.Link)
|
||||
slug := path.Base(ur.Path)
|
||||
|
@ -58,23 +56,19 @@ func ParseRSSFeeds() {
|
|||
var EP database.Episode
|
||||
database.Db.Where("slug = ?", slug).First(&EP)
|
||||
wg.Add(1)
|
||||
go UpdateEpisode(EP, bar)
|
||||
go UpdateEpisode(EP)
|
||||
} else {
|
||||
wg.Add(1)
|
||||
go ParseFeedEpisode(item.Link, bar)
|
||||
go ParseFeedEpisode(item.Link)
|
||||
}
|
||||
}
|
||||
wg.Wait()
|
||||
bar.FinishPrint("Parsed RSS feed for " + AT.Slug.String)
|
||||
}
|
||||
GenerateSitemap()
|
||||
}
|
||||
|
||||
func ParseFeedEpisode(u string, bar *pb.ProgressBar) {
|
||||
func ParseFeedEpisode(u string) {
|
||||
defer wg.Done()
|
||||
if bar != nil {
|
||||
defer bar.Increment()
|
||||
}
|
||||
ur, _ := url.Parse(u)
|
||||
slug := path.Base(ur.Path)
|
||||
var EP database.Episode
|
||||
|
@ -84,7 +78,7 @@ func ParseFeedEpisode(u string, bar *pb.ProgressBar) {
|
|||
if LP.ID == 0 {
|
||||
wg.Add(1)
|
||||
LP.Slug.String = path.Base(path.Dir(ur.Path))
|
||||
ParseLPPage(LP.Slug.String, &LP, nil)
|
||||
ParseLPPage(LP.Slug.String, &LP)
|
||||
if LP.ID == 0 {
|
||||
return
|
||||
}
|
||||
|
|
|
@ -7,7 +7,6 @@ import (
|
|||
"path"
|
||||
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
"github.com/cheggaaa/pb"
|
||||
|
||||
"git.1750studios.com/gronkhDE/gogronkh/config"
|
||||
"git.1750studios.com/gronkhDE/gogronkh/database"
|
||||
|
@ -16,7 +15,6 @@ import (
|
|||
|
||||
// Parse new Lets Plays
|
||||
func ParseLetsPlays() {
|
||||
bar := pb.StartNew(0)
|
||||
res, err := GetHTTPResource(config.C.GronkhUrl + "/lets-play")
|
||||
if err != nil {
|
||||
log.Printf("ERR LP Page: Request failed (%+v)", err)
|
||||
|
@ -40,17 +38,13 @@ func ParseLetsPlays() {
|
|||
var LP database.LetsPlay
|
||||
LP.Slug.String = slug
|
||||
wg.Add(1)
|
||||
go ParseLPPage(LP.Slug.String, &LP, bar)
|
||||
go ParseLPPage(LP.Slug.String, &LP)
|
||||
})
|
||||
wg.Wait()
|
||||
bar.FinishPrint("Parsed lets plays")
|
||||
}
|
||||
|
||||
func ParseLPPage(gslug string, LP *database.LetsPlay, bar *pb.ProgressBar) {
|
||||
func ParseLPPage(gslug string, LP *database.LetsPlay) {
|
||||
defer wg.Done()
|
||||
if bar != nil {
|
||||
defer bar.Increment()
|
||||
}
|
||||
res, err := GetHTTPResource(config.C.GronkhUrl + "/lets-play/" + gslug)
|
||||
if err != nil {
|
||||
log.Printf("ERR LP %s: Request failed (%+v)", gslug, err)
|
||||
|
@ -95,20 +89,15 @@ func UpdateLetsPlays() {
|
|||
var LPs []database.LetsPlay
|
||||
var count int
|
||||
database.Db.Find(&LPs).Count(&count)
|
||||
bar := pb.StartNew(count)
|
||||
for _, LP := range LPs {
|
||||
wg.Add(1)
|
||||
go UpdateLP(LP, bar)
|
||||
go UpdateLP(LP)
|
||||
}
|
||||
wg.Wait()
|
||||
bar.FinishPrint("Updated lets plays")
|
||||
}
|
||||
|
||||
func UpdateLP(LP database.LetsPlay, bar *pb.ProgressBar) {
|
||||
func UpdateLP(LP database.LetsPlay) {
|
||||
defer wg.Done()
|
||||
if bar != nil {
|
||||
defer bar.Increment()
|
||||
}
|
||||
if LP.PosterB.Valid == false {
|
||||
res, err := GetHTTPResource(config.C.GronkhUrl + "/lets-play/" + LP.Slug.String)
|
||||
if err != nil {
|
||||
|
|
|
@ -10,7 +10,6 @@ import (
|
|||
|
||||
"github.com/ChannelMeter/iso8601duration"
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
"github.com/cheggaaa/pb"
|
||||
|
||||
"git.1750studios.com/gronkhDE/gogronkh/config"
|
||||
"git.1750studios.com/gronkhDE/gogronkh/database"
|
||||
|
@ -20,17 +19,12 @@ import (
|
|||
|
||||
// Parse new Lets Tests
|
||||
func ParseLetsTests() {
|
||||
bar := pb.StartNew(0)
|
||||
ParseLTPage(1, bar)
|
||||
ParseLTPage(1)
|
||||
wg.Wait()
|
||||
bar.FinishPrint("Parsed lets tests")
|
||||
}
|
||||
|
||||
func ParseLT(i int, s *goquery.Selection, bar *pb.ProgressBar) {
|
||||
func ParseLT(i int, s *goquery.Selection) {
|
||||
defer wg.Done()
|
||||
if bar != nil {
|
||||
defer bar.Increment()
|
||||
}
|
||||
u, _ := s.Find("h1 > a").Attr("href")
|
||||
ur, _ := url.Parse(u)
|
||||
slug := path.Base(ur.Path)
|
||||
|
@ -117,7 +111,7 @@ func ParseLT(i int, s *goquery.Selection, bar *pb.ProgressBar) {
|
|||
}
|
||||
}
|
||||
|
||||
func ParseLTPage(page int, bar *pb.ProgressBar) {
|
||||
func ParseLTPage(page int) {
|
||||
res, err := GetHTTPResource(config.C.GronkhUrl + "/testet/page/" + strconv.Itoa(page))
|
||||
if err != nil {
|
||||
log.Printf("ERR LT Page: Request failed (%+v)", err)
|
||||
|
@ -131,12 +125,12 @@ func ParseLTPage(page int, bar *pb.ProgressBar) {
|
|||
// Parse the Episodes
|
||||
doc.Find(".entry.entry-letsplay").Each(func(i int, s *goquery.Selection) {
|
||||
wg.Add(1)
|
||||
go ParseLT(i, s, bar)
|
||||
go ParseLT(i, s)
|
||||
})
|
||||
|
||||
_, found := doc.Find(".next").Attr("href")
|
||||
if found {
|
||||
ParseLTPage(page+1, bar)
|
||||
ParseLTPage(page+1)
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -145,20 +139,15 @@ func UpdateLetsTests() {
|
|||
var LTs []database.LetsTest
|
||||
var count int
|
||||
database.Db.Find(<s).Count(&count)
|
||||
bar := pb.StartNew(count)
|
||||
for _, LT := range LTs {
|
||||
wg.Add(1)
|
||||
go UpdateLT(LT, bar)
|
||||
go UpdateLT(LT)
|
||||
}
|
||||
wg.Wait()
|
||||
bar.FinishPrint("Updated lets tests")
|
||||
}
|
||||
|
||||
func UpdateLT(LT database.LetsTest, bar *pb.ProgressBar) {
|
||||
func UpdateLT(LT database.LetsTest) {
|
||||
defer wg.Done()
|
||||
if bar != nil {
|
||||
defer bar.Increment()
|
||||
}
|
||||
var thumb youtube.Thumb
|
||||
var err error
|
||||
LT.Rating.Float64, LT.Votes.Int64, _, thumb = youtube.GetRatingAndVotesWithId(LT.Youtube.String, config.C.YoutubeKey)
|
||||
|
|
Loading…
Reference in a new issue