zenermerps/GoGronkh
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

Removing ProgressBar

Browse source
This commit is contained in:
Andreas Mieke 2016-02-27 12:57:14 +01:00
parent 62c2481bd3
commit 8faf206b11
6 changed files with 26 additions and 77 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.gitlab-ci.yml
View file

@ -16,7 +16,6 @@ build:
- go get github.com/lib/pq
- go get github.com/ChannelMeter/iso8601duration
- go get github.com/PuerkitoBio/goquery
- go get github.com/cheggaaa/pb
- go get github.com/robfig/cron
- go get github.com/gin-gonic/gin
- go get github.com/jinzhu/copier

19
gparser/authorparser.go
View file

@ -6,7 +6,6 @@ import (
"path"
"github.com/PuerkitoBio/goquery"
"github.com/cheggaaa/pb"
"git.1750studios.com/gronkhDE/gogronkh/config"
"git.1750studios.com/gronkhDE/gogronkh/database"
@ -16,7 +15,6 @@ import (
// Parse new authors
func ParseAuthors() {
bar := pb.StartNew(0)
res, err := GetHTTPResource(config.C.GronkhUrl + "/lets-player")
if err != nil {
log.Printf("ERR AT Page: Request failed (%+v)", err)
@ -30,17 +28,13 @@ func ParseAuthors() {
// Parse the Episodes
doc.Find("div.column > a").Each(func(i int, s *goquery.Selection) {
wg.Add(1)
go ParseAuthor(i, s, bar)
go ParseAuthor(i, s)
})
wg.Wait()
bar.FinishPrint("Parsed authors")
}
func ParseAuthor(i int, s *goquery.Selection, bar *pb.ProgressBar) {
func ParseAuthor(i int, s *goquery.Selection) {
defer wg.Done()
if bar != nil {
defer bar.Increment()
}
u, _ := s.Attr("href")
ur, _ := url.Parse(u)
slug := path.Base(ur.Path)
@ -96,20 +90,15 @@ func UpdateAuthors() {
var ATs []database.Author
var count int
database.Db.Find(&ATs).Count(&count)
bar := pb.StartNew(count)
for _, AT := range ATs {
wg.Add(1)
go UpdateAuthor(AT, bar)
go UpdateAuthor(AT)
}
wg.Wait()
bar.FinishPrint("Updated authors")
}
func UpdateAuthor(AT database.Author, bar *pb.ProgressBar) {
func UpdateAuthor(AT database.Author) {
defer wg.Done()
if bar != nil {
defer bar.Increment()
}
ytres, err := youtube.GetChannel([]string {"snippet", "brandingSettings"}, AT.Youtube.String, config.C.YoutubeKey)
AT.AvatarS.String, AT.AvatarB.String, err = image.ResizeAvatar(ytres.Items[0].Snippet.Thumbnails["high"].Url)
if err != nil {

25
gparser/episodeparser.go
View file

@ -10,7 +10,6 @@ import (
"github.com/ChannelMeter/iso8601duration"
"github.com/PuerkitoBio/goquery"
"github.com/cheggaaa/pb"
"git.1750studios.com/gronkhDE/gogronkh/config"
"git.1750studios.com/gronkhDE/gogronkh/database"
@ -20,18 +19,16 @@ import (
// Parse new episodes
func ParseEpisodes() {
bar := pb.StartNew(0)
var LPs []database.LetsPlay
database.Db.Find(&LPs)
for _, LP := range LPs {
wg.Add(1)
go ParseLPEpisodes(LP.Slug.String, 1, bar)
go ParseLPEpisodes(LP.Slug.String, 1)
}
wg.Wait()
bar.FinishPrint("Parsed episodes")
}
func ParseLPEpisodes(gslug string, page int, bar *pb.ProgressBar) {
func ParseLPEpisodes(gslug string, page int) {
defer wg.Done()
res, err := GetHTTPResource(config.C.GronkhUrl + "/lets-play/" + gslug + "/page/" + strconv.Itoa(page))
if err != nil {
@ -46,21 +43,18 @@ func ParseLPEpisodes(gslug string, page int, bar *pb.ProgressBar) {
// Parse the Episodes
doc.Find(".entry.entry-letsplay").Each(func(i int, s *goquery.Selection) {
wg.Add(1)
go ParseEpisode(i, s, bar)
go ParseEpisode(i, s)
})
_, found := doc.Find(".next").Attr("href")
if found {
wg.Add(1)
go ParseLPEpisodes(gslug, page+1, bar)
go ParseLPEpisodes(gslug, page+1)
}
}
func ParseEpisode(i int, s *goquery.Selection, bar *pb.ProgressBar) {
func ParseEpisode(i int, s *goquery.Selection) {
defer wg.Done()
if bar != nil {
defer bar.Increment()
}
gu, _ := s.Find("h2 > a").Attr("href")
ur, _ := url.Parse(gu)
slug := path.Base(ur.Path)
@ -162,20 +156,15 @@ func UpdateEpisodes() {
var EPs []database.Episode
var count int
database.Db.Find(&EPs).Count(&count)
bar := pb.StartNew(count)
for _, EP := range EPs {
wg.Add(1)
go UpdateEpisode(EP, bar)
go UpdateEpisode(EP)
}
wg.Wait()
bar.FinishPrint("Updated episodes")
}
func UpdateEpisode(EP database.Episode, bar *pb.ProgressBar) {
func UpdateEpisode(EP database.Episode) {
defer wg.Done()
if bar != nil {
defer bar.Increment()
}
var thumb youtube.Thumb
EP.Rating.Float64, EP.Votes.Int64, _, thumb = youtube.GetRatingAndVotesWithId(EP.Youtube.String, config.C.YoutubeKey)
var LP database.LetsPlay

14
gparser/feedparser.go
View file

@ -12,7 +12,6 @@ import (
"github.com/ChannelMeter/iso8601duration"
"github.com/PuerkitoBio/goquery"
"github.com/cheggaaa/pb"
"git.1750studios.com/gronkhDE/gogronkh/config"
"git.1750studios.com/gronkhDE/gogronkh/database"
@ -49,7 +48,6 @@ func ParseRSSFeeds() {
log.Printf("ERR RS %s: XML failure (%+v)", AT.Slug, err)
continue
}
bar := pb.StartNew(len(Feed.Items))
for _, item := range Feed.Items {
ur, _ := url.Parse(item.Link)
slug := path.Base(ur.Path)
@ -58,23 +56,19 @@ func ParseRSSFeeds() {
var EP database.Episode
database.Db.Where("slug = ?", slug).First(&EP)
wg.Add(1)
go UpdateEpisode(EP, bar)
go UpdateEpisode(EP)
} else {
wg.Add(1)
go ParseFeedEpisode(item.Link, bar)
go ParseFeedEpisode(item.Link)
}
}
wg.Wait()
bar.FinishPrint("Parsed RSS feed for " + AT.Slug.String)
}
GenerateSitemap()
}
func ParseFeedEpisode(u string, bar *pb.ProgressBar) {
func ParseFeedEpisode(u string) {
defer wg.Done()
if bar != nil {
defer bar.Increment()
}
ur, _ := url.Parse(u)
slug := path.Base(ur.Path)
var EP database.Episode
@ -84,7 +78,7 @@ func ParseFeedEpisode(u string, bar *pb.ProgressBar) {
if LP.ID == 0 {
wg.Add(1)
LP.Slug.String = path.Base(path.Dir(ur.Path))
ParseLPPage(LP.Slug.String, &LP, nil)
ParseLPPage(LP.Slug.String, &LP)
if LP.ID == 0 {
return
}

19
gparser/lpparser.go
View file

@ -7,7 +7,6 @@ import (
"path"
"github.com/PuerkitoBio/goquery"
"github.com/cheggaaa/pb"
"git.1750studios.com/gronkhDE/gogronkh/config"
"git.1750studios.com/gronkhDE/gogronkh/database"
@ -16,7 +15,6 @@ import (
// Parse new Lets Plays
func ParseLetsPlays() {
bar := pb.StartNew(0)
res, err := GetHTTPResource(config.C.GronkhUrl + "/lets-play")
if err != nil {
log.Printf("ERR LP Page: Request failed (%+v)", err)
@ -40,17 +38,13 @@ func ParseLetsPlays() {
var LP database.LetsPlay
LP.Slug.String = slug
wg.Add(1)
go ParseLPPage(LP.Slug.String, &LP, bar)
go ParseLPPage(LP.Slug.String, &LP)
})
wg.Wait()
bar.FinishPrint("Parsed lets plays")
}
func ParseLPPage(gslug string, LP *database.LetsPlay, bar *pb.ProgressBar) {
func ParseLPPage(gslug string, LP *database.LetsPlay) {
defer wg.Done()
if bar != nil {
defer bar.Increment()
}
res, err := GetHTTPResource(config.C.GronkhUrl + "/lets-play/" + gslug)
if err != nil {
log.Printf("ERR LP %s: Request failed (%+v)", gslug, err)
@ -95,20 +89,15 @@ func UpdateLetsPlays() {
var LPs []database.LetsPlay
var count int
database.Db.Find(&LPs).Count(&count)
bar := pb.StartNew(count)
for _, LP := range LPs {
wg.Add(1)
go UpdateLP(LP, bar)
go UpdateLP(LP)
}
wg.Wait()
bar.FinishPrint("Updated lets plays")
}
func UpdateLP(LP database.LetsPlay, bar *pb.ProgressBar) {
func UpdateLP(LP database.LetsPlay) {
defer wg.Done()
if bar != nil {
defer bar.Increment()
}
if LP.PosterB.Valid == false {
res, err := GetHTTPResource(config.C.GronkhUrl + "/lets-play/" + LP.Slug.String)
if err != nil {

25
gparser/ltparser.go
View file

@ -10,7 +10,6 @@ import (
"github.com/ChannelMeter/iso8601duration"
"github.com/PuerkitoBio/goquery"
"github.com/cheggaaa/pb"
"git.1750studios.com/gronkhDE/gogronkh/config"
"git.1750studios.com/gronkhDE/gogronkh/database"
@ -20,17 +19,12 @@ import (
// Parse new Lets Tests
func ParseLetsTests() {
bar := pb.StartNew(0)
ParseLTPage(1, bar)
ParseLTPage(1)
wg.Wait()
bar.FinishPrint("Parsed lets tests")
}
func ParseLT(i int, s *goquery.Selection, bar *pb.ProgressBar) {
func ParseLT(i int, s *goquery.Selection) {
defer wg.Done()
if bar != nil {
defer bar.Increment()
}
u, _ := s.Find("h1 > a").Attr("href")
ur, _ := url.Parse(u)
slug := path.Base(ur.Path)
@ -117,7 +111,7 @@ func ParseLT(i int, s *goquery.Selection, bar *pb.ProgressBar) {
}
}
func ParseLTPage(page int, bar *pb.ProgressBar) {
func ParseLTPage(page int) {
res, err := GetHTTPResource(config.C.GronkhUrl + "/testet/page/" + strconv.Itoa(page))
if err != nil {
log.Printf("ERR LT Page: Request failed (%+v)", err)
@ -131,12 +125,12 @@ func ParseLTPage(page int, bar *pb.ProgressBar) {
// Parse the Episodes
doc.Find(".entry.entry-letsplay").Each(func(i int, s *goquery.Selection) {
wg.Add(1)
go ParseLT(i, s, bar)
go ParseLT(i, s)
})
_, found := doc.Find(".next").Attr("href")
if found {
ParseLTPage(page+1, bar)
ParseLTPage(page+1)
}
}
@ -145,20 +139,15 @@ func UpdateLetsTests() {
var LTs []database.LetsTest
var count int
database.Db.Find(&LTs).Count(&count)
bar := pb.StartNew(count)
for _, LT := range LTs {
wg.Add(1)
go UpdateLT(LT, bar)
go UpdateLT(LT)
}
wg.Wait()
bar.FinishPrint("Updated lets tests")
}
func UpdateLT(LT database.LetsTest, bar *pb.ProgressBar) {
func UpdateLT(LT database.LetsTest) {
defer wg.Done()
if bar != nil {
defer bar.Increment()
}
var thumb youtube.Thumb
var err error
LT.Rating.Float64, LT.Votes.Int64, _, thumb = youtube.GetRatingAndVotesWithId(LT.Youtube.String, config.C.YoutubeKey)