diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index abe9553..1c72fc5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -16,7 +16,6 @@ build: - go get github.com/lib/pq - go get github.com/ChannelMeter/iso8601duration - go get github.com/PuerkitoBio/goquery - - go get github.com/cheggaaa/pb - go get github.com/robfig/cron - go get github.com/gin-gonic/gin - go get github.com/jinzhu/copier diff --git a/gparser/authorparser.go b/gparser/authorparser.go index 8d2f649..1461c6a 100644 --- a/gparser/authorparser.go +++ b/gparser/authorparser.go @@ -6,7 +6,6 @@ import ( "path" "github.com/PuerkitoBio/goquery" - "github.com/cheggaaa/pb" "git.1750studios.com/gronkhDE/gogronkh/config" "git.1750studios.com/gronkhDE/gogronkh/database" @@ -16,7 +15,6 @@ import ( // Parse new authors func ParseAuthors() { - bar := pb.StartNew(0) res, err := GetHTTPResource(config.C.GronkhUrl + "/lets-player") if err != nil { log.Printf("ERR AT Page: Request failed (%+v)", err) @@ -30,17 +28,13 @@ func ParseAuthors() { // Parse the Episodes doc.Find("div.column > a").Each(func(i int, s *goquery.Selection) { wg.Add(1) - go ParseAuthor(i, s, bar) + go ParseAuthor(i, s) }) wg.Wait() - bar.FinishPrint("Parsed authors") } -func ParseAuthor(i int, s *goquery.Selection, bar *pb.ProgressBar) { +func ParseAuthor(i int, s *goquery.Selection) { defer wg.Done() - if bar != nil { - defer bar.Increment() - } u, _ := s.Attr("href") ur, _ := url.Parse(u) slug := path.Base(ur.Path) @@ -96,20 +90,15 @@ func UpdateAuthors() { var ATs []database.Author var count int database.Db.Find(&ATs).Count(&count) - bar := pb.StartNew(count) for _, AT := range ATs { wg.Add(1) - go UpdateAuthor(AT, bar) + go UpdateAuthor(AT) } wg.Wait() - bar.FinishPrint("Updated authors") } -func UpdateAuthor(AT database.Author, bar *pb.ProgressBar) { +func UpdateAuthor(AT database.Author) { defer wg.Done() - if bar != nil { - defer bar.Increment() - } ytres, err := youtube.GetChannel([]string {"snippet", "brandingSettings"}, AT.Youtube.String, config.C.YoutubeKey) AT.AvatarS.String, AT.AvatarB.String, err = image.ResizeAvatar(ytres.Items[0].Snippet.Thumbnails["high"].Url) if err != nil { diff --git a/gparser/episodeparser.go b/gparser/episodeparser.go index abef63b..b205024 100644 --- a/gparser/episodeparser.go +++ b/gparser/episodeparser.go @@ -10,7 +10,6 @@ import ( "github.com/ChannelMeter/iso8601duration" "github.com/PuerkitoBio/goquery" - "github.com/cheggaaa/pb" "git.1750studios.com/gronkhDE/gogronkh/config" "git.1750studios.com/gronkhDE/gogronkh/database" @@ -20,18 +19,16 @@ import ( // Parse new episodes func ParseEpisodes() { - bar := pb.StartNew(0) var LPs []database.LetsPlay database.Db.Find(&LPs) for _, LP := range LPs { wg.Add(1) - go ParseLPEpisodes(LP.Slug.String, 1, bar) + go ParseLPEpisodes(LP.Slug.String, 1) } wg.Wait() - bar.FinishPrint("Parsed episodes") } -func ParseLPEpisodes(gslug string, page int, bar *pb.ProgressBar) { +func ParseLPEpisodes(gslug string, page int) { defer wg.Done() res, err := GetHTTPResource(config.C.GronkhUrl + "/lets-play/" + gslug + "/page/" + strconv.Itoa(page)) if err != nil { @@ -46,21 +43,18 @@ func ParseLPEpisodes(gslug string, page int, bar *pb.ProgressBar) { // Parse the Episodes doc.Find(".entry.entry-letsplay").Each(func(i int, s *goquery.Selection) { wg.Add(1) - go ParseEpisode(i, s, bar) + go ParseEpisode(i, s) }) _, found := doc.Find(".next").Attr("href") if found { wg.Add(1) - go ParseLPEpisodes(gslug, page+1, bar) + go ParseLPEpisodes(gslug, page+1) } } -func ParseEpisode(i int, s *goquery.Selection, bar *pb.ProgressBar) { +func ParseEpisode(i int, s *goquery.Selection) { defer wg.Done() - if bar != nil { - defer bar.Increment() - } gu, _ := s.Find("h2 > a").Attr("href") ur, _ := url.Parse(gu) slug := path.Base(ur.Path) @@ -162,20 +156,15 @@ func UpdateEpisodes() { var EPs []database.Episode var count int database.Db.Find(&EPs).Count(&count) - bar := pb.StartNew(count) for _, EP := range EPs { wg.Add(1) - go UpdateEpisode(EP, bar) + go UpdateEpisode(EP) } wg.Wait() - bar.FinishPrint("Updated episodes") } -func UpdateEpisode(EP database.Episode, bar *pb.ProgressBar) { +func UpdateEpisode(EP database.Episode) { defer wg.Done() - if bar != nil { - defer bar.Increment() - } var thumb youtube.Thumb EP.Rating.Float64, EP.Votes.Int64, _, thumb = youtube.GetRatingAndVotesWithId(EP.Youtube.String, config.C.YoutubeKey) var LP database.LetsPlay diff --git a/gparser/feedparser.go b/gparser/feedparser.go index a6ce218..6ac146f 100644 --- a/gparser/feedparser.go +++ b/gparser/feedparser.go @@ -12,7 +12,6 @@ import ( "github.com/ChannelMeter/iso8601duration" "github.com/PuerkitoBio/goquery" - "github.com/cheggaaa/pb" "git.1750studios.com/gronkhDE/gogronkh/config" "git.1750studios.com/gronkhDE/gogronkh/database" @@ -49,7 +48,6 @@ func ParseRSSFeeds() { log.Printf("ERR RS %s: XML failure (%+v)", AT.Slug, err) continue } - bar := pb.StartNew(len(Feed.Items)) for _, item := range Feed.Items { ur, _ := url.Parse(item.Link) slug := path.Base(ur.Path) @@ -58,23 +56,19 @@ func ParseRSSFeeds() { var EP database.Episode database.Db.Where("slug = ?", slug).First(&EP) wg.Add(1) - go UpdateEpisode(EP, bar) + go UpdateEpisode(EP) } else { wg.Add(1) - go ParseFeedEpisode(item.Link, bar) + go ParseFeedEpisode(item.Link) } } wg.Wait() - bar.FinishPrint("Parsed RSS feed for " + AT.Slug.String) } GenerateSitemap() } -func ParseFeedEpisode(u string, bar *pb.ProgressBar) { +func ParseFeedEpisode(u string) { defer wg.Done() - if bar != nil { - defer bar.Increment() - } ur, _ := url.Parse(u) slug := path.Base(ur.Path) var EP database.Episode @@ -84,7 +78,7 @@ func ParseFeedEpisode(u string, bar *pb.ProgressBar) { if LP.ID == 0 { wg.Add(1) LP.Slug.String = path.Base(path.Dir(ur.Path)) - ParseLPPage(LP.Slug.String, &LP, nil) + ParseLPPage(LP.Slug.String, &LP) if LP.ID == 0 { return } diff --git a/gparser/lpparser.go b/gparser/lpparser.go index a25da8f..3b51f73 100644 --- a/gparser/lpparser.go +++ b/gparser/lpparser.go @@ -7,7 +7,6 @@ import ( "path" "github.com/PuerkitoBio/goquery" - "github.com/cheggaaa/pb" "git.1750studios.com/gronkhDE/gogronkh/config" "git.1750studios.com/gronkhDE/gogronkh/database" @@ -16,7 +15,6 @@ import ( // Parse new Lets Plays func ParseLetsPlays() { - bar := pb.StartNew(0) res, err := GetHTTPResource(config.C.GronkhUrl + "/lets-play") if err != nil { log.Printf("ERR LP Page: Request failed (%+v)", err) @@ -40,17 +38,13 @@ func ParseLetsPlays() { var LP database.LetsPlay LP.Slug.String = slug wg.Add(1) - go ParseLPPage(LP.Slug.String, &LP, bar) + go ParseLPPage(LP.Slug.String, &LP) }) wg.Wait() - bar.FinishPrint("Parsed lets plays") } -func ParseLPPage(gslug string, LP *database.LetsPlay, bar *pb.ProgressBar) { +func ParseLPPage(gslug string, LP *database.LetsPlay) { defer wg.Done() - if bar != nil { - defer bar.Increment() - } res, err := GetHTTPResource(config.C.GronkhUrl + "/lets-play/" + gslug) if err != nil { log.Printf("ERR LP %s: Request failed (%+v)", gslug, err) @@ -95,20 +89,15 @@ func UpdateLetsPlays() { var LPs []database.LetsPlay var count int database.Db.Find(&LPs).Count(&count) - bar := pb.StartNew(count) for _, LP := range LPs { wg.Add(1) - go UpdateLP(LP, bar) + go UpdateLP(LP) } wg.Wait() - bar.FinishPrint("Updated lets plays") } -func UpdateLP(LP database.LetsPlay, bar *pb.ProgressBar) { +func UpdateLP(LP database.LetsPlay) { defer wg.Done() - if bar != nil { - defer bar.Increment() - } if LP.PosterB.Valid == false { res, err := GetHTTPResource(config.C.GronkhUrl + "/lets-play/" + LP.Slug.String) if err != nil { diff --git a/gparser/ltparser.go b/gparser/ltparser.go index 1b7e190..b33e15e 100644 --- a/gparser/ltparser.go +++ b/gparser/ltparser.go @@ -10,7 +10,6 @@ import ( "github.com/ChannelMeter/iso8601duration" "github.com/PuerkitoBio/goquery" - "github.com/cheggaaa/pb" "git.1750studios.com/gronkhDE/gogronkh/config" "git.1750studios.com/gronkhDE/gogronkh/database" @@ -20,17 +19,12 @@ import ( // Parse new Lets Tests func ParseLetsTests() { - bar := pb.StartNew(0) - ParseLTPage(1, bar) + ParseLTPage(1) wg.Wait() - bar.FinishPrint("Parsed lets tests") } -func ParseLT(i int, s *goquery.Selection, bar *pb.ProgressBar) { +func ParseLT(i int, s *goquery.Selection) { defer wg.Done() - if bar != nil { - defer bar.Increment() - } u, _ := s.Find("h1 > a").Attr("href") ur, _ := url.Parse(u) slug := path.Base(ur.Path) @@ -117,7 +111,7 @@ func ParseLT(i int, s *goquery.Selection, bar *pb.ProgressBar) { } } -func ParseLTPage(page int, bar *pb.ProgressBar) { +func ParseLTPage(page int) { res, err := GetHTTPResource(config.C.GronkhUrl + "/testet/page/" + strconv.Itoa(page)) if err != nil { log.Printf("ERR LT Page: Request failed (%+v)", err) @@ -131,12 +125,12 @@ func ParseLTPage(page int, bar *pb.ProgressBar) { // Parse the Episodes doc.Find(".entry.entry-letsplay").Each(func(i int, s *goquery.Selection) { wg.Add(1) - go ParseLT(i, s, bar) + go ParseLT(i, s) }) _, found := doc.Find(".next").Attr("href") if found { - ParseLTPage(page+1, bar) + ParseLTPage(page+1) } } @@ -145,20 +139,15 @@ func UpdateLetsTests() { var LTs []database.LetsTest var count int database.Db.Find(<s).Count(&count) - bar := pb.StartNew(count) for _, LT := range LTs { wg.Add(1) - go UpdateLT(LT, bar) + go UpdateLT(LT) } wg.Wait() - bar.FinishPrint("Updated lets tests") } -func UpdateLT(LT database.LetsTest, bar *pb.ProgressBar) { +func UpdateLT(LT database.LetsTest) { defer wg.Done() - if bar != nil { - defer bar.Increment() - } var thumb youtube.Thumb var err error LT.Rating.Float64, LT.Votes.Int64, _, thumb = youtube.GetRatingAndVotesWithId(LT.Youtube.String, config.C.YoutubeKey)