Removing ProgressBar
This commit is contained in:
parent
62c2481bd3
commit
8faf206b11
|
@ -16,7 +16,6 @@ build:
|
||||||
- go get github.com/lib/pq
|
- go get github.com/lib/pq
|
||||||
- go get github.com/ChannelMeter/iso8601duration
|
- go get github.com/ChannelMeter/iso8601duration
|
||||||
- go get github.com/PuerkitoBio/goquery
|
- go get github.com/PuerkitoBio/goquery
|
||||||
- go get github.com/cheggaaa/pb
|
|
||||||
- go get github.com/robfig/cron
|
- go get github.com/robfig/cron
|
||||||
- go get github.com/gin-gonic/gin
|
- go get github.com/gin-gonic/gin
|
||||||
- go get github.com/jinzhu/copier
|
- go get github.com/jinzhu/copier
|
||||||
|
|
|
@ -6,7 +6,6 @@ import (
|
||||||
"path"
|
"path"
|
||||||
|
|
||||||
"github.com/PuerkitoBio/goquery"
|
"github.com/PuerkitoBio/goquery"
|
||||||
"github.com/cheggaaa/pb"
|
|
||||||
|
|
||||||
"git.1750studios.com/gronkhDE/gogronkh/config"
|
"git.1750studios.com/gronkhDE/gogronkh/config"
|
||||||
"git.1750studios.com/gronkhDE/gogronkh/database"
|
"git.1750studios.com/gronkhDE/gogronkh/database"
|
||||||
|
@ -16,7 +15,6 @@ import (
|
||||||
|
|
||||||
// Parse new authors
|
// Parse new authors
|
||||||
func ParseAuthors() {
|
func ParseAuthors() {
|
||||||
bar := pb.StartNew(0)
|
|
||||||
res, err := GetHTTPResource(config.C.GronkhUrl + "/lets-player")
|
res, err := GetHTTPResource(config.C.GronkhUrl + "/lets-player")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Printf("ERR AT Page: Request failed (%+v)", err)
|
log.Printf("ERR AT Page: Request failed (%+v)", err)
|
||||||
|
@ -30,17 +28,13 @@ func ParseAuthors() {
|
||||||
// Parse the Episodes
|
// Parse the Episodes
|
||||||
doc.Find("div.column > a").Each(func(i int, s *goquery.Selection) {
|
doc.Find("div.column > a").Each(func(i int, s *goquery.Selection) {
|
||||||
wg.Add(1)
|
wg.Add(1)
|
||||||
go ParseAuthor(i, s, bar)
|
go ParseAuthor(i, s)
|
||||||
})
|
})
|
||||||
wg.Wait()
|
wg.Wait()
|
||||||
bar.FinishPrint("Parsed authors")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func ParseAuthor(i int, s *goquery.Selection, bar *pb.ProgressBar) {
|
func ParseAuthor(i int, s *goquery.Selection) {
|
||||||
defer wg.Done()
|
defer wg.Done()
|
||||||
if bar != nil {
|
|
||||||
defer bar.Increment()
|
|
||||||
}
|
|
||||||
u, _ := s.Attr("href")
|
u, _ := s.Attr("href")
|
||||||
ur, _ := url.Parse(u)
|
ur, _ := url.Parse(u)
|
||||||
slug := path.Base(ur.Path)
|
slug := path.Base(ur.Path)
|
||||||
|
@ -96,20 +90,15 @@ func UpdateAuthors() {
|
||||||
var ATs []database.Author
|
var ATs []database.Author
|
||||||
var count int
|
var count int
|
||||||
database.Db.Find(&ATs).Count(&count)
|
database.Db.Find(&ATs).Count(&count)
|
||||||
bar := pb.StartNew(count)
|
|
||||||
for _, AT := range ATs {
|
for _, AT := range ATs {
|
||||||
wg.Add(1)
|
wg.Add(1)
|
||||||
go UpdateAuthor(AT, bar)
|
go UpdateAuthor(AT)
|
||||||
}
|
}
|
||||||
wg.Wait()
|
wg.Wait()
|
||||||
bar.FinishPrint("Updated authors")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func UpdateAuthor(AT database.Author, bar *pb.ProgressBar) {
|
func UpdateAuthor(AT database.Author) {
|
||||||
defer wg.Done()
|
defer wg.Done()
|
||||||
if bar != nil {
|
|
||||||
defer bar.Increment()
|
|
||||||
}
|
|
||||||
ytres, err := youtube.GetChannel([]string {"snippet", "brandingSettings"}, AT.Youtube.String, config.C.YoutubeKey)
|
ytres, err := youtube.GetChannel([]string {"snippet", "brandingSettings"}, AT.Youtube.String, config.C.YoutubeKey)
|
||||||
AT.AvatarS.String, AT.AvatarB.String, err = image.ResizeAvatar(ytres.Items[0].Snippet.Thumbnails["high"].Url)
|
AT.AvatarS.String, AT.AvatarB.String, err = image.ResizeAvatar(ytres.Items[0].Snippet.Thumbnails["high"].Url)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
|
@ -10,7 +10,6 @@ import (
|
||||||
|
|
||||||
"github.com/ChannelMeter/iso8601duration"
|
"github.com/ChannelMeter/iso8601duration"
|
||||||
"github.com/PuerkitoBio/goquery"
|
"github.com/PuerkitoBio/goquery"
|
||||||
"github.com/cheggaaa/pb"
|
|
||||||
|
|
||||||
"git.1750studios.com/gronkhDE/gogronkh/config"
|
"git.1750studios.com/gronkhDE/gogronkh/config"
|
||||||
"git.1750studios.com/gronkhDE/gogronkh/database"
|
"git.1750studios.com/gronkhDE/gogronkh/database"
|
||||||
|
@ -20,18 +19,16 @@ import (
|
||||||
|
|
||||||
// Parse new episodes
|
// Parse new episodes
|
||||||
func ParseEpisodes() {
|
func ParseEpisodes() {
|
||||||
bar := pb.StartNew(0)
|
|
||||||
var LPs []database.LetsPlay
|
var LPs []database.LetsPlay
|
||||||
database.Db.Find(&LPs)
|
database.Db.Find(&LPs)
|
||||||
for _, LP := range LPs {
|
for _, LP := range LPs {
|
||||||
wg.Add(1)
|
wg.Add(1)
|
||||||
go ParseLPEpisodes(LP.Slug.String, 1, bar)
|
go ParseLPEpisodes(LP.Slug.String, 1)
|
||||||
}
|
}
|
||||||
wg.Wait()
|
wg.Wait()
|
||||||
bar.FinishPrint("Parsed episodes")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func ParseLPEpisodes(gslug string, page int, bar *pb.ProgressBar) {
|
func ParseLPEpisodes(gslug string, page int) {
|
||||||
defer wg.Done()
|
defer wg.Done()
|
||||||
res, err := GetHTTPResource(config.C.GronkhUrl + "/lets-play/" + gslug + "/page/" + strconv.Itoa(page))
|
res, err := GetHTTPResource(config.C.GronkhUrl + "/lets-play/" + gslug + "/page/" + strconv.Itoa(page))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -46,21 +43,18 @@ func ParseLPEpisodes(gslug string, page int, bar *pb.ProgressBar) {
|
||||||
// Parse the Episodes
|
// Parse the Episodes
|
||||||
doc.Find(".entry.entry-letsplay").Each(func(i int, s *goquery.Selection) {
|
doc.Find(".entry.entry-letsplay").Each(func(i int, s *goquery.Selection) {
|
||||||
wg.Add(1)
|
wg.Add(1)
|
||||||
go ParseEpisode(i, s, bar)
|
go ParseEpisode(i, s)
|
||||||
})
|
})
|
||||||
|
|
||||||
_, found := doc.Find(".next").Attr("href")
|
_, found := doc.Find(".next").Attr("href")
|
||||||
if found {
|
if found {
|
||||||
wg.Add(1)
|
wg.Add(1)
|
||||||
go ParseLPEpisodes(gslug, page+1, bar)
|
go ParseLPEpisodes(gslug, page+1)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func ParseEpisode(i int, s *goquery.Selection, bar *pb.ProgressBar) {
|
func ParseEpisode(i int, s *goquery.Selection) {
|
||||||
defer wg.Done()
|
defer wg.Done()
|
||||||
if bar != nil {
|
|
||||||
defer bar.Increment()
|
|
||||||
}
|
|
||||||
gu, _ := s.Find("h2 > a").Attr("href")
|
gu, _ := s.Find("h2 > a").Attr("href")
|
||||||
ur, _ := url.Parse(gu)
|
ur, _ := url.Parse(gu)
|
||||||
slug := path.Base(ur.Path)
|
slug := path.Base(ur.Path)
|
||||||
|
@ -162,20 +156,15 @@ func UpdateEpisodes() {
|
||||||
var EPs []database.Episode
|
var EPs []database.Episode
|
||||||
var count int
|
var count int
|
||||||
database.Db.Find(&EPs).Count(&count)
|
database.Db.Find(&EPs).Count(&count)
|
||||||
bar := pb.StartNew(count)
|
|
||||||
for _, EP := range EPs {
|
for _, EP := range EPs {
|
||||||
wg.Add(1)
|
wg.Add(1)
|
||||||
go UpdateEpisode(EP, bar)
|
go UpdateEpisode(EP)
|
||||||
}
|
}
|
||||||
wg.Wait()
|
wg.Wait()
|
||||||
bar.FinishPrint("Updated episodes")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func UpdateEpisode(EP database.Episode, bar *pb.ProgressBar) {
|
func UpdateEpisode(EP database.Episode) {
|
||||||
defer wg.Done()
|
defer wg.Done()
|
||||||
if bar != nil {
|
|
||||||
defer bar.Increment()
|
|
||||||
}
|
|
||||||
var thumb youtube.Thumb
|
var thumb youtube.Thumb
|
||||||
EP.Rating.Float64, EP.Votes.Int64, _, thumb = youtube.GetRatingAndVotesWithId(EP.Youtube.String, config.C.YoutubeKey)
|
EP.Rating.Float64, EP.Votes.Int64, _, thumb = youtube.GetRatingAndVotesWithId(EP.Youtube.String, config.C.YoutubeKey)
|
||||||
var LP database.LetsPlay
|
var LP database.LetsPlay
|
||||||
|
|
|
@ -12,7 +12,6 @@ import (
|
||||||
|
|
||||||
"github.com/ChannelMeter/iso8601duration"
|
"github.com/ChannelMeter/iso8601duration"
|
||||||
"github.com/PuerkitoBio/goquery"
|
"github.com/PuerkitoBio/goquery"
|
||||||
"github.com/cheggaaa/pb"
|
|
||||||
|
|
||||||
"git.1750studios.com/gronkhDE/gogronkh/config"
|
"git.1750studios.com/gronkhDE/gogronkh/config"
|
||||||
"git.1750studios.com/gronkhDE/gogronkh/database"
|
"git.1750studios.com/gronkhDE/gogronkh/database"
|
||||||
|
@ -49,7 +48,6 @@ func ParseRSSFeeds() {
|
||||||
log.Printf("ERR RS %s: XML failure (%+v)", AT.Slug, err)
|
log.Printf("ERR RS %s: XML failure (%+v)", AT.Slug, err)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
bar := pb.StartNew(len(Feed.Items))
|
|
||||||
for _, item := range Feed.Items {
|
for _, item := range Feed.Items {
|
||||||
ur, _ := url.Parse(item.Link)
|
ur, _ := url.Parse(item.Link)
|
||||||
slug := path.Base(ur.Path)
|
slug := path.Base(ur.Path)
|
||||||
|
@ -58,23 +56,19 @@ func ParseRSSFeeds() {
|
||||||
var EP database.Episode
|
var EP database.Episode
|
||||||
database.Db.Where("slug = ?", slug).First(&EP)
|
database.Db.Where("slug = ?", slug).First(&EP)
|
||||||
wg.Add(1)
|
wg.Add(1)
|
||||||
go UpdateEpisode(EP, bar)
|
go UpdateEpisode(EP)
|
||||||
} else {
|
} else {
|
||||||
wg.Add(1)
|
wg.Add(1)
|
||||||
go ParseFeedEpisode(item.Link, bar)
|
go ParseFeedEpisode(item.Link)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
wg.Wait()
|
wg.Wait()
|
||||||
bar.FinishPrint("Parsed RSS feed for " + AT.Slug.String)
|
|
||||||
}
|
}
|
||||||
GenerateSitemap()
|
GenerateSitemap()
|
||||||
}
|
}
|
||||||
|
|
||||||
func ParseFeedEpisode(u string, bar *pb.ProgressBar) {
|
func ParseFeedEpisode(u string) {
|
||||||
defer wg.Done()
|
defer wg.Done()
|
||||||
if bar != nil {
|
|
||||||
defer bar.Increment()
|
|
||||||
}
|
|
||||||
ur, _ := url.Parse(u)
|
ur, _ := url.Parse(u)
|
||||||
slug := path.Base(ur.Path)
|
slug := path.Base(ur.Path)
|
||||||
var EP database.Episode
|
var EP database.Episode
|
||||||
|
@ -84,7 +78,7 @@ func ParseFeedEpisode(u string, bar *pb.ProgressBar) {
|
||||||
if LP.ID == 0 {
|
if LP.ID == 0 {
|
||||||
wg.Add(1)
|
wg.Add(1)
|
||||||
LP.Slug.String = path.Base(path.Dir(ur.Path))
|
LP.Slug.String = path.Base(path.Dir(ur.Path))
|
||||||
ParseLPPage(LP.Slug.String, &LP, nil)
|
ParseLPPage(LP.Slug.String, &LP)
|
||||||
if LP.ID == 0 {
|
if LP.ID == 0 {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
|
@ -7,7 +7,6 @@ import (
|
||||||
"path"
|
"path"
|
||||||
|
|
||||||
"github.com/PuerkitoBio/goquery"
|
"github.com/PuerkitoBio/goquery"
|
||||||
"github.com/cheggaaa/pb"
|
|
||||||
|
|
||||||
"git.1750studios.com/gronkhDE/gogronkh/config"
|
"git.1750studios.com/gronkhDE/gogronkh/config"
|
||||||
"git.1750studios.com/gronkhDE/gogronkh/database"
|
"git.1750studios.com/gronkhDE/gogronkh/database"
|
||||||
|
@ -16,7 +15,6 @@ import (
|
||||||
|
|
||||||
// Parse new Lets Plays
|
// Parse new Lets Plays
|
||||||
func ParseLetsPlays() {
|
func ParseLetsPlays() {
|
||||||
bar := pb.StartNew(0)
|
|
||||||
res, err := GetHTTPResource(config.C.GronkhUrl + "/lets-play")
|
res, err := GetHTTPResource(config.C.GronkhUrl + "/lets-play")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Printf("ERR LP Page: Request failed (%+v)", err)
|
log.Printf("ERR LP Page: Request failed (%+v)", err)
|
||||||
|
@ -40,17 +38,13 @@ func ParseLetsPlays() {
|
||||||
var LP database.LetsPlay
|
var LP database.LetsPlay
|
||||||
LP.Slug.String = slug
|
LP.Slug.String = slug
|
||||||
wg.Add(1)
|
wg.Add(1)
|
||||||
go ParseLPPage(LP.Slug.String, &LP, bar)
|
go ParseLPPage(LP.Slug.String, &LP)
|
||||||
})
|
})
|
||||||
wg.Wait()
|
wg.Wait()
|
||||||
bar.FinishPrint("Parsed lets plays")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func ParseLPPage(gslug string, LP *database.LetsPlay, bar *pb.ProgressBar) {
|
func ParseLPPage(gslug string, LP *database.LetsPlay) {
|
||||||
defer wg.Done()
|
defer wg.Done()
|
||||||
if bar != nil {
|
|
||||||
defer bar.Increment()
|
|
||||||
}
|
|
||||||
res, err := GetHTTPResource(config.C.GronkhUrl + "/lets-play/" + gslug)
|
res, err := GetHTTPResource(config.C.GronkhUrl + "/lets-play/" + gslug)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Printf("ERR LP %s: Request failed (%+v)", gslug, err)
|
log.Printf("ERR LP %s: Request failed (%+v)", gslug, err)
|
||||||
|
@ -95,20 +89,15 @@ func UpdateLetsPlays() {
|
||||||
var LPs []database.LetsPlay
|
var LPs []database.LetsPlay
|
||||||
var count int
|
var count int
|
||||||
database.Db.Find(&LPs).Count(&count)
|
database.Db.Find(&LPs).Count(&count)
|
||||||
bar := pb.StartNew(count)
|
|
||||||
for _, LP := range LPs {
|
for _, LP := range LPs {
|
||||||
wg.Add(1)
|
wg.Add(1)
|
||||||
go UpdateLP(LP, bar)
|
go UpdateLP(LP)
|
||||||
}
|
}
|
||||||
wg.Wait()
|
wg.Wait()
|
||||||
bar.FinishPrint("Updated lets plays")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func UpdateLP(LP database.LetsPlay, bar *pb.ProgressBar) {
|
func UpdateLP(LP database.LetsPlay) {
|
||||||
defer wg.Done()
|
defer wg.Done()
|
||||||
if bar != nil {
|
|
||||||
defer bar.Increment()
|
|
||||||
}
|
|
||||||
if LP.PosterB.Valid == false {
|
if LP.PosterB.Valid == false {
|
||||||
res, err := GetHTTPResource(config.C.GronkhUrl + "/lets-play/" + LP.Slug.String)
|
res, err := GetHTTPResource(config.C.GronkhUrl + "/lets-play/" + LP.Slug.String)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
|
@ -10,7 +10,6 @@ import (
|
||||||
|
|
||||||
"github.com/ChannelMeter/iso8601duration"
|
"github.com/ChannelMeter/iso8601duration"
|
||||||
"github.com/PuerkitoBio/goquery"
|
"github.com/PuerkitoBio/goquery"
|
||||||
"github.com/cheggaaa/pb"
|
|
||||||
|
|
||||||
"git.1750studios.com/gronkhDE/gogronkh/config"
|
"git.1750studios.com/gronkhDE/gogronkh/config"
|
||||||
"git.1750studios.com/gronkhDE/gogronkh/database"
|
"git.1750studios.com/gronkhDE/gogronkh/database"
|
||||||
|
@ -20,17 +19,12 @@ import (
|
||||||
|
|
||||||
// Parse new Lets Tests
|
// Parse new Lets Tests
|
||||||
func ParseLetsTests() {
|
func ParseLetsTests() {
|
||||||
bar := pb.StartNew(0)
|
ParseLTPage(1)
|
||||||
ParseLTPage(1, bar)
|
|
||||||
wg.Wait()
|
wg.Wait()
|
||||||
bar.FinishPrint("Parsed lets tests")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func ParseLT(i int, s *goquery.Selection, bar *pb.ProgressBar) {
|
func ParseLT(i int, s *goquery.Selection) {
|
||||||
defer wg.Done()
|
defer wg.Done()
|
||||||
if bar != nil {
|
|
||||||
defer bar.Increment()
|
|
||||||
}
|
|
||||||
u, _ := s.Find("h1 > a").Attr("href")
|
u, _ := s.Find("h1 > a").Attr("href")
|
||||||
ur, _ := url.Parse(u)
|
ur, _ := url.Parse(u)
|
||||||
slug := path.Base(ur.Path)
|
slug := path.Base(ur.Path)
|
||||||
|
@ -117,7 +111,7 @@ func ParseLT(i int, s *goquery.Selection, bar *pb.ProgressBar) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func ParseLTPage(page int, bar *pb.ProgressBar) {
|
func ParseLTPage(page int) {
|
||||||
res, err := GetHTTPResource(config.C.GronkhUrl + "/testet/page/" + strconv.Itoa(page))
|
res, err := GetHTTPResource(config.C.GronkhUrl + "/testet/page/" + strconv.Itoa(page))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Printf("ERR LT Page: Request failed (%+v)", err)
|
log.Printf("ERR LT Page: Request failed (%+v)", err)
|
||||||
|
@ -131,12 +125,12 @@ func ParseLTPage(page int, bar *pb.ProgressBar) {
|
||||||
// Parse the Episodes
|
// Parse the Episodes
|
||||||
doc.Find(".entry.entry-letsplay").Each(func(i int, s *goquery.Selection) {
|
doc.Find(".entry.entry-letsplay").Each(func(i int, s *goquery.Selection) {
|
||||||
wg.Add(1)
|
wg.Add(1)
|
||||||
go ParseLT(i, s, bar)
|
go ParseLT(i, s)
|
||||||
})
|
})
|
||||||
|
|
||||||
_, found := doc.Find(".next").Attr("href")
|
_, found := doc.Find(".next").Attr("href")
|
||||||
if found {
|
if found {
|
||||||
ParseLTPage(page+1, bar)
|
ParseLTPage(page+1)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -145,20 +139,15 @@ func UpdateLetsTests() {
|
||||||
var LTs []database.LetsTest
|
var LTs []database.LetsTest
|
||||||
var count int
|
var count int
|
||||||
database.Db.Find(<s).Count(&count)
|
database.Db.Find(<s).Count(&count)
|
||||||
bar := pb.StartNew(count)
|
|
||||||
for _, LT := range LTs {
|
for _, LT := range LTs {
|
||||||
wg.Add(1)
|
wg.Add(1)
|
||||||
go UpdateLT(LT, bar)
|
go UpdateLT(LT)
|
||||||
}
|
}
|
||||||
wg.Wait()
|
wg.Wait()
|
||||||
bar.FinishPrint("Updated lets tests")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func UpdateLT(LT database.LetsTest, bar *pb.ProgressBar) {
|
func UpdateLT(LT database.LetsTest) {
|
||||||
defer wg.Done()
|
defer wg.Done()
|
||||||
if bar != nil {
|
|
||||||
defer bar.Increment()
|
|
||||||
}
|
|
||||||
var thumb youtube.Thumb
|
var thumb youtube.Thumb
|
||||||
var err error
|
var err error
|
||||||
LT.Rating.Float64, LT.Votes.Int64, _, thumb = youtube.GetRatingAndVotesWithId(LT.Youtube.String, config.C.YoutubeKey)
|
LT.Rating.Float64, LT.Votes.Int64, _, thumb = youtube.GetRatingAndVotesWithId(LT.Youtube.String, config.C.YoutubeKey)
|
||||||
|
|
Loading…
Reference in a new issue