Fixing too many open file descriptors (closing sockets after using them)
This commit is contained in:
parent
b8e0e74f73
commit
f52708e7d3
|
@ -37,8 +37,8 @@ func ParseRSSFeeds() {
|
|||
log.Printf("ERR RS %s: Request failed (%+v)", AT.Slug, err)
|
||||
continue
|
||||
}
|
||||
defer res.Body.Close()
|
||||
body, err := ioutil.ReadAll(res.Body)
|
||||
res.Body.Close()
|
||||
if err != nil {
|
||||
log.Fatalf("ERR RS %s: Document failure (%+v)", AT.Slug, err)
|
||||
continue
|
||||
|
@ -64,6 +64,7 @@ func ParseRSSFeeds() {
|
|||
}
|
||||
wg.Wait()
|
||||
}
|
||||
log.Printf("*** FINISHED PARSING RSS FEEDS ***")
|
||||
GenerateSitemap()
|
||||
}
|
||||
|
||||
|
|
|
@ -8,12 +8,13 @@ import (
|
|||
)
|
||||
|
||||
var sem = make(chan byte, 5)
|
||||
var client = &http.Client{}
|
||||
|
||||
|
||||
// Make an HTTP Get Request to u
|
||||
func GetHTTPResource(u string) (*http.Response, error) {
|
||||
sem <- 1
|
||||
// Prepare HTTP Client, Cookie and Request
|
||||
client := &http.Client {}
|
||||
cookie := &http.Cookie {
|
||||
Name : "POPUPCHECK",
|
||||
Value : strconv.FormatInt(time.Now().UnixNano() / 1000000, 10),
|
||||
|
@ -26,8 +27,8 @@ func GetHTTPResource(u string) (*http.Response, error) {
|
|||
req.Header.Add("Cache-Control", "max-age=0")
|
||||
req.Header.Add("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_4) AppleWebKit/600.7.12 (KHTML, like Gecko) Version/8.0.7 Safari/600.7.12")
|
||||
if err != nil {
|
||||
<- sem
|
||||
log.Fatalf("FAT HTTP - Failed to create new Request: %+v", err)
|
||||
<- sem
|
||||
return nil, err
|
||||
}
|
||||
// Execute HTTP Request
|
||||
|
@ -37,6 +38,7 @@ func GetHTTPResource(u string) (*http.Response, error) {
|
|||
return nil, err
|
||||
}
|
||||
if res.StatusCode == http.StatusNotFound {
|
||||
res.Body.Close()
|
||||
<- sem
|
||||
return nil, nil
|
||||
}
|
||||
|
|
|
@ -8,10 +8,13 @@ import (
|
|||
"errors"
|
||||
)
|
||||
|
||||
var sem = make(chan byte, 5)
|
||||
var client = &http.Client {}
|
||||
|
||||
// Make an HTTP Get Request to u
|
||||
func GetHTTPResource(u string) (*http.Response, error) {
|
||||
sem <- 1
|
||||
// Prepare HTTP Client, Cookie and Request
|
||||
client := &http.Client {}
|
||||
cookie := &http.Cookie {
|
||||
Name : "POPUPCHECK",
|
||||
Value : strconv.FormatInt(time.Now().UnixNano() / 1000000, 10),
|
||||
|
@ -25,15 +28,20 @@ func GetHTTPResource(u string) (*http.Response, error) {
|
|||
req.Header.Add("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_4) AppleWebKit/600.7.12 (KHTML, like Gecko) Version/8.0.7 Safari/600.7.12")
|
||||
if err != nil {
|
||||
log.Fatalf("FAT HTTP - Failed to create new Request: %+v", err)
|
||||
<- sem
|
||||
return nil, err
|
||||
}
|
||||
// Execute HTTP Request
|
||||
res, err := client.Do(req)
|
||||
if err != nil {
|
||||
<- sem
|
||||
return nil, err
|
||||
}
|
||||
if res.StatusCode != 200 {
|
||||
res.Body.Close()
|
||||
<- sem
|
||||
return nil, errors.New(strconv.Itoa(res.StatusCode))
|
||||
}
|
||||
<- sem
|
||||
return res, nil
|
||||
}
|
||||
|
|
|
@ -148,12 +148,13 @@ func ResizeImage(url string, out []string, size []string) (error) {
|
|||
if erro != nil {
|
||||
return erro
|
||||
}
|
||||
defer f.Close()
|
||||
tfile := f.Name()
|
||||
defer os.Remove(tfile)
|
||||
_, err = f.Write(blob)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
f.Close()
|
||||
for i := 0; i < len(out); i++ {
|
||||
cmd := exec.Command("convert", tfile, "-resize", size[i] + "^", "-quality", "70", "-gravity", "center", "-crop", size[i] + "+0+0", "+repage", out[i])
|
||||
err = cmd.Run()
|
||||
|
@ -161,6 +162,5 @@ func ResizeImage(url string, out []string, size []string) (error) {
|
|||
return err
|
||||
}
|
||||
}
|
||||
os.Remove(tfile)
|
||||
return nil
|
||||
}
|
||||
|
|
|
@ -63,21 +63,28 @@ type YTResult struct {
|
|||
|
||||
var videoEndpoint string = "https://www.googleapis.com/youtube/v3/videos"
|
||||
var channelEndpoint string = "https://www.googleapis.com/youtube/v3/channels"
|
||||
var sem = make(chan byte, 10)
|
||||
|
||||
var sem = make(chan byte, 5)
|
||||
var client = &http.Client {}
|
||||
|
||||
// Make an HTTP Get Request to u
|
||||
func GetHTTPResource(u string) (*http.Response, error) {
|
||||
sem <- 1
|
||||
// Prepare HTTP Client, Cookie and Request
|
||||
res, err := http.Get(u)
|
||||
req, err := http.NewRequest("GET", u, nil)
|
||||
if err != nil {
|
||||
log.Printf("[WARN] HTTP - Request to %s failed: %s", u, err)
|
||||
log.Fatalf("FAT HTTP - Failed to create new Request: %+v", err)
|
||||
<- sem
|
||||
return nil, err
|
||||
}
|
||||
if res.StatusCode == http.StatusNotFound {
|
||||
log.Printf("[WARN] HTTP - Received 404 Not Found for: %s", u)
|
||||
// Execute HTTP Request
|
||||
res, err := client.Do(req)
|
||||
if err != nil {
|
||||
<- sem
|
||||
return nil, err
|
||||
}
|
||||
if res.StatusCode == 404 {
|
||||
res.Body.Close()
|
||||
<- sem
|
||||
return nil, nil
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue