Timeout in goroutines and Http requests

Question

I am checking the status of a server. The server has a sleep of more than 15 seconds and I am checking for the timeout.

package main

import (
    "fmt"
    "net/http"
    "time"
)

var urls = []string{
    "http://site-centos-64:8080/examples/abc1.jsp",
    }

type HttpResponse struct {
    url      string
    response *http.Response
    err      error
}
var ch = make(chan *HttpResponse, 100) // buffered
var count int   
func asyncHttpGets(urls []string) []*HttpResponse {
    responses := []*HttpResponse{}
    count:=0
    timeout := make(chan bool, 100)
    for i:=0;i<500;i++{
        go func(){
                for _, url := range urls {
                    resp, err := http.Get(url)
                    count++;
                    go func() {
                    time.Sleep(1 * time.Second)
                    timeout <- true
                    }() 
                    ch <- &HttpResponse{url, resp, err}
                    if err != nil {
                        return
                    }
                    resp.Body.Close()
                }
        }()
    }
    for {
        select {
        case r := <-ch:
            responses = append(responses, r)
            if count == 500 {
                return responses
            }
        case <-timeout:
                fmt.Println("Timed Out")
                if count == 500 {
                return responses
            }
        }
    }
    return responses

}

func main() {
    now:=time.Now()
    results := asyncHttpGets(urls)
    for _, result := range results {
        fmt.Printf("%s status: %s\n", result.url,result.response.Status)
    }
    fmt.Println( time.Since(now))
}

But what is happening is that initially it prints a "Timed Out" but the last 150-200 requests show the "200 OK" status which it should not. Also when trying to do it for a 1000 times it shows "panic: runtime error: invalid memory address or nil pointer dereference"

Answer 1

You are doing the resp, err := http.Get(url) before you initiate the timeout goroutine. This will cause everything to block until the response is ready, then send on both channels simultaneously.

Just move starting the timeout goroutine to the line before sending the request, and it will be fine. i.e.:

  for _, url := range urls {


                go func() {
                   time.Sleep(1 * time.Second)
                   timeout <- true
                   count++;
                }() 

                resp, err := http.Get(url)
                count++; //I think this is what you meant, right?
                ch <- &HttpResponse{url, resp, err}
                if err != nil {
                    return
                }
                resp.Body.Close()
            }

BTW try to use atomic increments to count, and maybe use a waitgroup, and a time.After channel instead of sleep.

Answer 2

If you would like to avoid mixing concurrency logic with business logic, I wrote this library https://github.com/shomali11/parallelizer to help you with that. It encapsulates the concurrency logic so you do not have to worry about it.

So in your example:

package main

import (
    "github.com/shomali11/parallelizer"
    "fmt"
)

func main() {
    urls := []string{ ... }
    results = make([]*HttpResponse, len(urls)

    options := &Options{ Timeout: time.Second }
    group := parallelizer.NewGroup(options)
    for index, url := range urls {
        group.Add(func(index int, url string, results *[]*HttpResponse) {
            return func () {
                ...

                results[index] = &HttpResponse{url, response, err}
            }
        }(index, url, &results))
    }

    err := group.Run()

    fmt.Println("Done")
    fmt.Println(fmt.Sprintf("Results: %v", results))
    fmt.Printf("Error: %v", err) // nil if it completed, err if timed out
}