Time response for HTTP GET request when using goroutines

Question

I have a simple code that prints GET response time for each URL listed in a text file (url_list.txt).

When the requests are fired sequentially the returned times correspond to the expected response times of individual URLs.

However, when the same code is executed concurrently the returned response times are typically higher than expected.

It seems that the time_start I capture before the http.Get(url) is called is not the time of when the request is actually sent. I guess the execution of http.Get(url) is queued to some extend.

Is there a better way to capture URL response time when using goroutines?

Here is my code:

Sequential requests:

package main

import ("fmt"
        "net/http"
        "io/ioutil"
        "time"
        "strings"
)

func get_resp_time(url string) {
        time_start := time.Now()
        resp, err := http.Get(url)
        if err != nil {
            panic(err)
        }
        defer resp.Body.Close()
        fmt.Println(time.Since(time_start), url)
}

func main() {
    content, _ := ioutil.ReadFile("url_list.txt")
    urls := strings.Split(string(content), "\n")

    for _, url := range urls {
        get_resp_time(url)
        //go get_resp_time(url)
    }

    //time.Sleep(20 * time.Second)
}

Concurrent requests:

package main

import ("fmt"
        "net/http"
        "io/ioutil"
        "time"
        "strings"
)

func get_resp_time(url string) {
        time_start := time.Now()
        resp, err := http.Get(url)
        if err != nil {
            panic(err)
        }
        defer resp.Body.Close()
        fmt.Println(time.Since(time_start), url)
}

func main() {
    content, _ := ioutil.ReadFile("url_list.txt")
    urls := strings.Split(string(content), "\n")

    for _, url := range urls {
        //get_resp_time(url)
        go get_resp_time(url)
    }

    time.Sleep(20 * time.Second)
} 

Answer 1

You are starting all the requests at once. If there are 1000s of urls in the file then you are starting 1000s of go routines all at once. This may work, but may give you errors about being out of sockets or file handles. I'd recommend starting a limited number of fetches at once, like this code below.

This should help with the timing also.

package main

import (
    "fmt"
    "io/ioutil"
    "log"
    "net/http"
    "strings"
    "sync"
    "time"
)

func get_resp_time(url string) {
    time_start := time.Now()
    resp, err := http.Get(url)
    if err != nil {
        log.Printf("Error fetching: %v", err)
    }
    defer resp.Body.Close()
    fmt.Println(time.Since(time_start), url)
}

func main() {
    content, _ := ioutil.ReadFile("url_list.txt")
    urls := strings.Split(string(content), "\n")

    const workers = 25

    wg := new(sync.WaitGroup)
    in := make(chan string, 2*workers)

    for i := 0; i < workers; i++ {
        wg.Add(1)
        go func() {
            defer wg.Done()
            for url := range in {
                get_resp_time(url)
            }
        }()
    }

    for _, url := range urls {
        if url != "" {
            in <- url
        }
    }
    close(in)
    wg.Wait()
}