package main import ( "bufio" "flag" "fmt" "io" "net/http" "os" "path" "strings" ) var ( input = flag.String("input", "", "Input file containing all the urls") output = flag.String("output", "/tmp", "Output directory") worker = flag.Int("worker", 4, "Number of workers downloading the images") ) func main() { flag.Parse() file, err := os.Open(*input) if err != nil { fmt.Fprintf(os.Stderr, "Error: %v.\n", err) os.Exit(1) } defer file.Close() c := make(chan string, *worker) quit := make(chan bool) for i := 0; i < *worker; i++ { go processFileEntry(c, quit) } scanner := bufio.NewScanner(file) for scanner.Scan() { c <- scanner.Text() } close(c) for i := 0; i < *worker; i++ { <-quit } } func processFileEntry(c <-chan string, quit chan<- bool) { for str := range c { downloadFile(str) } quit <- true } func downloadFile(url string) { resp, err := http.Get(url) if err != nil { fmt.Fprintf(os.Stderr, "Error GET %v: %v", url, err) return } defer resp.Body.Close() splits := strings.Split(url, "/") filename := splits[len(splits)-1] outFile := path.Join(*output, filename) out, err := os.Create(outFile) if err != nil { fmt.Fprintf(os.Stderr, "Error creating %v: %v", out, err) return } defer out.Close() _, err = io.Copy(out, resp.Body) if err != nil { fmt.Fprintf(os.Stderr, "Error copying %v to %v", filename, outFile) return } fmt.Printf("Successfully downloading %v to %v\n", url, outFile) }