filedl/main.go

package main

import (
	"bufio"
	"flag"
	"fmt"
	"io"
	"net/http"
	"os"
	"path"
	"strings"
)

var (
	input  = flag.String("input", "", "Input file containing all the urls")
	output = flag.String("output", "/tmp", "Output directory")
	worker = flag.Int("worker", 4, "Number of workers downloading the images")
)

func main() {
	flag.Parse()

	file, err := os.Open(*input)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error: %v.\n", err)
		os.Exit(1)
	}
	defer file.Close()

	c := make(chan string, *worker)
	quit := make(chan bool)
	for i := 0; i < *worker; i++ {
		go processFileEntry(c, quit)
	}

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		c <- scanner.Text()
	}
	close(c)

	for i := 0; i < *worker; i++ {
		<-quit
	}
}

func processFileEntry(c <-chan string, quit chan<- bool) {
	for str := range c {
		downloadFile(str)
	}
	quit <- true
}

func downloadFile(url string) {
	resp, err := http.Get(url)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error GET %v: %v", url, err)
		return
	}
	defer resp.Body.Close()

	splits := strings.Split(url, "/")
	filename := splits[len(splits)-1]
	outFile := path.Join(*output, filename)

	out, err := os.Create(outFile)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error creating %v: %v", out, err)
		return
	}
	defer out.Close()

	_, err = io.Copy(out, resp.Body)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error copying %v to %v", filename, outFile)
		return
	}

	fmt.Printf("Successfully downloading %v to %v\n", url, outFile)
}