filedl/main.go

81 lines
1.4 KiB
Go
Raw Permalink Normal View History

2014-12-16 21:35:43 +01:00
package main
import (
"bufio"
"flag"
"fmt"
"io"
"net/http"
"os"
"path"
"strings"
)
var (
input = flag.String("input", "", "Input file containing all the urls")
output = flag.String("output", "/tmp", "Output directory")
worker = flag.Int("worker", 4, "Number of workers downloading the images")
)
func main() {
flag.Parse()
file, err := os.Open(*input)
if err != nil {
fmt.Fprintf(os.Stderr, "Error: %v.\n", err)
os.Exit(1)
}
defer file.Close()
c := make(chan string, *worker)
2014-12-16 22:02:32 +01:00
quit := make(chan bool)
2014-12-16 21:35:43 +01:00
for i := 0; i < *worker; i++ {
2014-12-16 22:02:32 +01:00
go processFileEntry(c, quit)
2014-12-16 21:35:43 +01:00
}
scanner := bufio.NewScanner(file)
for scanner.Scan() {
c <- scanner.Text()
}
close(c)
2014-12-16 22:02:32 +01:00
for i := 0; i < *worker; i++ {
<-quit
}
2014-12-16 21:35:43 +01:00
}
2014-12-16 22:02:32 +01:00
func processFileEntry(c <-chan string, quit chan<- bool) {
2014-12-16 21:35:43 +01:00
for str := range c {
downloadFile(str)
}
2014-12-16 22:02:32 +01:00
quit <- true
2014-12-16 21:35:43 +01:00
}
func downloadFile(url string) {
resp, err := http.Get(url)
if err != nil {
fmt.Fprintf(os.Stderr, "Error GET %v: %v", url, err)
return
}
defer resp.Body.Close()
splits := strings.Split(url, "/")
filename := splits[len(splits)-1]
outFile := path.Join(*output, filename)
out, err := os.Create(outFile)
if err != nil {
fmt.Fprintf(os.Stderr, "Error creating %v: %v", out, err)
return
}
defer out.Close()
_, err = io.Copy(out, resp.Body)
if err != nil {
fmt.Fprintf(os.Stderr, "Error copying %v to %v", filename, outFile)
return
}
fmt.Printf("Successfully downloading %v to %v\n", url, outFile)
}