diff --git a/.gitignore b/.gitignore index daf913b..dde85ad 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,5 @@ _testmain.go *.exe *.test *.prof + +*.swp diff --git a/main.go b/main.go new file mode 100644 index 0000000..6e99f82 --- /dev/null +++ b/main.go @@ -0,0 +1,78 @@ +package main + +import ( + "bufio" + "flag" + "fmt" + "io" + "net/http" + "os" + "path" + "strings" + "time" +) + +var ( + input = flag.String("input", "", "Input file containing all the urls") + output = flag.String("output", "/tmp", "Output directory") + worker = flag.Int("worker", 4, "Number of workers downloading the images") +) + +func main() { + flag.Parse() + + file, err := os.Open(*input) + if err != nil { + fmt.Fprintf(os.Stderr, "Error: %v.\n", err) + os.Exit(1) + } + defer file.Close() + + c := make(chan string, *worker) + for i := 0; i < *worker; i++ { + go processFileEntry(c) + } + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + c <- scanner.Text() + } + close(c) + + // Let's wait 10 sec for the last images to download. + time.Sleep(10 * 1e9) +} + +func processFileEntry(c <-chan string) { + for str := range c { + downloadFile(str) + } +} + +func downloadFile(url string) { + resp, err := http.Get(url) + if err != nil { + fmt.Fprintf(os.Stderr, "Error GET %v: %v", url, err) + return + } + defer resp.Body.Close() + + splits := strings.Split(url, "/") + filename := splits[len(splits)-1] + outFile := path.Join(*output, filename) + + out, err := os.Create(outFile) + if err != nil { + fmt.Fprintf(os.Stderr, "Error creating %v: %v", out, err) + return + } + defer out.Close() + + _, err = io.Copy(out, resp.Body) + if err != nil { + fmt.Fprintf(os.Stderr, "Error copying %v to %v", filename, outFile) + return + } + + fmt.Printf("Successfully downloading %v to %v\n", url, outFile) +}