From 5a19118daeefbae1f9712f6e7ef8abed3362ecf6 Mon Sep 17 00:00:00 2001
From: Fabien Freling <public@ffreling.com>
Date: Tue, 16 Dec 2014 21:35:43 +0100
Subject: [PATCH] Add main to download files.

---
 .gitignore |  2 ++
 main.go    | 78 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 80 insertions(+)
 create mode 100644 main.go

diff --git a/.gitignore b/.gitignore
index daf913b..dde85ad 100644
--- a/.gitignore
+++ b/.gitignore
@@ -22,3 +22,5 @@ _testmain.go
 *.exe
 *.test
 *.prof
+
+*.swp
diff --git a/main.go b/main.go
new file mode 100644
index 0000000..6e99f82
--- /dev/null
+++ b/main.go
@@ -0,0 +1,78 @@
+package main
+
+import (
+	"bufio"
+	"flag"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"path"
+	"strings"
+	"time"
+)
+
+var (
+	input  = flag.String("input", "", "Input file containing all the urls")
+	output = flag.String("output", "/tmp", "Output directory")
+	worker = flag.Int("worker", 4, "Number of workers downloading the images")
+)
+
+func main() {
+	flag.Parse()
+
+	file, err := os.Open(*input)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %v.\n", err)
+		os.Exit(1)
+	}
+	defer file.Close()
+
+	c := make(chan string, *worker)
+	for i := 0; i < *worker; i++ {
+		go processFileEntry(c)
+	}
+
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		c <- scanner.Text()
+	}
+	close(c)
+
+	// Let's wait 10 sec for the last images to download.
+	time.Sleep(10 * 1e9)
+}
+
+func processFileEntry(c <-chan string) {
+	for str := range c {
+		downloadFile(str)
+	}
+}
+
+func downloadFile(url string) {
+	resp, err := http.Get(url)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error GET %v: %v", url, err)
+		return
+	}
+	defer resp.Body.Close()
+
+	splits := strings.Split(url, "/")
+	filename := splits[len(splits)-1]
+	outFile := path.Join(*output, filename)
+
+	out, err := os.Create(outFile)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error creating %v: %v", out, err)
+		return
+	}
+	defer out.Close()
+
+	_, err = io.Copy(out, resp.Body)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error copying %v to %v", filename, outFile)
+		return
+	}
+
+	fmt.Printf("Successfully downloading %v to %v\n", url, outFile)
+}