79 lines
1.4 KiB
Go
79 lines
1.4 KiB
Go
|
package main
|
||
|
|
|
||
|
|
import (
|
||
|
|
"bufio"
|
||
|
|
"flag"
|
||
|
|
"fmt"
|
||
|
|
"io"
|
||
|
|
"net/http"
|
||
|
|
"os"
|
||
|
|
"path"
|
||
|
|
"strings"
|
||
|
|
"time"
|
||
|
|
)
|
||
|
|
|
||
|
|
var (
|
||
|
|
input = flag.String("input", "", "Input file containing all the urls")
|
||
|
|
output = flag.String("output", "/tmp", "Output directory")
|
||
|
|
worker = flag.Int("worker", 4, "Number of workers downloading the images")
|
||
|
|
)
|
||
|
|
|
||
|
|
func main() {
|
||
|
|
flag.Parse()
|
||
|
|
|
||
|
|
file, err := os.Open(*input)
|
||
|
|
if err != nil {
|
||
|
|
fmt.Fprintf(os.Stderr, "Error: %v.\n", err)
|
||
|
|
os.Exit(1)
|
||
|
|
}
|
||
|
|
defer file.Close()
|
||
|
|
|
||
|
|
c := make(chan string, *worker)
|
||
|
|
for i := 0; i < *worker; i++ {
|
||
|
|
go processFileEntry(c)
|
||
|
|
}
|
||
|
|
|
||
|
|
scanner := bufio.NewScanner(file)
|
||
|
|
for scanner.Scan() {
|
||
|
|
c <- scanner.Text()
|
||
|
|
}
|
||
|
|
close(c)
|
||
|
|
|
||
|
|
// Let's wait 10 sec for the last images to download.
|
||
|
|
time.Sleep(10 * 1e9)
|
||
|
|
}
|
||
|
|
|
||
|
|
func processFileEntry(c <-chan string) {
|
||
|
|
for str := range c {
|
||
|
|
downloadFile(str)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func downloadFile(url string) {
|
||
|
|
resp, err := http.Get(url)
|
||
|
|
if err != nil {
|
||
|
|
fmt.Fprintf(os.Stderr, "Error GET %v: %v", url, err)
|
||
|
|
return
|
||
|
|
}
|
||
|
|
defer resp.Body.Close()
|
||
|
|
|
||
|
|
splits := strings.Split(url, "/")
|
||
|
|
filename := splits[len(splits)-1]
|
||
|
|
outFile := path.Join(*output, filename)
|
||
|
|
|
||
|
|
out, err := os.Create(outFile)
|
||
|
|
if err != nil {
|
||
|
|
fmt.Fprintf(os.Stderr, "Error creating %v: %v", out, err)
|
||
|
|
return
|
||
|
|
}
|
||
|
|
defer out.Close()
|
||
|
|
|
||
|
|
_, err = io.Copy(out, resp.Body)
|
||
|
|
if err != nil {
|
||
|
|
fmt.Fprintf(os.Stderr, "Error copying %v to %v", filename, outFile)
|
||
|
|
return
|
||
|
|
}
|
||
|
|
|
||
|
|
fmt.Printf("Successfully downloading %v to %v\n", url, outFile)
|
||
|
|
}
|