init

LICENSE

@@ -0,0 +1,27 @@
+Copyright (c) 2018, Stephen McQuay (smcquay)
+
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+ * Redistributions of source code must retain the above copyright notice,
+   this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+ * Neither the name of pid nor the names of its contributors
+   may be used to endorse or promote products derived from this software
+   without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

README.md

@@ -0,0 +1,6 @@
+# bigfiles
+
+Answer to an interview question:
+
+Find the largest N files hosted at `hmm.clstr.co/size/` named `000` through
+`999`, inclusive, then make it go as fast as you can.

main.go

@@ -0,0 +1,105 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"runtime"
+	"sort"
+	"sync"
+)
+
+const max = 1000
+
+type pair struct {
+	u string
+	s int64
+}
+
+type pairs []pair
+
+func (p pairs) Len() int           { return len(p) }
+func (p pairs) Swap(a, b int)      { p[a], p[b] = p[b], p[a] }
+func (p pairs) Less(a, b int) bool { return p[a].s < p[b].s }
+
+var workers = flag.Int("workers", runtime.NumCPU()*4, "number of downloaders")
+
+func main() {
+	flag.Parse()
+	for r := range srt(attack(urls(max), *workers)) {
+		fmt.Printf("%v\t%v\n", r.s, r.u)
+	}
+}
+
+func urls(max int) chan string {
+	r := make(chan string)
+
+	go func() {
+		for i := 0; i < max; i++ {
+			r <- fmt.Sprintf("https://hmm.clstr.co/size/%03d", i)
+		}
+		close(r)
+	}()
+
+	return r
+}
+
+func attack(urls chan string, workers int) chan pair {
+	r := make(chan pair)
+	go func() {
+		c := http.Client{}
+		s := make(chan bool, workers)
+		wg := sync.WaitGroup{}
+		for u := range urls {
+			wg.Add(1)
+			s <- true
+			go func(url string) {
+				resp, err := c.Get(url)
+				if err != nil {
+					log.Printf("probalo: %v", err)
+					return
+				}
+				r <- pair{url, resp.ContentLength}
+
+				if n, err := io.Copy(ioutil.Discard, resp.Body); err != nil {
+					log.Printf("after %v bytes: %v", n, err)
+				}
+				if err := resp.Body.Close(); err != nil {
+					log.Printf("close body: %v", err)
+				}
+				<-s
+				wg.Done()
+			}(u)
+		}
+		wg.Wait()
+		close(r)
+	}()
+	return r
+}
+
+func srt(in chan pair) chan pair {
+	r := make(chan pair)
+	go func() {
+		ps := pairs{}
+		for r := range in {
+			ps = append(ps, r)
+		}
+
+		sort.Sort(ps)
+
+		for _, p := range ps {
+			r <- p
+		}
+		close(r)
+	}()
+	return r
+}
+
+func print() {
+	for i := 0; i < 1000; i++ {
+		fmt.Sprintf("hmm.clstr.co/size/%03d", i)
+	}
+}