spider/cmd/crawl/main.go

package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"

	"mcquay.me/spider"
)

const usage = "crawl <url>"

func main() {
	if len(os.Args) < 2 {
		fmt.Fprintf(os.Stderr, "%s\n", usage)
		os.Exit(1)
	}

	failures := []spider.Link{}
	for p := range spider.Pages(os.Args[1]) {
		resp, err := http.Get(p.To)
		if err != nil {
			p.Err = err
			failures = append(failures, p)
			continue
		}
		io.Copy(ioutil.Discard, resp.Body)
		resp.Body.Close()
		if resp.StatusCode != http.StatusOK {
			p.Err = fmt.Errorf("http status; got %s, want %s", http.StatusText(resp.StatusCode), http.StatusText(http.StatusOK))
			failures = append(failures, p)
		}
	}

	if len(failures) > 0 {
		for _, f := range failures {
			fmt.Fprintf(os.Stderr, "%+v\n", f)
		}
		os.Exit(1)
	}
}