44 lines
773 B
Go
44 lines
773 B
Go
package main
|
|
|
|
import (
|
|
"fmt"
|
|
"io"
|
|
"io/ioutil"
|
|
"net/http"
|
|
"os"
|
|
|
|
"mcquay.me/spider"
|
|
)
|
|
|
|
const usage = "crawl <url>"
|
|
|
|
func main() {
|
|
if len(os.Args) < 2 {
|
|
fmt.Fprintf(os.Stderr, "%s\n", usage)
|
|
os.Exit(1)
|
|
}
|
|
|
|
failures := []spider.Link{}
|
|
for p := range spider.Pages(os.Args[1]) {
|
|
resp, err := http.Get(p.To)
|
|
if err != nil {
|
|
p.Err = err
|
|
failures = append(failures, p)
|
|
continue
|
|
}
|
|
io.Copy(ioutil.Discard, resp.Body)
|
|
resp.Body.Close()
|
|
if resp.StatusCode != http.StatusOK {
|
|
p.Err = fmt.Errorf("http status; got %s, want %s", http.StatusText(resp.StatusCode), http.StatusText(http.StatusOK))
|
|
failures = append(failures, p)
|
|
}
|
|
}
|
|
|
|
if len(failures) > 0 {
|
|
for _, f := range failures {
|
|
fmt.Fprintf(os.Stderr, "%+v\n", f)
|
|
}
|
|
os.Exit(1)
|
|
}
|
|
}
|