37 lines
550 B
Go
37 lines
550 B
Go
|
package main
|
||
|
|
|
||
|
|
import (
|
||
|
|
"fmt"
|
||
|
|
"net/http"
|
||
|
|
"os"
|
||
|
|
|
||
|
|
"mcquay.me/spider"
|
||
|
|
)
|
||
|
|
|
||
|
|
const usage = "crawl <url>"
|
||
|
|
|
||
|
|
func main() {
|
||
|
|
if len(os.Args) < 2 {
|
||
|
|
fmt.Fprintf(os.Stderr, "%s\n", usage)
|
||
|
|
os.Exit(1)
|
||
|
|
}
|
||
|
|
|
||
|
|
failures := []spider.Link{}
|
||
|
|
for p := range spider.Pages(os.Args[1]) {
|
||
|
|
resp, err := http.Get(p.To)
|
||
|
|
if err != nil {
|
||
|
|
failures = append(failures, p)
|
||
|
|
}
|
||
|
|
if resp.StatusCode != http.StatusOK {
|
||
|
|
failures = append(failures, p)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
if len(failures) > 0 {
|
||
|
|
for _, f := range failures {
|
||
|
|
fmt.Fprintf(os.Stderr, "%+v\n", f)
|
||
|
|
}
|
||
|
|
os.Exit(1)
|
||
|
|
}
|
||
|
|
}
|