sm
/
spider
1
0
Fork 0
Code Issues 1 Pull Requests Activity

simplify "stay on domain" logic

This commit is contained in:
Stephen McQuay 2016-11-24 09:09:20 -08:00
parent eea0aee587
commit d80caeb6bb
No known key found for this signature in database
GPG Key ID: 1ABF428F71BAFC3D
1 changed files with 4 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
spider.go
View File

@ -62,12 +62,12 @@ func Pages(base string) <-chan Link {
for len(links) > 0 {
l := links[0]
links = links[1:]
if !(strings.HasPrefix(l.To, base) || strings.HasPrefix(l.To, "/")) {
continue
}
if !strings.HasPrefix(l.To, base) && strings.HasPrefix(l.To, "/") {
if strings.HasPrefix(l.To, "/") {
l.To = base + l.To
}
if !strings.HasPrefix(l.To, base) {
continue
}
// drop fragments
h := strings.Index(l.To, "#")