sm/spider
1
0
Fork 0
Code Issues 1 Pull Requests Activity

simplify "stay on domain" logic

Browse Source
This commit is contained in:
Stephen McQuay 2016-11-24 09:09:20 -08:00
parent eea0aee587
commit d80caeb6bb
No known key found for this signature in database
GPG Key ID: 1ABF428F71BAFC3D
1 changed files with 4 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
spider.go
View File

@ -62,12 +62,12 @@ func Pages(base string) <-chan Link {
for len(links) > 0 { for len(links) > 0 {
l := links[0] l := links[0]
links = links[1:] links = links[1:]
if !(strings.HasPrefix(l.To, base) || strings.HasPrefix(l.To, "/")) { if strings.HasPrefix(l.To, "/") {
continue
}
if !strings.HasPrefix(l.To, base) && strings.HasPrefix(l.To, "/") {
l.To = base + l.To l.To = base + l.To
} }
if !strings.HasPrefix(l.To, base) {
continue
}
// drop fragments // drop fragments
h := strings.Index(l.To, "#") h := strings.Index(l.To, "#")