implement parallel checksumming

This commit is contained in:
Stephen McQuay 2016-11-16 00:52:39 -08:00
parent b11c4701c3
commit 802792d620
No known key found for this signature in database
GPG Key ID: 1ABF428F71BAFC3D
2 changed files with 91 additions and 22 deletions

View File

@ -95,7 +95,7 @@ func check(files []string) chan error {
results := []<-chan error{} results := []<-chan error{}
for w := 0; w < *ngo; w++ { for w := 0; w < *ngo; w++ {
results = append(results, compute(jobs)) results = append(results, verify(jobs))
} }
return merge(results) return merge(results)
@ -125,7 +125,7 @@ func merge(cs []<-chan error) chan error {
return out return out
} }
func compute(jobs chan work) chan error { func verify(jobs chan work) chan error {
r := make(chan error) r := make(chan error)
go func() { go func() {
for job := range jobs { for job := range jobs {

125
main.go
View File

@ -7,9 +7,11 @@ import (
"crypto/sha512" "crypto/sha512"
"flag" "flag"
"fmt" "fmt"
"hash"
"io" "io"
"os" "os"
"runtime" "runtime"
"sync"
) )
var algo = flag.String("a", "sha1", "algorithm to use") var algo = flag.String("a", "sha1", "algorithm to use")
@ -30,51 +32,118 @@ func main() {
os.Exit(1) os.Exit(1)
} }
case false: case false:
if err := hsh(files); err != nil { c := 0
fmt.Fprintf(os.Stderr, "%v\n", err) for res := range hsh(files) {
if res.err != nil {
c++
fmt.Fprintf(os.Stderr, "%v\n", res.err)
} else {
fmt.Printf("%v\n", res.msg)
}
}
if c > 0 {
os.Exit(1) os.Exit(1)
} }
} }
} }
func hsh(files []string) error { type hashr func() hash.Hash
h := sha256.New()
func hsh(files []string) chan result {
var h hashr
switch *algo { switch *algo {
case "sha1", "1": case "sha1", "1":
h = sha1.New() h = sha1.New
case "sha256", "256": case "sha256", "256":
h = sha256.New() h = sha256.New
case "sha512", "512": case "sha512", "512":
h = sha512.New() h = sha512.New
case "md5": case "md5":
h = md5.New() h = md5.New
default: default:
return fmt.Errorf("unsupported algorithm: %v", *algo) r := make(chan result)
go func() {
r <- result{err: fmt.Errorf("unsupported algorithm: %v", *algo)}
}()
return r
} }
if len(files) == 0 { if len(files) == 0 {
_, err := io.Copy(h, os.Stdin) hsh := h()
_, err := io.Copy(hsh, os.Stdin)
if err != nil { if err != nil {
fmt.Fprintf(os.Stderr, "%v\n", err) fmt.Fprintf(os.Stderr, "%v\n", err)
os.Exit(1) os.Exit(1)
} }
fmt.Printf("%x -\n", h.Sum(nil)) fmt.Printf("%x -\n", hsh.Sum(nil))
} else {
for _, name := range files {
f, err := os.Open(name)
if err != nil {
fmt.Fprintf(os.Stderr, "%v\n", err)
continue
}
h.Reset()
_, err = io.Copy(h, f)
f.Close()
if err != nil {
fmt.Fprintf(os.Stderr, "%v\n", err)
continue
}
fmt.Printf("%x %s\n", h.Sum(nil), name)
}
}
return nil return nil
} }
jobs := make(chan work)
go func() {
for _, name := range files {
jobs <- work{cs: checksum{filename: name}}
}
close(jobs)
}()
res := []<-chan result{}
for w := 0; w < *ngo; w++ {
res = append(res, compute(h, jobs))
}
return rmerge(res)
}
type result struct {
msg string
err error
}
func compute(h hashr, jobs chan work) chan result {
hsh := h()
r := make(chan result)
go func() {
for job := range jobs {
f, err := os.Open(job.cs.filename)
if err != nil {
r <- result{err: err}
continue
}
hsh.Reset()
_, err = io.Copy(hsh, f)
f.Close()
if err != nil {
r <- result{err: err}
continue
}
r <- result{msg: fmt.Sprintf("%x %s", hsh.Sum(nil), job.cs.filename)}
}
close(r)
}()
return r
}
func rmerge(cs []<-chan result) chan result {
out := make(chan result)
var wg sync.WaitGroup
output := func(c <-chan result) {
for n := range c {
out <- n
}
wg.Done()
}
wg.Add(len(cs))
for _, c := range cs {
go output(c)
}
go func() {
wg.Wait()
close(out)
}()
return out
}