check.go

@@ -15,78 +15,18 @@ import (
 	"sync"
 )
 
-// input contains a file-ish piece of work to perform
-type input struct {
-	f   io.ReadCloser
-	err error
-}
-
-// checksum contains the path to a file, a way to hash it, and the results of
-// the hash
 type checksum struct {
 	filename string
 	hash     hash.Hash
 	checksum string
-	err      error
 }
 
-// check is the entry point for -c operation.
+func parseCS(line string) (checksum, error) {
-func check(args []string, verbose bool) chan error {
-	jobs := make(chan checksum)
-
-	go func() {
-		for i := range toInput(args) {
-			if i.err != nil {
-				jobs <- checksum{err: i.err}
-				break
-			}
-			s := bufio.NewScanner(i.f)
-			for s.Scan() {
-				jobs <- parseCS(s.Text())
-			}
-			i.f.Close()
-			if s.Err() != nil {
-				jobs <- checksum{err: s.Err()}
-			}
-		}
-		close(jobs)
-	}()
-
-	results := []<-chan error{}
-
-	for w := 0; w < *ngo; w++ {
-		results = append(results, verify(jobs, verbose))
-	}
-
-	return merge(results)
-}
-
-// toInput converts args to a stream of input
-func toInput(args []string) chan input {
-	r := make(chan input)
-
-	go func() {
-		for _, name := range args {
-			f, err := os.Open(name)
-			r <- input{f, err}
-		}
-		if len(args) == 0 {
-			r <- input{f: os.Stdin}
-		}
-		close(r)
-	}()
-
-	return r
-}
-
-// parseCS picks apart a line from a checksum file and returns everything
-// needed to perform a checksum.
-func parseCS(line string) checksum {
 	elems := strings.Fields(line)
-	if len(elems) < 1 {
+	if len(elems) != 2 {
-		return checksum{err: fmt.Errorf("couldn't find checksum in %q", line)}
+		return checksum{}, fmt.Errorf("unexpected content: %d != 2", len(elems))
 	}
-	cs := elems[0]
+	cs, f := elems[0], elems[1]
 	var hsh hash.Hash
 	switch len(cs) {
 	case 32:
@@ -98,43 +38,70 @@ func parseCS(line string) checksum {
 	case 128:
 		hsh = sha512.New()
 	default:
-		return checksum{err: fmt.Errorf("unknown format: %q", line)}
+		return checksum{}, fmt.Errorf("unknown format: %q", line)
+	}
+	return checksum{filename: f, hash: hsh, checksum: cs}, nil
 }
 
-	return checksum{filename: strings.TrimSpace(line[len(cs):]), hash: hsh, checksum: cs}
+type input struct {
+	f   io.ReadCloser
+	err error
 }
 
-// verify does grunt work of verifying a stream of jobs (filenames).
+type work struct {
-func verify(jobs chan checksum, verbose bool) chan error {
+	cs  checksum
-	r := make(chan error)
+	err error
+}
+
+func streams(files []string) chan input {
+	r := make(chan input)
+
 	go func() {
-		for job := range jobs {
+		for _, name := range files {
-			if job.err != nil {
+			f, err := os.Open(name)
-				log.Printf("%+v", job.err)
+			r <- input{f, err}
-				continue
-			}
-			f, err := os.Open(job.filename)
-			if err != nil {
-				r <- err
-				continue
-			}
-			if _, err := io.Copy(job.hash, f); err != nil {
-				r <- err
-				continue
-			}
-			f.Close()
-			if fmt.Sprintf("%x", job.hash.Sum(nil)) != job.checksum {
-				r <- fmt.Errorf("%s: bad", job.filename)
-			} else if verbose {
-				fmt.Fprintf(os.Stderr, "ok: %v\n", job.filename)
 		}
+		if len(files) == 0 {
+			r <- input{f: os.Stdin}
 		}
 		close(r)
 	}()
+
 	return r
 }
 
-// merge is simple error fan-in
+func check(files []string) chan error {
+	jobs := make(chan work)
+
+	go func() {
+		for stream := range streams(files) {
+			if stream.err != nil {
+				jobs <- work{err: stream.err}
+				break
+			}
+			s := bufio.NewScanner(stream.f)
+			for s.Scan() {
+				cs, err := parseCS(s.Text())
+				jobs <- work{cs, err}
+			}
+			stream.f.Close()
+			if s.Err() != nil {
+				jobs <- work{err: s.Err()}
+			}
+		}
+		close(jobs)
+	}()
+
+	results := []<-chan error{}
+
+	workers := 32
+	for w := 0; w < workers; w++ {
+		results = append(results, compute(jobs))
+	}
+
+	return merge(results)
+}
+
 func merge(cs []<-chan error) chan error {
 	out := make(chan error)
 
@@ -158,3 +125,30 @@ func merge(cs []<-chan error) chan error {
 	}()
 	return out
 }
+
+func compute(jobs chan work) chan error {
+	r := make(chan error)
+	go func() {
+		for job := range jobs {
+			if job.err != nil {
+				log.Printf("%+v", job.err)
+				continue
+			}
+			f, err := os.Open(job.cs.filename)
+			if err != nil {
+				r <- fmt.Errorf("open: %v", err)
+				continue
+			}
+			if _, err := io.Copy(job.cs.hash, f); err != nil {
+				r <- err
+				continue
+			}
+			f.Close()
+			if fmt.Sprintf("%x", job.cs.hash.Sum(nil)) != job.cs.checksum {
+				r <- fmt.Errorf("%s: bad", job.cs.filename)
+			}
+		}
+		close(r)
+	}()
+	return r
+}

hash.go

@@ -1,149 +0,0 @@
-package main
-
-import (
-	"crypto/md5"
-	"crypto/sha1"
-	"crypto/sha256"
-	"crypto/sha512"
-	"fmt"
-	"hash"
-	"io"
-	"os"
-	"sort"
-	"sync"
-)
-
-// result is a message or error payload
-type result struct {
-	f   string
-	cs  string
-	err error
-}
-
-// results exists to sort a slice of result
-type results []result
-
-func (r results) Len() int           { return len(r) }
-func (r results) Swap(i, j int)      { r[i], r[j] = r[j], r[i] }
-func (r results) Less(i, j int) bool { return r[i].f < r[j].f }
-
-// hashr exists so that we can make a thing that can return valid hash.Hash
-// interfaces.
-type hashr func() hash.Hash
-
-// hsh figures out which hash algo to use, and distributes the work of hashing
-func hsh(files []string, verbose bool) chan result {
-	var h hashr
-	switch *algo {
-	case "sha1", "1":
-		h = sha1.New
-	case "sha256", "256":
-		h = sha256.New
-	case "sha512", "512":
-		h = sha512.New
-	case "md5":
-		h = md5.New
-	default:
-		r := make(chan result)
-		go func() {
-			r <- result{err: fmt.Errorf("unsupported algorithm: %v (supported: md5, sha1, sha256, sha512)", *algo)}
-			close(r)
-		}()
-		return r
-	}
-
-	if len(files) == 0 {
-		r := make(chan result)
-		go func() {
-			hsh := h()
-			_, err := io.Copy(hsh, os.Stdin)
-			if err != nil {
-				fmt.Fprintf(os.Stderr, "%v\n", err)
-				os.Exit(1)
-			}
-			r <- result{cs: fmt.Sprintf("%x", hsh.Sum(nil)), f: "-"}
-			close(r)
-		}()
-		return r
-	}
-
-	jobs := make(chan checksum)
-	go func() {
-		for _, name := range files {
-			jobs <- checksum{filename: name}
-		}
-		close(jobs)
-	}()
-
-	res := []<-chan result{}
-	for w := 0; w < *ngo; w++ {
-		res = append(res, compute(h, jobs, verbose))
-	}
-
-	o := make(chan result)
-	go func() {
-		rs := results{}
-		for r := range rmerge(res) {
-			rs = append(rs, r)
-		}
-		sort.Sort(rs)
-		for _, r := range rs {
-			o <- r
-		}
-		close(o)
-	}()
-	return o
-}
-
-// compute is the checksumming workhorse
-func compute(h hashr, jobs chan checksum, verbose bool) chan result {
-	hsh := h()
-	r := make(chan result)
-	go func() {
-		for job := range jobs {
-			f, err := os.Open(job.filename)
-			if err != nil {
-				r <- result{err: err}
-				continue
-			}
-			hsh.Reset()
-			_, err = io.Copy(hsh, f)
-			f.Close()
-			if err != nil {
-				r <- result{err: err}
-				continue
-			}
-			if verbose {
-				fmt.Fprintf(os.Stderr, "%v\n", job.filename)
-			}
-			r <- result{f: job.filename, cs: fmt.Sprintf("%x", hsh.Sum(nil))}
-		}
-		close(r)
-	}()
-	return r
-}
-
-// rmerge implements fan-in
-func rmerge(cs []<-chan result) chan result {
-	out := make(chan result)
-
-	var wg sync.WaitGroup
-
-	output := func(c <-chan result) {
-		for n := range c {
-			out <- n
-		}
-		wg.Done()
-	}
-
-	wg.Add(len(cs))
-	for _, c := range cs {
-		go output(c)
-	}
-
-	go func() {
-		wg.Wait()
-		close(out)
-	}()
-	return out
-}

license

@@ -1,5 +1,5 @@
 MIT License
-Copyright (c) 2016 derek mcquay, stephen mcquay
+Copyright (c) 2015 smcquay
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of
 this software and associated documentation files (the "Software"), to deal in

main.go

@@ -1,42 +1,78 @@
 package main
 
 import (
+	"crypto/md5"
+	"crypto/sha1"
+	"crypto/sha256"
+	"crypto/sha512"
 	"flag"
 	"fmt"
+	"io"
 	"os"
-	"runtime"
 )
 
-var algo = flag.String("a", "sha256", "algorithm to use")
+var algo = flag.String("a", "sha1", "algorithm to use")
 var mode = flag.Bool("c", false, "check")
-var ngo = flag.Int("n", runtime.NumCPU(), "number of goroutines")
-var verbose = flag.Bool("v", false, "vebose")
 
 func main() {
 	flag.Parse()
 	files := flag.Args()
 	switch *mode {
 	case true:
-		ec := 0
+		c := 0
-		for err := range check(files, *verbose) {
+		for err := range check(files) {
-			ec++
+			c++
 			fmt.Fprintf(os.Stderr, "%v\n", err)
 		}
-		if ec > 0 {
+		if c > 0 {
 			os.Exit(1)
 		}
 	case false:
-		ec := 0
+		if err := hsh(files); err != nil {
-		for res := range hsh(files, *verbose) {
+			fmt.Fprintf(os.Stderr, "%v\n", err)
-			if res.err != nil {
-				ec++
-				fmt.Fprintf(os.Stderr, "%v\n", res.err)
-			} else {
-				fmt.Printf("%v  %v\n", res.cs, res.f)
-			}
-		}
-		if ec > 0 {
 			os.Exit(1)
 		}
 	}
 }
+
+func hsh(files []string) error {
+	h := sha256.New()
+	switch *algo {
+	case "sha1", "1":
+		h = sha1.New()
+	case "sha256", "256":
+		h = sha256.New()
+	case "sha512", "512":
+		h = sha512.New()
+	case "md5":
+		h = md5.New()
+	default:
+		return fmt.Errorf("unsupported algorithm: %v", *algo)
+	}
+
+	if len(files) == 0 {
+		_, err := io.Copy(h, os.Stdin)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "%v\n", err)
+			os.Exit(1)
+		}
+		fmt.Printf("%x  -\n", h.Sum(nil))
+	} else {
+		for _, name := range files {
+			f, err := os.Open(name)
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "%v\n", err)
+				continue
+			}
+			h.Reset()
+			_, err = io.Copy(h, f)
+			f.Close()
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "%v\n", err)
+				continue
+			}
+			fmt.Printf("%x  %s\n", h.Sum(nil), name)
+		}
+	}
+	return nil
+}

readme.md

@@ -1,20 +1,12 @@
 # cs
 
-concurrently calculate/verify checksums (cs)
+calculate checksums
 
-It's a simpler version of shasum + md5sum, but concurrently and only with
+It's a simpler version of shasum + md5sum, but only for md5, sha1, sha256, and
-support for md5, sha1, sha256, and sha512.
+sha512.
 
 ## usage
 
-    # create checksums
     cs -a 256 < foo.txt
     cs foo.txt
-    cs -a sha1 foo.txt foo.txt foo.txt > checksums.sha1
+    cs -a sha1 foo.txt foo.txt foo.txt
-
-    # verify
-    cat checksums.sha1 | cs -c 
-    cs -c checksums.sha1
-
-    # both
-    cs $(find ~/src/mcquay.me | grep  '\.go$') | cs -c