Browse Source

add missing link cleanup subcommand.

Stephen McQuay 3 years ago
parent
commit
1abff34b3d
No known key found for this signature in database
4 changed files with 118 additions and 2 deletions
  1. 25
    0
      arrange.go
  2. 74
    0
      cmd/am/clean.go
  3. 13
    1
      cmd/am/main.go
  4. 6
    1
      media.go

+ 25
- 0
arrange.go View File

@@ -111,6 +111,31 @@ func Parse(in <-chan string) <-chan Media {
111 111
 	return out
112 112
 }
113 113
 
114
+// MissingLink detects if the values coming from medias is a duplicate file
115
+// rather than a hardlink to the content store.
116
+func MissingLink(medias <-chan Media, root string) (<-chan Media, <-chan error) {
117
+	out := make(chan Media)
118
+	errs := make(chan error)
119
+	go func() {
120
+		for m := range medias {
121
+			var d, c os.FileInfo
122
+			var err error
123
+			if d, err = os.Stat(m.Path); err != nil {
124
+				errs <- err
125
+			}
126
+			if c, err = os.Stat(m.Content(root)); err != nil {
127
+				errs <- err
128
+			}
129
+			if !os.SameFile(d, c) {
130
+				out <- m
131
+			}
132
+		}
133
+		close(errs)
134
+		close(out)
135
+	}()
136
+	return out, errs
137
+}
138
+
114 139
 // Move calls Move on each Media on input chan. It is the first step in the
115 140
 // pipeline after fan-in.
116 141
 func Move(in <-chan Media, root string) <-chan error {

+ 74
- 0
cmd/am/clean.go View File

@@ -0,0 +1,74 @@
1
+package main
2
+
3
+import (
4
+	"fmt"
5
+	"log"
6
+	"os"
7
+	"path/filepath"
8
+	"runtime"
9
+	"sync"
10
+
11
+	"mcquay.me/arrange"
12
+)
13
+
14
+func clean(dir string) error {
15
+	dateDir := filepath.Join(dir, "date")
16
+	if _, err := os.Stat(dateDir); os.IsNotExist(err) {
17
+		return fmt.Errorf("couldn't find 'date' dir in %q", dir)
18
+	}
19
+
20
+	work := arrange.Source(dateDir)
21
+	streams := []<-chan arrange.Media{}
22
+	errs := []<-chan error{}
23
+
24
+	workers := runtime.NumCPU()
25
+	if *cores != 0 {
26
+		workers = *cores
27
+	}
28
+
29
+	for w := 0; w < workers; w++ {
30
+		s, e := arrange.MissingLink(arrange.Parse(work), dir)
31
+		streams = append(streams, s)
32
+		errs = append(errs, e)
33
+	}
34
+
35
+	var err error
36
+	go func() {
37
+		for e := range eMerge(errs) {
38
+			log.Printf("%+v", e)
39
+			err = fmt.Errorf("%v, %v", err, e)
40
+		}
41
+	}()
42
+
43
+	for m := range arrange.Merge(streams) {
44
+		log.Printf("%q > %q", m.Path, m.Content(dir))
45
+		if err := os.Remove(m.Path); err != nil {
46
+			log.Printf("%+v", err)
47
+		}
48
+		if err := os.Link(m.Content(dir), m.Path); err != nil {
49
+			log.Printf("%+v", err)
50
+		}
51
+	}
52
+
53
+	return err
54
+}
55
+
56
+func eMerge(cs []<-chan error) <-chan error {
57
+	out := make(chan error)
58
+	var wg sync.WaitGroup
59
+	output := func(c <-chan error) {
60
+		for n := range c {
61
+			out <- n
62
+		}
63
+		wg.Done()
64
+	}
65
+	for _, c := range cs {
66
+		go output(c)
67
+	}
68
+	wg.Add(len(cs))
69
+	go func() {
70
+		wg.Wait()
71
+		close(out)
72
+	}()
73
+	return out
74
+}

+ 13
- 1
cmd/am/main.go View File

@@ -7,8 +7,9 @@ import (
7 7
 	"os"
8 8
 )
9 9
 
10
-const usage = "am <arr|help> [flags]"
10
+const usage = "am <arr|clean|help> [flags]"
11 11
 const arrUsage = "am arr [-h|-cores=N] <in> <out>"
12
+const cleanUsage = "am clean [-h|-cores=N] <directory>"
12 13
 
13 14
 type stats struct {
14 15
 	total int
@@ -41,6 +42,17 @@ func main() {
41 42
 			fmt.Fprintf(os.Stderr, "problem arranging media: %v\n", err)
42 43
 			os.Exit(1)
43 44
 		}
45
+	case "c", "cl", "clean":
46
+		args := flag.Args()
47
+		if len(args) != 1 {
48
+			fmt.Fprintf(os.Stderr, "%s\n", cleanUsage)
49
+			os.Exit(1)
50
+		}
51
+		dir := args[0]
52
+		if err := clean(dir); err != nil {
53
+			fmt.Fprintf(os.Stderr, "problem cleaning: %v\n", err)
54
+			os.Exit(1)
55
+		}
44 56
 	default:
45 57
 		fmt.Fprintf(os.Stderr, "%s\n", usage)
46 58
 		os.Exit(1)

+ 6
- 1
media.go View File

@@ -25,7 +25,7 @@ func (m Media) Move(root string) error {
25 25
 	}
26 26
 	defer f.Close()
27 27
 
28
-	content := filepath.Join(root, "content", m.Hash[:2], m.Hash[2:]+m.Extension)
28
+	content := m.Content(root)
29 29
 
30 30
 	if _, err := os.Stat(content); !os.IsNotExist(err) {
31 31
 		return Dup{content}
@@ -67,3 +67,8 @@ func (m Media) Move(root string) error {
67 67
 	// return os.Symlink(rel, name)
68 68
 	return os.Link(content, name)
69 69
 }
70
+
71
+// Content returns the content-address path starting at root.
72
+func (m Media) Content(root string) string {
73
+	return filepath.Join(root, "content", m.Hash[:2], m.Hash[2:]+m.Extension)
74
+}

Loading…
Cancel
Save