pm ls <pkg> lists installed files

Fixes #13

Gopkg.lock

@@ -19,7 +19,13 @@
     "openpgp/packet",
     "openpgp/s2k"
   ]
-  revision = "49796115aa4b964c318aad4f3084fdb41e9aa067"
+  revision = "8c653846df49742c4c85ec37e5d9f8d3ba657895"
+
+[[projects]]
+  name = "gopkg.in/yaml.v2"
+  packages = ["."]
+  revision = "7f97868eec74b32b0982dd158a51a446d1da7eb5"
+  version = "v2.1.1"
 
 [[projects]]
   name = "mcquay.me/fs"
@@ -30,6 +36,6 @@
 [solve-meta]
   analyzer-name = "dep"
   analyzer-version = 1
-  inputs-digest = "9bcd44fda9f03cef35ea82538ef76216e012e27acc1b32a8f5a684f814942428"
+  inputs-digest = "a4d32970fbd71fcaec77a9ad03f9bbe56005cd80f00fff4329b314a3458e8d1f"
   solver-name = "gps-cdcl"
   solver-version = 1

Gopkg.toml

@@ -40,3 +40,7 @@
 [prune]
   go-tests = true
   unused-packages = true
+
+[[constraint]]
+  name = "gopkg.in/yaml.v2"
+  version = "2.1.1"

Makefile

@@ -0,0 +1,10 @@
+default: lint test
+
+.PHONY: lint
+lint:
+	@golint $(shell go list mcquay.me/pm/...)
+	@go vet $(shell go list mcquay.me/pm/...)
+
+.PHONY: test
+test:
+	@go test -cover $(shell go list mcquay.me/pm/...)

README.md

@@ -49,7 +49,6 @@ following files:
 ```yaml
 name: foo
 version: 2.3.29
-namespace: /darwin/amd64
 description: Foo is the world's simplest frobnicator
 deps: [baz, bar@0.9.2]
 ```
@@ -88,8 +87,8 @@ The example remote url:
 encodes a remote that is served over `https` on the host `pm.mcquay.me` and
 informs the client to pull packages from the `/darwin/amd64/testing` namespace,
 specified by the Path. `pm pull` will collect available package information
-from the remote for a given namespace and will populate its local database with
-the contents of the response. `pm` can then list available packages, and
+from configured remote and will populate its local database with the contents
+of the response. `pm` can then list available packages, and the user can then
 request that they be installed.
 
 As a practical example a client can be configured to pull from two `remotes`

available.go

@@ -0,0 +1,187 @@
+package pm
+
+import (
+	"fmt"
+	"net/url"
+	"sort"
+	"strings"
+
+	"github.com/pkg/errors"
+)
+
+// Name exists to document the keys in Available
+type Name string
+
+// Names is a slice of names ... with sorting!
+type Names []Name
+
+func (n Names) Len() int           { return len(n) }
+func (n Names) Swap(a, b int)      { n[a], n[b] = n[b], n[a] }
+func (n Names) Less(a, b int) bool { return n[a] < n[b] }
+
+// Version exists to document the keys in Available
+type Version string
+
+// Versions is a slice of Version ... with sorting!
+type Versions []Version
+
+// TODO (sm): make this semver sort?
+func (v Versions) Len() int           { return len(v) }
+func (v Versions) Swap(a, b int)      { v[a], v[b] = v[b], v[a] }
+func (v Versions) Less(a, b int) bool { return v[a] < v[b] }
+
+type label struct {
+	n Name
+	v Version
+}
+
+type labels []label
+
+// TODO (sm): make this semver sort?
+func (n labels) Len() int      { return len(n) }
+func (n labels) Swap(a, b int) { n[a], n[b] = n[b], n[a] }
+func (n labels) Less(a, b int) bool {
+	if n[a].n != n[b].n {
+		return n[a].n < n[b].n
+	}
+	return n[a].v < n[b].v
+}
+
+// Available is the structure used to represent the collection of all packages
+// that can be installed.
+type Available map[Name]map[Version]Meta
+
+// Get returns the meta stored at a[n][v] or an error explaining why it could
+// not be Get.
+func (a Available) Get(n Name, v Version) (Meta, error) {
+	if _, ok := a[n]; !ok {
+		return Meta{}, errors.Errorf("could not find package named %q", n)
+	}
+
+	if v == "" {
+		if len(a[n]) == 0 {
+			return Meta{}, errors.Errorf("no configured versions for %q", n)
+		}
+		vers := Versions{}
+		for ver := range a[n] {
+			vers = append(vers, ver)
+		}
+		sort.Sort(vers)
+		v = vers[len(vers)-1]
+	}
+
+	if _, ok := a[n][v]; !ok {
+		return Meta{}, errors.Errorf("could not find %v@%v in database", n, v)
+	}
+	return a[n][v], nil
+}
+
+// Add inserts m into a.
+func (a Available) Add(m Meta) error {
+	if _, err := m.Valid(); err != nil {
+		return errors.Wrap(err, "invalid meta")
+	}
+
+	if _, ok := a[Name(m.Name)]; !ok {
+		a[m.Name] = map[Version]Meta{}
+	}
+	a[m.Name][m.Version] = m
+	return nil
+}
+
+// Update inserts all data from o into a.
+func (a Available) Update(o Available) error {
+	for _, vers := range o {
+		for _, m := range vers {
+			if err := a.Add(m); err != nil {
+				return errors.Wrap(err, "adding")
+			}
+		}
+	}
+	return nil
+}
+
+// SetRemote adds the information in the url to the database.
+func (a Available) SetRemote(u url.URL) {
+	for n, vers := range a {
+		for v := range vers {
+			m := a[n][v]
+			m.Remote = u
+			a[n][v] = m
+		}
+	}
+}
+
+// Traverse returns a chan of Meta that will be sanely sorted.
+func (a Available) Traverse() <-chan Meta {
+	r := make(chan Meta)
+	go func() {
+		names := Names{}
+		nvs := map[Name]Versions{}
+		for n, vers := range a {
+			names = append(names, n)
+			for v := range vers {
+				nvs[n] = append(nvs[n], v)
+			}
+			sort.Sort(nvs[n])
+		}
+		sort.Sort(names)
+
+		for _, n := range names {
+			for _, v := range nvs[n] {
+				r <- a[n][v]
+			}
+		}
+		close(r)
+	}()
+	return r
+}
+
+func labelForString(s string) (label, error) {
+	r := label{}
+	c := strings.Count(s, "@")
+	switch c {
+	case 0:
+		r.n = Name(s)
+	case 1:
+		sp := strings.Split(s, "@")
+		r.n, r.v = Name(sp[0]), Version(sp[1])
+	default:
+		return r, fmt.Errorf("unexpected number of '@' found, got %v, want 1", c)
+	}
+	if r.n == "" {
+		return r, fmt.Errorf("name cannot be empty")
+	}
+	return r, nil
+}
+
+// Installable calculates if the packages requested in "in" can be installed.
+func (a Available) Installable(in []string) (Metas, error) {
+	ls := labels{}
+	for _, i := range in {
+		l, err := labelForString(i)
+		if err != nil {
+			return nil, errors.Wrap(err, "parsing name/version")
+		}
+		ls = append(ls, l)
+	}
+
+	seen := map[Name]bool{}
+	for _, l := range ls {
+		if _, ok := seen[l.n]; ok {
+			return nil, fmt.Errorf("can only ask to install %q once", l.n)
+		}
+		seen[l.n] = true
+	}
+
+	ms := Metas{}
+	for _, l := range ls {
+		m, err := a.Get(l.n, l.v)
+		if err != nil {
+			return ms, errors.Wrapf(err, "getting %v", l)
+		}
+		ms = append(ms, m)
+	}
+
+	return ms, nil
+}

available_test.go

@@ -0,0 +1,105 @@
+package pm
+
+import (
+	"errors"
+	"net/url"
+	"testing"
+)
+
+func TestAvailableAdd(t *testing.T) {
+
+	tests := []struct {
+		label string
+		m     Meta
+		count int
+		err   error
+	}{
+		{
+			label: "good",
+			m:     Meta{Name: "a", Version: "v1.0.0", Description: "test"},
+			count: 1,
+		},
+		{
+			label: "bad meta",
+			m:     Meta{Name: "a"},
+			count: 1,
+			err:   errors.New("missing"),
+		},
+		{
+			label: "dupe is last in",
+			m:     Meta{Name: "a", Version: "v1.0.0", Description: "better version"},
+			count: 1,
+		},
+		{
+			label: "another good",
+			m:     Meta{Name: "a", Version: "v1.0.0", Description: "better version"},
+			count: 1,
+		},
+	}
+
+	a := Available{}
+	for _, test := range tests {
+		t.Run(test.label, func(t *testing.T) {
+			if err := a.Add(test.m); (err == nil) != (test.err == nil) {
+				t.Fatalf("adding meta%v", err)
+			}
+
+			if got, want := len(a), test.count; got != want {
+				t.Fatalf("unexpected length after Add: got %v, want %v", got, want)
+			}
+		})
+	}
+
+	if got, want := a["a"]["v1.0.0"].Description, "better version"; got != want {
+		t.Fatalf("version: got %v, want %v", got, want)
+	}
+}
+
+func TestAvailableUpdate(t *testing.T) {
+	a := Available{}
+	if err := a.Add(Meta{Name: "a", Version: "v1.0.0", Description: "test"}); err != nil {
+		t.Fatalf("add: %v", err)
+	}
+	if err := a.Add(Meta{Name: "b", Version: "v2.0.0", Description: "test"}); err != nil {
+		t.Fatalf("add: %v", err)
+	}
+
+	b := Available{}
+	a.Update(b)
+	if got, want := len(a), 2; got != want {
+		t.Fatalf("len after empty update: got %v, want %v", got, want)
+	}
+
+	if err := b.Add(Meta{Name: "a", Version: "v1.0.0", Description: "test last in"}); err != nil {
+		t.Fatalf("add: %v", err)
+	}
+	if err := b.Add(Meta{Name: "b", Version: "v2.1.0", Description: "test"}); err != nil {
+		t.Fatalf("add: %v", err)
+	}
+
+	if err := a.Update(b); err != nil {
+		t.Fatalf("update: %v", err)
+	}
+	if got, want := len(a), 2; got != want {
+		t.Fatalf("len after update: got %v, want %v", got, want)
+	}
+	if got, want := len(a["a"]), 1; got != want {
+		t.Fatalf("len after update: got %v, want %v", got, want)
+	}
+	if got, want := len(a["b"]), 2; got != want {
+		t.Fatalf("len after update: got %v, want %v", got, want)
+	}
+
+	if got, want := a["a"]["v1.0.0"].Description, "test last in"; got != want {
+		t.Fatalf("last in didn't override")
+	}
+
+	u, err := url.Parse("https://pm.mcquay.me/darwin/amd64")
+	if err != nil {
+		t.Fatalf("parsing url: %v", err)
+	}
+	a.SetRemote(*u)
+	if got, want := a["a"]["v1.0.0"].Remote, *u; got != want {
+		t.Fatalf("last in didn't override")
+	}
+}

cmd/pm/main.go

@@ -4,15 +4,31 @@ import (
 	"bufio"
 	"fmt"
 	"os"
+	"path/filepath"
 
+	"github.com/pkg/errors"
+	"mcquay.me/fs"
+	"mcquay.me/pm/db"
 	"mcquay.me/pm/keyring"
+	"mcquay.me/pm/pkg"
 )
 
+// Version stores the current version, and is updated at build time.
+const Version = "dev"
+
 const usage = `pm: simple, cross-platform system package manager
 
 subcommands:
+  available  (av)  -- print out all installable packages
   environ    (env) -- print environment information
+  install    (in)  -- install packages
   keyring    (key) -- interact with pm's OpenPGP keyring
+  ls               -- list installed packages
+  package    (pkg) -- create packages
+  pull             -- fetch all available packages from all configured remotes
+  remote           -- configure remote pmd servers
+  rm               -- remove packages
+  version    (v)   -- print version information
 `
 
 const keyUsage = `pm keyring: interact with pm's OpenPGP keyring
@@ -21,12 +37,26 @@ subcommands:
   create      (c)  --  create a fresh keypair
   export      (e)  --  export a public key to stdout
   import      (i)  --  import a public key from stdin
-  list        (ls) --  list configured key info
-  remove      (rm) --  remove a key from the keyring
+  ls               --  list configured key info
+  rm               --  remove a key from the keyring
   sign        (s)  --  sign a file
   verify      (v)  --  verify a detached signature
 `
 
+const pkgUsage = `pm package: generate pm-compatible packages
+
+subcommands:
+  create      (c)  --  create a fresh keypair
+`
+
+const remoteUsage = `pm remote: configure remote pmd servers
+
+subcommands:
+  add         (a)  --  add a URI
+  ls               --  list configured remotes
+  rm               --  remove a URI
+`
+
 func main() {
 	if len(os.Args) < 2 {
 		fatalf("pm: missing subcommand\n\n%v", usage)
@@ -49,7 +79,7 @@ func main() {
 		}
 		sub, args := os.Args[2], os.Args[3:]
 		switch sub {
-		case "ls", "list":
+		case "ls":
 			if err := keyring.ListKeys(root, os.Stdout); err != nil {
 				fatalf("listing keypair: %v\n", err)
 			}
@@ -90,7 +120,11 @@ func main() {
 			if signID == "" {
 				fatalf("must set PM_PGP_ID\n")
 			}
-			if err := keyring.Sign(root, signID, os.Stdin, os.Stdout); err != nil {
+			e, err := keyring.FindSecretEntity(root, signID)
+			if err != nil {
+				fatalf("find secret key: %v\n", err)
+			}
+			if err := keyring.Sign(e, os.Stdin, os.Stdout); err != nil {
 				fatalf("signing: %v\n", err)
 			}
 		case "verify", "v":
@@ -115,7 +149,7 @@ func main() {
 			if err := keyring.Import(root, os.Stdin); err != nil {
 				fatalf("importing key: %v\n", err)
 			}
-		case "remove", "rm":
+		case "rm":
 			if len(args) != 1 {
 				fatalf("missing key id\n\nusage: pm key remove <id>\n")
 			}
@@ -126,6 +160,100 @@ func main() {
 		default:
 			fatalf("unknown keyring subcommand: %q\n\nusage: %v", sub, keyUsage)
 		}
+	case "package", "pkg":
+		if len(os.Args[1:]) < 2 {
+			fatalf("pm package: insufficient args\n\nusage: %v", pkgUsage)
+		}
+		sub := os.Args[2]
+		switch sub {
+		case "create", "creat", "c":
+			if signID == "" {
+				fatalf("must set PM_PGP_ID\n")
+			}
+			args := os.Args[3:]
+			if len(args) != 1 {
+				fatalf("usage: pm package create <directory>\n")
+			}
+			dir := args[0]
+			e, err := keyring.FindSecretEntity(root, signID)
+			if err != nil {
+				fatalf("find secret key: %v\n", err)
+			}
+			if err := pkg.Create(e, dir); err != nil {
+				fatalf("creating package: %v\n", err)
+			}
+		default:
+			fatalf("unknown package subcommand: %q\n\nusage: %v", sub, pkgUsage)
+		}
+	case "remote":
+		if len(os.Args[1:]) < 2 {
+			fatalf("pm remote: insufficient args\n\nusage: %v", remoteUsage)
+		}
+		sub := os.Args[2]
+		args := os.Args[3:]
+
+		if err := mkdirs(root); err != nil {
+			fatalf("making pm var directories: %v\n", err)
+		}
+
+		switch sub {
+		case "add", "a":
+			if len(args) < 1 {
+				fatalf("missing arg\n\nusage: pm remote add [<uris>]\n")
+			}
+			if err := db.AddRemotes(root, args); err != nil {
+				fatalf("remote add: %v\n", err)
+			}
+		case "rm":
+			if len(args) < 1 {
+				fatalf("missing arg\n\nusage: pm remote rm [<uris>]\n")
+			}
+			if err := db.RemoveRemotes(root, args); err != nil {
+				fatalf("remote remove: %v\n", err)
+			}
+		case "ls":
+			if err := db.ListRemotes(root, os.Stdout); err != nil {
+				fatalf("list: %v\n", err)
+			}
+		default:
+			fatalf("unknown package subcommand: %q\n\nusage: %v", sub, remoteUsage)
+		}
+	case "pull":
+		if err := db.Pull(root); err != nil {
+			fatalf("pulling available packages: %v\n", err)
+		}
+	case "available", "av":
+		if err := db.ListAvailable(root, os.Stdout); err != nil {
+			fatalf("pulling available packages: %v\n", err)
+		}
+	case "install", "in":
+		if len(os.Args[1:]) < 2 {
+			fatalf("pm install: insufficient args\n\nusage: pm install [pkg1, pkg2, ..., pkgN]\n")
+		}
+		pkgs := os.Args[2:]
+		if err := pkg.Install(root, pkgs); err != nil {
+			fatalf("installing: %v\n", err)
+		}
+	case "ls":
+		if len(os.Args[1:]) == 1 {
+			if err := db.ListInstalled(root, os.Stdout); err != nil {
+				fatalf("listing installed: %v\n", err)
+			}
+		} else {
+			if err := db.ListInstalledFiles(root, os.Stdout, os.Args[2:]); err != nil {
+				fatalf("listing installed: %v\n", err)
+			}
+		}
+	case "rm":
+		if len(os.Args[1:]) < 2 {
+			fatalf("pm rm: insufficient args\n\nusage: pm rm [pkg1, pkg2, ..., pkgN]\n")
+		}
+		pkgs := os.Args[2:]
+		if err := pkg.Remove(root, pkgs); err != nil {
+			fatalf("removing: %v\n", err)
+		}
+	case "version", "v":
+		fmt.Printf("pm: version %v\n", Version)
 	default:
 		fatalf("uknown subcommand %q\n\nusage: %v", cmd, usage)
 	}
@@ -135,3 +263,13 @@ func fatalf(f string, args ...interface{}) {
 	fmt.Fprintf(os.Stderr, f, args...)
 	os.Exit(1)
 }
+
+func mkdirs(root string) error {
+	d := filepath.Join(root, "var", "lib", "pm")
+	if !fs.Exists(d) {
+		if err := os.MkdirAll(d, 0700); err != nil {
+			return errors.Wrap(err, "mk pm dir")
+		}
+	}
+	return nil
+}

cs.go

@@ -0,0 +1,22 @@
+package pm
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"strings"
+)
+
+// ParseCS returns a parsed checksum file.
+func ParseCS(f io.Reader) (map[string]string, error) {
+	cs := map[string]string{}
+	s := bufio.NewScanner(f)
+	for s.Scan() {
+		elems := strings.Split(s.Text(), "\t")
+		if len(elems) != 2 {
+			return nil, fmt.Errorf("manifest format error; got %d elements, want 2", len(elems))
+		}
+		cs[elems[1]] = elems[0]
+	}
+	return cs, nil
+}

db/available.go

@@ -0,0 +1,99 @@
+package db
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"path/filepath"
+
+	"github.com/pkg/errors"
+	"mcquay.me/fs"
+	"mcquay.me/pm"
+)
+
+const an = "var/lib/pm/available.json"
+
+// Pull updates the available package database.
+func Pull(root string) error {
+	db, err := load(root)
+	if err != nil {
+		return errors.Wrap(err, "loading db")
+	}
+
+	o := pm.Available{}
+
+	// Order here is important: the guarantee made is that any packages that
+	// exist in multiple remotes will be fetched by the first configured
+	// remote, which is why we traverse the database in reverse.
+	//
+	// TODO (sm): make this concurrent
+	for i := range db {
+		u := db[len(db)-i-1]
+		resp, err := http.Get(u.String() + "/available.json")
+		if err != nil {
+			return errors.Wrap(err, "http get")
+		}
+
+		a := pm.Available{}
+		if err := json.NewDecoder(resp.Body).Decode(&a); err != nil {
+			return errors.Wrapf(err, "decode remote available for %q", u.String())
+		}
+		a.SetRemote(u)
+		o.Update(a)
+	}
+	if err := saveAvailable(root, o); err != nil {
+		return errors.Wrap(err, "saving available db")
+	}
+	return nil
+}
+
+// ListAvailable prints all installable packages
+func ListAvailable(root string, w io.Writer) error {
+	db, err := LoadAvailable(root)
+	if err != nil {
+		return errors.Wrap(err, "loading")
+	}
+	for m := range db.Traverse() {
+		fmt.Fprintf(w, "%v\t%v\t%v\n", m.Name, m.Version, m.Remote.String())
+	}
+	return nil
+}
+
+// LoadAvailable returns the collection of available packages
+func LoadAvailable(root string) (pm.Available, error) {
+	r := pm.Available{}
+	dbn := filepath.Join(root, rn)
+
+	if !fs.Exists(dbn) {
+		return r, nil
+	}
+
+	f, err := os.Open(filepath.Join(root, an))
+	if err != nil {
+		return r, errors.Wrap(err, "open")
+	}
+
+	if err := json.NewDecoder(f).Decode(&r); err != nil {
+		return r, errors.Wrap(err, "decoding db")
+	}
+
+	return r, nil
+}
+
+func saveAvailable(root string, db pm.Available) error {
+	f, err := os.Create(filepath.Join(root, an))
+	if err != nil {
+		return errors.Wrap(err, "create")
+	}
+	enc := json.NewEncoder(f)
+	enc.SetIndent("", "\t")
+	if err := enc.Encode(&db); err != nil {
+		return errors.Wrap(err, "decoding db")
+	}
+	if err := f.Close(); err != nil {
+		return errors.Wrap(err, "close db")
+	}
+	return nil
+}

db/installed.go

@@ -0,0 +1,138 @@
+package db
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"sort"
+
+	"github.com/pkg/errors"
+	"mcquay.me/fs"
+	"mcquay.me/pm"
+)
+
+const in = "var/lib/pm/installed.json"
+
+// AddInstalled adds m to the installed package database.
+func AddInstalled(root string, m pm.Meta) error {
+	db, err := loadi(root)
+	if err != nil {
+		return errors.Wrap(err, "loading installed db")
+	}
+	db[m.Name] = m
+	return savei(root, db)
+}
+
+// RemoveInstalled adds m to the installed package database.
+func RemoveInstalled(root string, m pm.Meta) error {
+	db, err := loadi(root)
+	if err != nil {
+		return errors.Wrap(err, "loading installed db")
+	}
+	delete(db, m.Name)
+	return savei(root, db)
+}
+
+// IsInstalled checks if m is in the installed package database.
+func IsInstalled(root string, m pm.Meta) (bool, error) {
+	db, err := loadi(root)
+	if err != nil {
+		return false, errors.Wrap(err, "loading installed db")
+	}
+
+	_, r := db[m.Name]
+	return r, nil
+}
+
+// ListInstalled pretty prints the installed database to w.
+func ListInstalled(root string, w io.Writer) error {
+	db, err := loadi(root)
+	if err != nil {
+		return errors.Wrap(err, "loading installed db")
+	}
+
+	for m := range db.Traverse() {
+		fmt.Fprintf(w, "%v\t%v\t%v\n", m.Name, m.Version, m.Remote.String())
+	}
+	return nil
+}
+
+// ListInstalledFiles prints the contents of a package.
+func ListInstalledFiles(root string, w io.Writer, names []string) error {
+	for _, name := range names {
+		ok, err := IsInstalled(root, pm.Meta{Name: pm.Name(name)})
+		if err != nil {
+			return errors.Wrap(err, "is installed")
+		}
+		if !ok {
+			return fmt.Errorf("%v not installed", name)
+		}
+	}
+
+	for _, name := range names {
+		fn := filepath.Join(root, "var", "lib", "pm", "installed", name, "bom.sha256")
+		f, err := os.Open(fn)
+		if err != nil {
+			return errors.Wrapf(err, "opening %v's bom", name)
+		}
+		bom, err := pm.ParseCS(f)
+		if err != nil {
+			return errors.Wrapf(err, "parsing %v's bom", name)
+		}
+		if err := f.Close(); err != nil {
+			return errors.Wrapf(err, "closing %v's bom", name)
+		}
+
+		ks := []string{}
+		for k := range bom {
+			ks = append(ks, k)
+		}
+		sort.Strings(ks)
+		for _, k := range ks {
+			fmt.Fprintf(w, "%v\n", k)
+		}
+	}
+	return nil
+}
+
+func LoadInstalled(root string) (pm.Installed, error) {
+	return loadi(root)
+}
+
+func loadi(root string) (pm.Installed, error) {
+	r := pm.Installed{}
+	dbn := filepath.Join(root, in)
+
+	if !fs.Exists(dbn) {
+		return r, nil
+	}
+
+	f, err := os.Open(dbn)
+	if err != nil {
+		return r, errors.Wrap(err, "open")
+	}
+
+	if err := json.NewDecoder(f).Decode(&r); err != nil {
+		return r, errors.Wrap(err, "decoding db")
+	}
+
+	return r, nil
+}
+
+func savei(root string, db pm.Installed) error {
+	f, err := os.Create(filepath.Join(root, in))
+	if err != nil {
+		return errors.Wrap(err, "create")
+	}
+	enc := json.NewEncoder(f)
+	enc.SetIndent("", "\t")
+	if err := enc.Encode(&db); err != nil {
+		return errors.Wrap(err, "decoding db")
+	}
+	if err := f.Close(); err != nil {
+		return errors.Wrap(err, "close db")
+	}
+	return nil
+}

db/remote.go

@@ -0,0 +1,150 @@
+package db
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/url"
+	"os"
+	"path/filepath"
+
+	"github.com/pkg/errors"
+	"mcquay.me/fs"
+)
+
+// DB is a slice of available URI
+type DB []url.URL
+
+const rn = "var/lib/pm/remotes.json"
+
+// AddRemotes appends the provided uri to the list of configured remotes.
+func AddRemotes(root string, uris []string) error {
+	db, err := load(root)
+	if err != nil {
+		return errors.Wrap(err, "loading")
+	}
+
+	dbm := map[string]bool{}
+	for _, u := range db {
+		dbm[u.String()] = true
+	}
+
+	for _, uri := range uris {
+		pu, err := url.Parse(uri)
+		if err != nil {
+			return errors.Wrap(err, "url parse")
+		}
+
+		u := strip(*pu)
+
+		if _, ok := dbm[u.String()]; ok {
+			return fmt.Errorf("%q already in db", u.String())
+		}
+		db = append(db, u)
+	}
+
+	return save(root, db)
+}
+
+// RemoveRemotes removes the given uri from the list of configured remotes.
+func RemoveRemotes(root string, uris []string) error {
+	db, err := load(root)
+	if err != nil {
+		return errors.Wrap(err, "loading")
+	}
+
+	rms := map[string]bool{}
+	for _, uri := range uris {
+		pu, err := url.Parse(uri)
+		if err != nil {
+			return errors.Wrap(err, "url parse")
+		}
+
+		u := strip(*pu)
+
+		rms[u.String()] = true
+	}
+
+	o := DB{}
+	for _, d := range db {
+		if _, ok := rms[d.String()]; !ok {
+			o = append(o, d)
+		}
+	}
+
+	if len(o) == len(db) {
+		return errors.New("found no matching remotes")
+	}
+
+	return save(root, o)
+}
+
+// ListRemotes prints all configured remotes to w.
+func ListRemotes(root string, w io.Writer) error {
+	db, err := load(root)
+	if err != nil {
+		return errors.Wrap(err, "loading")
+	}
+	for _, u := range db {
+		fmt.Fprintf(w, "%s\n", u.String())
+	}
+	return nil
+}
+
+func load(root string) (DB, error) {
+	r := DB{}
+	dbn := filepath.Join(root, rn)
+
+	if !fs.Exists(dbn) {
+		return r, nil
+	}
+
+	f, err := os.Open(filepath.Join(root, rn))
+	if err != nil {
+		return r, errors.Wrap(err, "open")
+	}
+
+	if err := json.NewDecoder(f).Decode(&r); err != nil {
+		return r, errors.Wrap(err, "decoding db")
+	}
+
+	return r, nil
+}
+
+func save(root string, db DB) error {
+	f, err := os.Create(filepath.Join(root, rn))
+	if err != nil {
+		return errors.Wrap(err, "create")
+	}
+	enc := json.NewEncoder(f)
+	enc.SetIndent("", "\t")
+
+	if err := enc.Encode(&db); err != nil {
+		return errors.Wrap(err, "decoding db")
+	}
+
+	if err := f.Close(); err != nil {
+		return errors.Wrap(err, "close db")
+	}
+
+	return nil
+}
+
+// strip removes all fields we don't currently need.
+func strip(u url.URL) url.URL {
+	return url.URL{
+		Scheme: u.Scheme,
+		Host:   u.Host,
+		Path:   u.Path,
+	}
+}
+
+func mkdirs(root string) error {
+	d, _ := filepath.Split(filepath.Join(root, rn))
+	if !fs.Exists(d) {
+		if err := os.MkdirAll(d, 0700); err != nil {
+			return errors.Wrap(err, "mk pm dir")
+		}
+	}
+	return nil
+}

db/remote_test.go

@@ -0,0 +1,157 @@
+package db
+
+import (
+	"bytes"
+	"io/ioutil"
+	"os"
+	"strings"
+	"testing"
+)
+
+// TODO (sm): add more tests, including
+// - empty add
+// - removing db to empty
+// - bad uris
+
+func dirMe(t *testing.T) (string, func()) {
+	root, err := ioutil.TempDir("", "pm-tests-")
+	if err != nil {
+		t.Fatalf("tmpdir: %v", err)
+	}
+	if err := mkdirs(root); err != nil {
+		t.Fatalf("making pm dirs: %v", err)
+	}
+	return root, func() {
+		if err := os.RemoveAll(root); err != nil {
+			t.Fatalf("cleanup: %v", err)
+		}
+	}
+}
+
+func TestAdd(t *testing.T) {
+	root, del := dirMe(t)
+	defer del()
+
+	{
+		db, err := load(root)
+		if err != nil {
+			t.Fatalf("load: %v", err)
+		}
+		if got, want := len(db), 0; got != want {
+			t.Fatalf("empty db not empty: got %v, want %v", got, want)
+		}
+	}
+
+	bad := []string{
+		"http\ns://\nFoo|n",
+	}
+
+	if err := AddRemotes(root, bad); err == nil {
+		t.Fatalf("didn't detect bad url")
+	}
+
+	uris := []string{
+		"https://pm.mcquay.me/darwin/amd64",
+	}
+
+	if err := AddRemotes(root, uris); err != nil {
+		t.Fatalf("add: %v", err)
+	}
+
+	db, err := load(root)
+	if err != nil {
+		t.Fatalf("load: %v", err)
+	}
+
+	if got, want := len(db), len(uris); got != want {
+		t.Fatalf("unepected number of uris; got %v, want %v", got, want)
+	}
+
+	for _, u := range uris {
+		found := false
+		for _, d := range db {
+			if d.String() == u {
+				found = true
+			}
+		}
+		if !found {
+			t.Fatalf("did not find %v in the db", u)
+		}
+	}
+
+	if err := AddRemotes(root, uris); err == nil {
+		t.Fatalf("did not detect duplicate, and should have")
+	}
+}
+
+func TestRemove(t *testing.T) {
+	root, del := dirMe(t)
+	defer del()
+
+	if err := RemoveRemotes(root, nil); err == nil {
+		t.Fatalf("should have returned error on empty db")
+	}
+
+	uris := []string{
+		"https://pm.mcquay.me/foo",
+		"https://pm.mcquay.me/bar",
+		"https://pm.mcquay.me/baz",
+	}
+
+	if err := RemoveRemotes(root, uris); err == nil {
+		t.Fatalf("should have returned error asking to remove many uri on empty db")
+	}
+
+	if err := AddRemotes(root, uris); err != nil {
+		t.Fatalf("add: %v", err)
+	}
+	if err := RemoveRemotes(root, uris[1:2]); err != nil {
+		t.Fatalf("remove: %v", err)
+	}
+	db, err := load(root)
+	if err != nil {
+		t.Fatalf("load: %v", err)
+	}
+	if got, want := len(db), len(uris)-1; got != want {
+		t.Fatalf("unepected number of uris; got %v, want %v", got, want)
+	}
+
+	found := false
+	for _, d := range db {
+		if d.String() == uris[1] {
+			found = true
+		}
+	}
+	if found {
+		for _, v := range db {
+			t.Logf("%v", v.String())
+		}
+		t.Fatalf("failed to remove %v", uris[1:2])
+	}
+}
+
+func TestList(t *testing.T) {
+	root, del := dirMe(t)
+	defer del()
+	uris := []string{
+		"https://pm.mcquay.me/foo",
+		"https://pm.mcquay.me/bar",
+		"https://pm.mcquay.me/baz",
+	}
+
+	if err := AddRemotes(root, uris); err != nil {
+		t.Fatalf("add: %v", err)
+	}
+
+	buf := &bytes.Buffer{}
+	if err := ListRemotes(root, buf); err != nil {
+		t.Fatalf("list: %v", err)
+	}
+
+	for _, u := range uris {
+		if !strings.Contains(buf.String(), u) {
+			t.Fatalf("could not find %q in output\n%v", u, buf.String())
+		}
+	}
+
+}

installed.go

@@ -0,0 +1,73 @@
+package pm
+
+import (
+	"errors"
+	"fmt"
+	"sort"
+	"strings"
+)
+
+// Installed tracks installed packages.
+type Installed map[Name]Meta
+
+// Traverse returns a chan of Meta that will be sanely sorted.
+func (i Installed) Traverse() <-chan Meta {
+	r := make(chan Meta)
+	go func() {
+		names := Names{}
+		for n := range i {
+			names = append(names, n)
+		}
+		sort.Sort(names)
+
+		for _, n := range names {
+			r <- i[n]
+		}
+		close(r)
+	}()
+	return r
+}
+
+// Removable calculates if the packages requested in "in" can all be removed.
+func (i Installed) Removable(names []string) (Metas, error) {
+	inm := map[Name]bool{}
+
+	// XXX (sm): here we simply check if the package exists; eventually we'll
+	// have to check transitive dependencies, and deal with explicitly and
+	// implicitly installed packages.
+
+	found := map[Name]Meta{}
+	for _, name := range names {
+		n := Name(name)
+		inm[n] = true
+		if m, ok := i[n]; ok {
+			found[n] = m
+		}
+	}
+
+	if len(found) > len(inm) {
+		return nil, errors.New("should not have been able to find more than asked for, but did; internals are inconsistent.")
+	} else if len(inm) > len(found) {
+		// user asked for something that isn't installed.
+		missing := []string{}
+		for _, name := range names {
+			if _, ok := found[Name(name)]; !ok {
+				missing = append(missing, name)
+			}
+		}
+		return nil, fmt.Errorf("packages not installed: %v", strings.Join(missing, ", "))
+	}
+
+	if len(found) != len(inm) {
+		return nil, fmt.Errorf("escapes logic")
+	}
+
+	// XXX (sm): the ordering here will also eventually depend on transitive
+	// dependencies.
+	r := Metas{}
+	for _, m := range found {
+		r = append(r, m)
+	}
+
+	return r, nil
+}

keyring/keyring.go

@@ -78,8 +78,8 @@ func NewKeyPair(root, name, email string) error {
 
 // ListKeys prints keyring information to w.
 func ListKeys(root string, w io.Writer) error {
-	if err := checkDir(root); err != nil {
-		return errors.Wrap(err, "can't find pgp dir")
+	if err := ensureDir(root); err != nil {
+		return errors.Wrap(err, "can't find or create pgp dir")
 	}
 	srn, prn := getNames(root)
 	secs, pubs, err := getELs(srn, prn)
@@ -105,8 +105,8 @@ func ListKeys(root string, w io.Writer) error {
 
 // Export prints pubkey information associated with email to w.
 func Export(root string, w io.Writer, email string) error {
-	if err := checkDir(root); err != nil {
-		return errors.Wrap(err, "can't find pgp dir")
+	if err := ensureDir(root); err != nil {
+		return errors.Wrap(err, "can't find or create pgp dir")
 	}
 	srn, prn := getNames(root)
 	_, pubs, err := getELs(srn, prn)
@@ -142,8 +142,8 @@ func Import(root string, w io.Reader) error {
 		return errors.Wrap(err, "reading keyring")
 	}
 
-	if err := checkDir(root); err != nil {
-		return errors.Wrap(err, "can't find pgp dir")
+	if err := ensureDir(root); err != nil {
+		return errors.Wrap(err, "can't find or create pgp dir")
 	}
 	srn, prn := getNames(root)
 	_, pubs, err := getELs(srn, prn)
@@ -184,20 +184,8 @@ func Import(root string, w io.Reader) error {
 }
 
 // Sign takes an id and a reader and writes the signature for that id to sig.
-func Sign(root, id string, in io.Reader, sig io.Writer) error {
-	if err := checkDir(root); err != nil {
-		return errors.Wrap(err, "can't find pgp dir")
-	}
-	srn, prn := getNames(root)
-	secs, _, err := getELs(srn, prn)
-	if err != nil {
-		return errors.Wrap(err, "getting existing keyrings")
-	}
-	e, err := findKey(secs, id)
-	if err != nil {
-		return errors.Wrapf(err, "finding key %q", id)
-	}
-	if err := openpgp.ArmoredDetachSign(sig, e, in, nil); err != nil {
+func Sign(key *openpgp.Entity, in io.Reader, sig io.Writer) error {
+	if err := openpgp.ArmoredDetachSign(sig, key, in, nil); err != nil {
 		return errors.Wrap(err, "armored detach sign")
 	}
 	fmt.Fprintf(sig, "\n")
@@ -206,8 +194,8 @@ func Sign(root, id string, in io.Reader, sig io.Writer) error {
 
 // Verify verifies a file's deatched signature.
 func Verify(root string, file, sig io.Reader) error {
-	if err := checkDir(root); err != nil {
-		return errors.Wrap(err, "can't find pgp dir")
+	if err := ensureDir(root); err != nil {
+		return errors.Wrap(err, "can't find or create pgp dir")
 	}
 	srn, prn := getNames(root)
 	_, pubs, err := getELs(srn, prn)
@@ -225,8 +213,8 @@ func Verify(root string, file, sig io.Reader) error {
 // It skips public keys that have matching secret keys, and does not effect
 // private keys.
 func Remove(root string, id string) error {
-	if err := checkDir(root); err != nil {
-		return errors.Wrap(err, "can't find pgp dir")
+	if err := ensureDir(root); err != nil {
+		return errors.Wrap(err, "can't find or create pgp dir")
 	}
 	srn, prn := getNames(root)
 	secs, pubs, err := getELs(srn, prn)
@@ -266,14 +254,6 @@ func pGPDir(root string) string {
 	return filepath.Join(root, "var", "lib", "pm", "pgp")
 }
 
-func checkDir(root string) error {
-	d := pGPDir(root)
-	if !fs.Exists(d) {
-		return fmt.Errorf("pgp dir does not exist")
-	}
-	return nil
-}
-
 func ensureDir(root string) error {
 	d := pGPDir(root)
 	if !fs.Exists(d) {
@@ -348,3 +328,16 @@ func findKey(el openpgp.EntityList, id string) (*openpgp.Entity, error) {
 	}
 	return e, fmt.Errorf("key %q not found", id)
 }
+
+// FindSecretEntity searches for id in the secret keyring.
+func FindSecretEntity(root, id string) (*openpgp.Entity, error) {
+	if err := ensureDir(root); err != nil {
+		return nil, errors.Wrap(err, "can't find or create pgp dir")
+	}
+	srn, prn := getNames(root)
+	secs, _, err := getELs(srn, prn)
+	if err != nil {
+		return nil, errors.Wrap(err, "getting existing keyrings")
+	}
+	return findKey(secs, id)
+}

keyring/keyring_test.go

@@ -1,90 +0,0 @@
-package keyring
-
-import (
-	"bytes"
-	"io/ioutil"
-	"os"
-	"regexp"
-	"testing"
-)
-
-const (
-	ExpectedListKeysRegex = "sec: [[:alnum:]]{8}:[[:space:]]*foo[[:space:]]*\\(pm\\)[[:space:]]*<bar>\npub: [[:alnum:]]{8}:[[:space:]]*foo[[:space:]]*\\(pm\\)[[:space:]]*<bar>"
-)
-
-func TestNewKeyPair(t *testing.T) {
-	tmpdir, err := ioutil.TempDir("", "")
-	if err != nil {
-		t.Fatalf("Couldn't create tmpdir")
-	}
-	defer os.RemoveAll(tmpdir)
-
-	var newKeyPair = []struct {
-		r, n, e  string
-		expected bool
-	}{
-		{"", "", "", false},
-		{"", "foo", "", false},
-		{"", "foo", "bad<>email", false},
-		{tmpdir, "foo", "bar", true},
-	}
-	for _, rt := range newKeyPair {
-		actual := NewKeyPair(rt.r, rt.n, rt.e)
-		if (actual == nil) != rt.expected {
-			t.Errorf(
-				"failed NewKeyPair with an error: %v\n\troot: %s\n\tname: %s\n\temail:%s\n\texpected: %t\n\t  actual: %t",
-				actual,
-				rt.r,
-				rt.n,
-				rt.e,
-				(actual == nil),
-				rt.expected,
-			)
-		}
-	}
-}
-
-func TestListKeys(t *testing.T) {
-	tmpdir, err := ioutil.TempDir("", "")
-	if err != nil {
-		t.Fatalf("Couldn't create tmpdir")
-	}
-	defer os.RemoveAll(tmpdir)
-
-	err = NewKeyPair(tmpdir, "foo", "bar")
-	if err != nil {
-		t.Fatalf("Couldn't create New Key Pair")
-	}
-
-	var newKeyPair = []struct {
-		r        string
-		regex    string
-		w        *bytes.Buffer
-		expected bool
-	}{
-		{"", "", bytes.NewBuffer(nil), false},
-		{tmpdir, ExpectedListKeysRegex, bytes.NewBuffer(nil), true},
-	}
-	for _, rt := range newKeyPair {
-		actual := ListKeys(rt.r, rt.w)
-		if (actual == nil) != rt.expected {
-			t.Errorf(
-				"failed ListKeys with an error: %v\n\troot: %s\n\texpected: %t\n\t  actual: %t",
-				actual,
-				rt.r,
-				(actual == nil),
-				rt.expected,
-			)
-		}
-		if rt.expected {
-			matched, err := regexp.MatchString(rt.regex, rt.w.String())
-			if err != nil {
-				t.Fatalf("error%v, ", err)
-			}
-			if !matched {
-				t.Errorf("did not match")
-				t.Errorf("ListKeys did not match expected regex; wanted: [%q], got: [%s]", rt.regex, rt.w.String())
-			}
-		}
-	}
-}

meta.go

@@ -0,0 +1,47 @@
+package pm
+
+import (
+	"errors"
+	"fmt"
+	"net/url"
+)
+
+// Meta tracks metadata for a package
+type Meta struct {
+	Name        Name    `json:"name"`
+	Version     Version `json:"version"`
+	Description string  `json:"description"`
+
+	Remote url.URL `json:"remote"`
+}
+
+// Valid validates the contents of a Meta for requires fields.
+func (m Meta) Valid() (bool, error) {
+	if m.Name == "" {
+		return false, errors.New("name cannot be empty")
+	}
+	if m.Version == "" {
+		return false, errors.New("version cannot be empty")
+	}
+	if m.Description == "" {
+		return false, errors.New("description cannot be empty")
+	}
+	return true, nil
+}
+
+// Pkg returns the string name the .pkg should have on disk.
+func (m Meta) Pkg() string {
+	return fmt.Sprintf("%s-%s.pkg", m.Name, m.Version)
+}
+
+// URL returns the http location of this package.
+func (m Meta) URL() string {
+	return fmt.Sprintf("%s/%s", m.Remote.String(), m.Pkg())
+}
+
+func (m Meta) String() string {
+	return m.URL()
+}
+
+// Metas is a slice of Meta
+type Metas []Meta

meta_test.go

@@ -0,0 +1,85 @@
+package pm
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"testing"
+)
+
+func TestValid(t *testing.T) {
+	tests := []struct {
+		label string
+		m     Meta
+		ok    bool
+		err   error
+	}{
+		{
+			label: "valid",
+			m: Meta{
+				Name:        "heat",
+				Version:     "1.1.0",
+				Description: "some description",
+			},
+			ok: true,
+		},
+		{
+			label: "missing name",
+			m: Meta{
+				Version:     "1.1.0",
+				Description: "some description",
+			},
+			err: errors.New("name"),
+		},
+		{
+			label: "missing version",
+			m: Meta{
+				Name:        "heat",
+				Description: "some description",
+			},
+			err: errors.New("version"),
+		},
+		{
+			label: "missing description",
+			m: Meta{
+				Name:    "heat",
+				Version: "1.1.0",
+			},
+			err: errors.New("description"),
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.label, func(t *testing.T) {
+			ok, err := test.m.Valid()
+			if got, want := ok, test.ok; got != want {
+				t.Fatalf("validity: got %v, want %v", got, want)
+			}
+
+			if got, want := err, test.err; (err == nil) != (test.err == nil) {
+				t.Fatalf("error: got %v, want %v", got, want)
+			}
+		})
+	}
+}
+
+func TestJsonRoundTrip(t *testing.T) {
+	b := Meta{
+		Name:        "heat",
+		Version:     "1.1.0",
+		Description: "make heat using cpus",
+	}
+
+	buf := &bytes.Buffer{}
+	if err := json.NewEncoder(buf).Encode(&b); err != nil {
+		t.Fatalf("encode: %v", err)
+	}
+
+	a := Meta{}
+	if err := json.NewDecoder(buf).Decode(&a); err != nil {
+		t.Fatalf("decode: %v", err)
+	}
+	if b != a {
+		t.Fatalf("a != b: %v != %v", a, b)
+	}
+}

pkg/install.go

@@ -0,0 +1,359 @@
+package pkg
+
+import (
+	"archive/tar"
+	"bufio"
+	"compress/bzip2"
+	"crypto/sha256"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+
+	"github.com/pkg/errors"
+	"mcquay.me/fs"
+	"mcquay.me/pm"
+	"mcquay.me/pm/db"
+	"mcquay.me/pm/keyring"
+)
+
+const cache = "var/cache/pm"
+const installed = "var/lib/pm/installed"
+
+// Install fetches and installs pkgs from appropriate remotes.
+func Install(root string, pkgs []string) error {
+	av, err := db.LoadAvailable(root)
+	if err != nil {
+		return errors.Wrap(err, "loading available db")
+	}
+
+	ms, err := av.Installable(pkgs)
+	if err != nil {
+		return errors.Wrap(err, "checking ability to install")
+	}
+
+	cacheDir := filepath.Join(root, cache)
+	if !fs.Exists(cacheDir) {
+		if err := os.MkdirAll(cacheDir, 0755); err != nil {
+			return errors.Wrap(err, "creating non-existent cache dir")
+		}
+	}
+	if !fs.IsDir(cacheDir) {
+		return errors.Errorf("%q is not a directory!", cacheDir)
+	}
+	installedDir := filepath.Join(root, installed)
+	if !fs.Exists(installedDir) {
+		if err := os.MkdirAll(installedDir, 0755); err != nil {
+			return errors.Wrap(err, "creating non-existent cache dir")
+		}
+	}
+	if !fs.IsDir(cacheDir) {
+		return errors.Errorf("%q is not a directory!", cacheDir)
+	}
+
+	if err := download(cacheDir, ms); err != nil {
+		return errors.Wrap(err, "downloading")
+	}
+
+	for _, m := range ms {
+		if err := install(root, m); err != nil {
+			return errors.Wrapf(err, "installing %v", m.Name)
+		}
+	}
+	return nil
+}
+
+func download(cache string, ms pm.Metas) error {
+	// TODO (sm): concurrently fetch
+	for _, m := range ms {
+		resp, err := http.Get(m.URL())
+		if err != nil {
+			return errors.Wrap(err, "http get")
+		}
+		fn := filepath.Join(cache, m.Pkg())
+		f, err := os.Create(fn)
+		if err != nil {
+			return errors.Wrap(err, "creating")
+		}
+
+		if n, err := io.Copy(f, resp.Body); err != nil {
+			return errors.Wrapf(err, "copy %q to disk after %d bytes", m.URL(), n)
+		}
+
+		if err := resp.Body.Close(); err != nil {
+			return errors.Wrap(err, "closing resp body")
+		}
+	}
+	return nil
+}
+
+func verifyManifestIntegrity(root string, m pm.Meta) error {
+	pn := filepath.Join(root, cache, m.Pkg())
+	man, err := getReadCloser(pn, "manifest.sha256")
+	if err != nil {
+		return errors.Wrap(err, "getting manifest reader")
+	}
+	sig, err := getReadCloser(pn, "manifest.sha256.asc")
+	if err != nil {
+		return errors.Wrap(err, "getting manifest reader")
+	}
+
+	if err := keyring.Verify(root, man, sig); err != nil {
+		return errors.Wrap(err, "verifying manifest")
+	}
+	if err := man.Close(); err != nil {
+		return errors.Wrap(err, "closing manifest reader")
+	}
+	if err := sig.Close(); err != nil {
+		return errors.Wrap(err, "closing manifest signature reader")
+	}
+	return nil
+}
+
+func expandPkgContents(root string, m pm.Meta) error {
+	pn := filepath.Join(root, cache, m.Pkg())
+	man, err := getReadCloser(pn, "manifest.sha256")
+	if err != nil {
+		return errors.Wrap(err, "getting manifest reader")
+	}
+
+	ip := filepath.Join(root, installed, string(m.Name))
+	if err := os.MkdirAll(ip, 0755); err != nil {
+		return errors.Wrapf(err, "making install dir for %q", m.Name)
+	}
+
+	cs := map[string]string{}
+	s := bufio.NewScanner(man)
+	for s.Scan() {
+		elems := strings.Split(s.Text(), "\t")
+		if len(elems) != 2 {
+			return errors.Errorf("manifest format error; got %d elements, want 2", len(elems))
+		}
+		cs[elems[1]] = elems[0]
+	}
+	if err := man.Close(); err != nil {
+		return errors.Wrap(err, "closing manifest reader")
+	}
+	if err := s.Err(); err != nil {
+		return errors.Wrap(err, "scanning manifest")
+	}
+
+	pf, err := os.Open(pn)
+	if err != nil {
+		return errors.Wrap(err, "opening pkg file")
+	}
+	tr := tar.NewReader(pf)
+	for {
+		hdr, err := tr.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return errors.Wrap(err, "tar traversal")
+		}
+
+		if hdr.Name == "manifest.sha256" || hdr.Name == "manifest.sha256.asc" {
+			continue
+		}
+
+		if hdr.FileInfo().IsDir() {
+			if hdr.Name != "bin" {
+				return errors.Errorf("%v is unexpected", hdr.Name)
+			}
+			if err := os.MkdirAll(filepath.Join(ip, hdr.Name), hdr.FileInfo().Mode()); err != nil {
+				return errors.Wrapf(err, "mkdir for %v", hdr.Name)
+			}
+			continue
+		}
+
+		sha, ok := cs[hdr.Name]
+		if !ok {
+			return errors.Errorf("extra file %q found in tarfile!", hdr.Name)
+		}
+
+		name := filepath.Join(ip, hdr.Name)
+		sr := sha256.New()
+		var o io.WriteCloser
+		o = close{ioutil.Discard}
+		if hdr.Name != "root.tar.bz2" {
+			f, err := os.OpenFile(filepath.Join(ip, hdr.Name), os.O_WRONLY|os.O_CREATE, hdr.FileInfo().Mode())
+			if err != nil {
+				return errors.Wrap(err, "open file in install dir")
+			}
+			o = f
+		}
+
+		w := io.MultiWriter(o, sr)
+
+		if n, err := io.Copy(w, tr); err != nil {
+			return errors.Wrapf(err, "copying file %q after %v bytes", hdr.Name, n)
+		}
+
+		if sha != fmt.Sprintf("%x", sr.Sum(nil)) {
+			return errors.Errorf("%q checksum was incorrect", hdr.Name)
+		}
+
+		if err := o.Close(); err != nil {
+			return errors.Wrapf(err, "closing %v", name)
+		}
+	}
+	return nil
+}
+
+type tarSlurper struct {
+	f  *os.File
+	tr *tar.Reader
+}
+
+func (ts *tarSlurper) Close() error {
+	return ts.f.Close()
+}
+
+func (ts *tarSlurper) Read(p []byte) (int, error) {
+	return ts.tr.Read(p)
+}
+
+func getReadCloser(tn, fn string) (io.ReadCloser, error) {
+	pf, err := os.Open(tn)
+	if err != nil {
+		return nil, errors.Wrap(err, "opening pkg file")
+	}
+	tr := tar.NewReader(pf)
+	for {
+		hdr, err := tr.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return nil, errors.Wrap(err, "tar traversal")
+		}
+		if hdr.Name == fn {
+			return &tarSlurper{pf, tr}, nil
+		}
+	}
+	return nil, errors.Errorf("%q not found", fn)
+}
+
+// close should be used to wrap ioutil.Discard to give it a noop Close method.
+type close struct {
+	io.Writer
+}
+
+func (close) Close() error {
+	return nil
+}
+
+func script(root string, m pm.Meta, name string) error {
+	bin := filepath.Join(root, installed, string(m.Name), "bin", name)
+	if !fs.Exists(bin) {
+		return nil
+	}
+	cmd := exec.Command(bin)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	return cmd.Run()
+}
+
+func expandRoot(root string, m pm.Meta) error {
+
+	bomn := filepath.Join(root, installed, string(m.Name), "bom.sha256")
+	bf, err := os.Open(bomn)
+	if err != nil {
+		return errors.Wrap(err, "opening bom")
+	}
+
+	cs := map[string]string{}
+	s := bufio.NewScanner(bf)
+	for s.Scan() {
+		elems := strings.Split(s.Text(), "\t")
+		if len(elems) != 2 {
+			return errors.Errorf("manifest format error; got %d elements, want 2", len(elems))
+		}
+		cs[elems[1]] = elems[0]
+	}
+
+	pn := filepath.Join(root, cache, m.Pkg())
+	tbz, err := getReadCloser(pn, "root.tar.bz2")
+	if err != nil {
+		return errors.Wrap(err, "getting root.tar.bz2 reader")
+	}
+	tr := tar.NewReader(bzip2.NewReader(tbz))
+	for {
+		hdr, err := tr.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return errors.Wrap(err, "tar traversal")
+		}
+		if hdr.FileInfo().IsDir() {
+			d := filepath.Join(root, hdr.Name)
+			if err := os.MkdirAll(d, hdr.FileInfo().Mode()); err != nil {
+				return errors.Wrapf(err, "making directory %q", d)
+			}
+			continue
+		}
+		f, err := os.OpenFile(filepath.Join(root, hdr.Name), os.O_WRONLY|os.O_CREATE, hdr.FileInfo().Mode())
+		if err != nil {
+			return errors.Wrapf(err, "open output file %q", hdr.Name)
+		}
+		if n, err := io.Copy(f, tr); err != nil {
+			return errors.Wrapf(err, "copy file %q after %v bytes", hdr.Name, n)
+		}
+		if err := f.Close(); err != nil {
+			return errors.Wrapf(err, "closing %q", hdr.Name)
+		}
+	}
+	return nil
+}
+
+func install(root string, m pm.Meta) error {
+	defer func() {
+		cached := filepath.Join(root, cache, m.Pkg())
+		if !fs.Exists(cached) {
+			return
+		}
+		if err := os.Remove(cached); err != nil {
+			log.Printf("cleaning up cache: %v", err)
+		}
+	}()
+
+	already, err := db.IsInstalled(root, m)
+	if err != nil {
+		return errors.Wrapf(err, "is installed %v", m.Name)
+	}
+	if already {
+		return errors.Errorf("%v already installed!", m.Name)
+	}
+	if err := verifyManifestIntegrity(root, m); err != nil {
+		return errors.Wrap(err, "verifying pkg integrity")
+	}
+	if err := expandPkgContents(root, m); err != nil {
+		if err := os.RemoveAll(filepath.Join(root, installed, string(m.Name))); err != nil {
+			err = errors.Wrap(err, "cleaning up")
+		}
+		return errors.Wrap(err, "verifying pkg contents")
+	}
+
+	if err := script(root, m, "pre-install"); err != nil {
+		return errors.Wrap(err, "pre-install")
+	}
+
+	if err := expandRoot(root, m); err != nil {
+		return errors.Wrap(err, "root expansion")
+	}
+
+	if err := script(root, m, "post-install"); err != nil {
+		return errors.Wrap(err, "pre-install")
+	}
+
+	if err := db.AddInstalled(root, m); err != nil {
+		return errors.Wrapf(err, "adding ", m.Name)
+	}
+	return nil
+}

pkg/pkg.go

@@ -0,0 +1,243 @@
+package pkg
+
+import (
+	"archive/tar"
+	"compress/bzip2"
+	"crypto/sha256"
+	"fmt"
+	"io"
+	"log"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+
+	"github.com/pkg/errors"
+	"golang.org/x/crypto/openpgp"
+	yaml "gopkg.in/yaml.v2"
+
+	"mcquay.me/fs"
+	"mcquay.me/pm"
+
+	"mcquay.me/pm/keyring"
+)
+
+var validNames map[string]bool
+var crypto []string
+
+func init() {
+	validNames = map[string]bool{
+		"root.tar.bz2":     true,
+		"meta.yaml":        true,
+		"bin/pre-install":  true,
+		"bin/post-install": true,
+		"bin/pre-upgrade":  true,
+		"bin/post-upgrade": true,
+		"bin/pre-remove":   true,
+		"bin/post-remove":  true,
+	}
+
+	crypto = []string{
+		"bom.sha256",
+		"manifest.sha256",
+		"manifest.sha256.asc",
+	}
+}
+
+// Create traverses the contents of dir and emits a valid pkg, signed by id
+func Create(key *openpgp.Entity, dir string) error {
+	if !fs.Exists(dir) {
+		return fmt.Errorf("%q: doesn't exist", dir)
+	}
+	if !fs.IsDir(dir) {
+		return fmt.Errorf("%q: is not a directory", dir)
+	}
+
+	if err := clean(dir); err != nil {
+		return errors.Wrap(err, "cleaning up directory")
+	}
+
+	files := []string{}
+	found := map[string]bool{}
+	err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+		if path == dir {
+			return nil
+		}
+		if info.IsDir() {
+			return nil
+		}
+		p := strings.TrimPrefix(path, dir)[1:]
+
+		if _, ok := validNames[p]; !ok {
+			return fmt.Errorf("unxpected filename: %q", p)
+		}
+
+		if strings.HasPrefix(p, "bin") && info.Mode()&0100 == 0 {
+			return fmt.Errorf("%q is not executable", path)
+		}
+		files = append(files, p)
+		found[p] = true
+		return nil
+	})
+	if err != nil {
+		return errors.Wrap(err, "checking directory contents")
+	}
+
+	if _, ok := found["root.tar.bz2"]; !ok {
+		return fmt.Errorf("did not find root.tar.bz2")
+	}
+	if _, ok := found["meta.yaml"]; !ok {
+		return fmt.Errorf("did not find meta.yaml")
+	}
+
+	mf, err := os.Open(filepath.Join(dir, "meta.yaml"))
+	if err != nil {
+		return errors.Wrap(err, "opening metadata file")
+	}
+	md := pm.Meta{}
+	if err := yaml.NewDecoder(mf).Decode(&md); err != nil {
+		return errors.Wrap(err, "decoding metadata file")
+	}
+	if _, err := md.Valid(); err != nil {
+		return errors.Wrap(err, "invalid metadata")
+	}
+
+	f, err := os.Open(filepath.Join(dir, "root.tar.bz2"))
+	if err != nil {
+		return errors.Wrap(err, "opening overlay tarball")
+	}
+
+	bom, err := os.Create(filepath.Join(dir, "bom.sha256"))
+	if err != nil {
+		return errors.Wrap(err, "creating bom.sha256")
+	}
+	tr := tar.NewReader(bzip2.NewReader(f))
+	for {
+		hdr, err := tr.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			errors.Wrap(err, "traversing tarball")
+		}
+		if hdr.FileInfo().IsDir() {
+			continue
+		}
+		s := sha256.New()
+		if c, err := io.Copy(s, tr); err != nil {
+			return errors.Wrapf(err, "copy after %d bytes", c)
+		}
+		fmt.Fprintf(bom, "%x\t%s\n", s.Sum(nil), hdr.Name)
+	}
+	if err := bom.Close(); err != nil {
+		return errors.Wrap(err, "closing bom")
+	}
+
+	files = append(files, "bom.sha256")
+	sort.Strings(files)
+
+	manifest, err := os.Create(filepath.Join(dir, "manifest.sha256"))
+	if err != nil {
+		return errors.Wrap(err, "creating manifest.sha256")
+	}
+	for _, fn := range files {
+		full := filepath.Join(dir, fn)
+		f, err := os.Open(full)
+		if err != nil {
+			return errors.Wrap(err, "opening file for manifest checksumming")
+		}
+		s := sha256.New()
+		if c, err := io.Copy(s, f); err != nil {
+			return errors.Wrapf(err, "copy after %d bytes", c)
+		}
+		if err := f.Close(); err != nil {
+			return errors.Wrapf(err, "closing %q", f.Name())
+		}
+		fmt.Fprintf(manifest, "%x\t%s\n", s.Sum(nil), fn)
+	}
+	if err := manifest.Close(); err != nil {
+		return errors.Wrap(err, "closing manifest")
+	}
+
+	sig, err := os.Create(filepath.Join(dir, "manifest.sha256.asc"))
+	if err != nil {
+		return errors.Wrap(err, "creating sig file")
+	}
+	mfi, err := os.Open(filepath.Join(dir, "manifest.sha256"))
+	if err != nil {
+		return errors.Wrap(err, "opening manifest")
+	}
+	if err := keyring.Sign(key, mfi, sig); err != nil {
+		return errors.Wrap(err, "signing")
+	}
+
+	files = append(files, "manifest.sha256")
+	files = append(files, "manifest.sha256.asc")
+	sort.Strings(files)
+
+	tn, pn := filepath.Split(dir)
+	tn = filepath.Join(tn, fmt.Sprintf("%v-%v.pkg", pn, md.Version))
+	tf, err := os.Create(tn)
+	if err != nil {
+		return errors.Wrap(err, "opening final .pkg")
+	}
+
+	tw := tar.NewWriter(tf)
+	err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+		if path == dir {
+			return nil
+		}
+		p := strings.TrimPrefix(path, dir)[1:]
+		hdr, err := tar.FileInfoHeader(info, "")
+		if err != nil {
+			return errors.Wrap(err, "file info header")
+		}
+		hdr.Name = p
+		if err := tw.WriteHeader(hdr); err != nil {
+			return errors.Wrapf(err, "writing tar header for %v", p)
+		}
+
+		// only need to do real writing for non-directories
+		if info.IsDir() {
+			return nil
+		}
+
+		f, err := os.Open(path)
+		if err != nil {
+			return errors.Wrap(err, "opening file for tar creation")
+		}
+		if c, err := io.Copy(tw, f); err != nil {
+			log.Printf("%+v", err == io.EOF)
+			return errors.Wrapf(err, "copy after %d bytes", c)
+		}
+		if err := f.Close(); err != nil {
+			return errors.Wrap(err, "closing file during tar creation")
+		}
+		return nil
+	})
+	if err != nil {
+		return errors.Wrap(err, "traversing the prepared directory")
+	}
+	if err := tw.Close(); err != nil {
+		return errors.Wrap(err, "closing tar writer")
+	}
+
+	if err := tf.Close(); err != nil {
+		return errors.Wrap(err, "closing final .pkg")
+	}
+
+	return nil
+}
+
+func clean(root string) error {
+	for _, f := range crypto {
+		path := filepath.Join(root, f)
+		if !fs.Exists(path) {
+			continue
+		}
+		if err := os.Remove(path); err != nil {
+			return errors.Wrapf(err, "removing %q", path)
+		}
+	}
+	return nil
+}

pkg/remove.go

@@ -0,0 +1,61 @@
+package pkg
+
+import (
+	"os"
+	"path/filepath"
+
+	"github.com/pkg/errors"
+	"mcquay.me/pm"
+	"mcquay.me/pm/db"
+)
+
+// Remove uninstalls packages.
+func Remove(root string, pkgs []string) error {
+	iDB, err := db.LoadInstalled(root)
+	if err != nil {
+		return errors.Wrap(err, "loading available db")
+	}
+
+	ms, err := iDB.Removable(pkgs)
+	if err != nil {
+		return errors.Wrap(err, "checking ability to remove")
+	}
+
+	for _, m := range ms {
+		if err := script(root, m, "pre-remove"); err != nil {
+			return errors.Wrap(err, "pre-remove")
+		}
+
+		mdir := filepath.Join(root, installed, string(m.Name))
+		bom := filepath.Join(mdir, "bom.sha256")
+		bf, err := os.Open(bom)
+		if err != nil {
+			return errors.Wrapf(err, "%q: opening bom", m.Name)
+		}
+
+		cs, err := pm.ParseCS(bf)
+		if err != nil {
+			return errors.Wrapf(err, "%q: parsing bom", m.Name)
+		}
+
+		for n := range cs {
+			if err := os.Remove(filepath.Join(root, n)); err != nil {
+				return errors.Wrapf(err, "pkg %q", m.Name)
+			}
+		}
+
+		if err := script(root, m, "post-remove"); err != nil {
+			return errors.Wrap(err, "post-remove")
+		}
+
+		if err := db.RemoveInstalled(root, m); err != nil {
+			return errors.Wrapf(err, "removing %q", m.Name)
+		}
+
+		if err := os.RemoveAll(mdir); err != nil {
+			return errors.Wrapf(err, "%q: removing pm install dir", m.Name)
+		}
+	}
+
+	return nil
+}