From 77570c98867cf5f2971a8de4992e31182b621df1 Mon Sep 17 00:00:00 2001
From: Derek McGowan <derek@mcgstyle.net>
Date: Thu, 12 Mar 2015 19:24:35 -0700
Subject: [PATCH 01/10] Add digest set implementation

Set represents a unique set of digests which allow for efficient lookup.
Dumping short codes is a function which takes in a digest set.
Any operation involving short codes may be considered secure if the list of digests added to the set is the complete list of referenceable digests.
Contains benchmarks for Add, Lookup, and Dump.

Signed-off-by: Derek McGowan <derek@mcgstyle.net>
---
 digestset/set.go      | 195 ++++++++++++++++++++++++++++++
 digestset/set_test.go | 272 ++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 467 insertions(+)
 create mode 100644 digestset/set.go
 create mode 100644 digestset/set_test.go

diff --git a/digestset/set.go b/digestset/set.go
new file mode 100644
index 0000000..7a9a038
--- /dev/null
+++ b/digestset/set.go
@@ -0,0 +1,195 @@
+package digest
+
+import (
+	"errors"
+	"sort"
+	"strings"
+)
+
+var (
+	// ErrDigestNotFound is used when a matching digest
+	// could not be found in a set.
+	ErrDigestNotFound = errors.New("digest not found")
+
+	// ErrDigestAmbiguous is used when multiple digests
+	// are found in a set. None of the matching digests
+	// should be considered valid matches.
+	ErrDigestAmbiguous = errors.New("ambiguous digest string")
+)
+
+// Set is used to hold a unique set of digests which
+// may be easily referenced by easily  referenced by a string
+// representation of the digest as well as short representation.
+// The uniqueness of the short representation is based on other
+// digests in the set. If digests are ommited from this set,
+// collisions in a larger set may not be detected, therefore it
+// is important to always do short representation lookups on
+// the complete set of digests. To mitigate collisions, an
+// appropriately long short code should be used.
+type Set struct {
+	entries digestEntries
+}
+
+// NewSet creates an empty set of digests
+// which may have digests added.
+func NewSet() *Set {
+	return &Set{
+		entries: digestEntries{},
+	}
+}
+
+// checkShortMatch checks whether two digests match as either whole
+// values or short values. This function does not test equality,
+// rather whether the second value could match against the first
+// value.
+func checkShortMatch(alg, hex, shortAlg, shortHex string) bool {
+	if len(hex) == len(shortHex) {
+		if hex != shortHex {
+			return false
+		}
+		if len(shortAlg) > 0 && alg != shortAlg {
+			return false
+		}
+	} else if !strings.HasPrefix(hex, shortHex) {
+		return false
+	} else if len(shortAlg) > 0 && alg != shortAlg {
+		return false
+	}
+	return true
+}
+
+// Lookup looks for a digest matching the given string representation.
+// If no digests could be found ErrDigestNotFound will be returned
+// with an empty digest value. If multiple matches are found
+// ErrDigestAmbiguous will be returned with an empty digest value.
+func (dst *Set) Lookup(d string) (Digest, error) {
+	if len(dst.entries) == 0 {
+		return "", ErrDigestNotFound
+	}
+	var (
+		searchFunc func(int) bool
+		alg        string
+		hex        string
+	)
+	dgst, err := ParseDigest(d)
+	if err == ErrDigestInvalidFormat {
+		hex = d
+		searchFunc = func(i int) bool {
+			return dst.entries[i].val >= d
+		}
+	} else {
+		hex = dgst.Hex()
+		alg = dgst.Algorithm()
+		searchFunc = func(i int) bool {
+			if dst.entries[i].val == hex {
+				return dst.entries[i].alg >= alg
+			}
+			return dst.entries[i].val >= hex
+		}
+	}
+	idx := sort.Search(len(dst.entries), searchFunc)
+	if idx == len(dst.entries) || !checkShortMatch(dst.entries[idx].alg, dst.entries[idx].val, alg, hex) {
+		return "", ErrDigestNotFound
+	}
+	if dst.entries[idx].alg == alg && dst.entries[idx].val == hex {
+		return dst.entries[idx].digest, nil
+	}
+	if idx+1 < len(dst.entries) && checkShortMatch(dst.entries[idx+1].alg, dst.entries[idx+1].val, alg, hex) {
+		return "", ErrDigestAmbiguous
+	}
+
+	return dst.entries[idx].digest, nil
+}
+
+// Add adds the given digests to the set. An error will be returned
+// if the given digest is invalid. If the digest already exists in the
+// table, this operation will be a no-op.
+func (dst *Set) Add(d Digest) error {
+	if err := d.Validate(); err != nil {
+		return err
+	}
+	entry := &digestEntry{alg: d.Algorithm(), val: d.Hex(), digest: d}
+	searchFunc := func(i int) bool {
+		if dst.entries[i].val == entry.val {
+			return dst.entries[i].alg >= entry.alg
+		}
+		return dst.entries[i].val >= entry.val
+	}
+	idx := sort.Search(len(dst.entries), searchFunc)
+	if idx == len(dst.entries) {
+		dst.entries = append(dst.entries, entry)
+		return nil
+	} else if dst.entries[idx].digest == d {
+		return nil
+	}
+
+	entries := append(dst.entries, nil)
+	copy(entries[idx+1:], entries[idx:len(entries)-1])
+	entries[idx] = entry
+	dst.entries = entries
+	return nil
+}
+
+// ShortCodeTable returns a map of Digest to unique short codes. The
+// length represents the minimum value, the maximum length may be the
+// entire value of digest if uniqueness cannot be achieved without the
+// full value. This function will attempt to make short codes as short
+// as possible to be unique.
+func ShortCodeTable(dst *Set, length int) map[Digest]string {
+	m := make(map[Digest]string, len(dst.entries))
+	l := length
+	resetIdx := 0
+	for i := 0; i < len(dst.entries); i++ {
+		var short string
+		extended := true
+		for extended {
+			extended = false
+			if len(dst.entries[i].val) <= l {
+				short = dst.entries[i].digest.String()
+			} else {
+				short = dst.entries[i].val[:l]
+				for j := i + 1; j < len(dst.entries); j++ {
+					if checkShortMatch(dst.entries[j].alg, dst.entries[j].val, "", short) {
+						if j > resetIdx {
+							resetIdx = j
+						}
+						extended = true
+					} else {
+						break
+					}
+				}
+				if extended {
+					l++
+				}
+			}
+		}
+		m[dst.entries[i].digest] = short
+		if i >= resetIdx {
+			l = length
+		}
+	}
+	return m
+}
+
+type digestEntry struct {
+	alg    string
+	val    string
+	digest Digest
+}
+
+type digestEntries []*digestEntry
+
+func (d digestEntries) Len() int {
+	return len(d)
+}
+
+func (d digestEntries) Less(i, j int) bool {
+	if d[i].val != d[j].val {
+		return d[i].val < d[j].val
+	}
+	return d[i].alg < d[j].alg
+}
+
+func (d digestEntries) Swap(i, j int) {
+	d[i], d[j] = d[j], d[i]
+}
diff --git a/digestset/set_test.go b/digestset/set_test.go
new file mode 100644
index 0000000..faeba6d
--- /dev/null
+++ b/digestset/set_test.go
@@ -0,0 +1,272 @@
+package digest
+
+import (
+	"crypto/sha256"
+	"encoding/binary"
+	"math/rand"
+	"testing"
+)
+
+func assertEqualDigests(t *testing.T, d1, d2 Digest) {
+	if d1 != d2 {
+		t.Fatalf("Digests do not match:\n\tActual: %s\n\tExpected: %s", d1, d2)
+	}
+}
+
+func TestLookup(t *testing.T) {
+	digests := []Digest{
+		"sha256:12345",
+		"sha256:1234",
+		"sha256:12346",
+		"sha256:54321",
+		"sha256:65431",
+		"sha256:64321",
+		"sha256:65421",
+		"sha256:65321",
+	}
+
+	dset := NewSet()
+	for i := range digests {
+		if err := dset.Add(digests[i]); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	dgst, err := dset.Lookup("54")
+	if err != nil {
+		t.Fatal(err)
+	}
+	assertEqualDigests(t, dgst, digests[3])
+
+	dgst, err = dset.Lookup("1234")
+	if err == nil {
+		t.Fatal("Expected ambiguous error looking up: 1234")
+	}
+	if err != ErrDigestAmbiguous {
+		t.Fatal(err)
+	}
+
+	dgst, err = dset.Lookup("9876")
+	if err == nil {
+		t.Fatal("Expected ambiguous error looking up: 9876")
+	}
+	if err != ErrDigestNotFound {
+		t.Fatal(err)
+	}
+
+	dgst, err = dset.Lookup("sha256:1234")
+	if err != nil {
+		t.Fatal(err)
+	}
+	assertEqualDigests(t, dgst, digests[1])
+
+	dgst, err = dset.Lookup("sha256:12345")
+	if err != nil {
+		t.Fatal(err)
+	}
+	assertEqualDigests(t, dgst, digests[0])
+
+	dgst, err = dset.Lookup("sha256:12346")
+	if err != nil {
+		t.Fatal(err)
+	}
+	assertEqualDigests(t, dgst, digests[2])
+
+	dgst, err = dset.Lookup("12346")
+	if err != nil {
+		t.Fatal(err)
+	}
+	assertEqualDigests(t, dgst, digests[2])
+
+	dgst, err = dset.Lookup("12345")
+	if err != nil {
+		t.Fatal(err)
+	}
+	assertEqualDigests(t, dgst, digests[0])
+}
+
+func TestAddDuplication(t *testing.T) {
+	digests := []Digest{
+		"sha256:1234",
+		"sha256:12345",
+		"sha256:12346",
+		"sha256:54321",
+		"sha256:65431",
+		"sha512:65431",
+		"sha512:65421",
+		"sha512:65321",
+	}
+
+	dset := NewSet()
+	for i := range digests {
+		if err := dset.Add(digests[i]); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	if len(dset.entries) != 8 {
+		t.Fatal("Invalid dset size")
+	}
+
+	if err := dset.Add(Digest("sha256:12345")); err != nil {
+		t.Fatal(err)
+	}
+
+	if len(dset.entries) != 8 {
+		t.Fatal("Duplicate digest insert allowed")
+	}
+
+	if err := dset.Add(Digest("sha384:12345")); err != nil {
+		t.Fatal(err)
+	}
+
+	if len(dset.entries) != 9 {
+		t.Fatal("Insert with different algorithm not allowed")
+	}
+}
+
+func assertEqualShort(t *testing.T, actual, expected string) {
+	if actual != expected {
+		t.Fatalf("Unexpected short value:\n\tExpected: %s\n\tActual: %s", expected, actual)
+	}
+}
+
+func TestShortCodeTable(t *testing.T) {
+	digests := []Digest{
+		"sha256:1234",
+		"sha256:12345",
+		"sha256:12346",
+		"sha256:54321",
+		"sha256:65431",
+		"sha256:64321",
+		"sha256:65421",
+		"sha256:65321",
+	}
+
+	dset := NewSet()
+	for i := range digests {
+		if err := dset.Add(digests[i]); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	dump := ShortCodeTable(dset, 2)
+
+	if len(dump) < len(digests) {
+		t.Fatalf("Error unexpected size: %d, expecting %d", len(dump), len(digests))
+	}
+
+	assertEqualShort(t, dump[digests[0]], "sha256:1234")
+	assertEqualShort(t, dump[digests[1]], "sha256:12345")
+	assertEqualShort(t, dump[digests[2]], "sha256:12346")
+	assertEqualShort(t, dump[digests[3]], "54")
+	assertEqualShort(t, dump[digests[4]], "6543")
+	assertEqualShort(t, dump[digests[5]], "64")
+	assertEqualShort(t, dump[digests[6]], "6542")
+	assertEqualShort(t, dump[digests[7]], "653")
+}
+
+func createDigests(count int) ([]Digest, error) {
+	r := rand.New(rand.NewSource(25823))
+	digests := make([]Digest, count)
+	for i := range digests {
+		h := sha256.New()
+		if err := binary.Write(h, binary.BigEndian, r.Int63()); err != nil {
+			return nil, err
+		}
+		digests[i] = NewDigest("sha256", h)
+	}
+	return digests, nil
+}
+
+func benchAddNTable(b *testing.B, n int) {
+	digests, err := createDigests(n)
+	if err != nil {
+		b.Fatal(err)
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		dset := &Set{entries: digestEntries(make([]*digestEntry, 0, n))}
+		for j := range digests {
+			if err = dset.Add(digests[j]); err != nil {
+				b.Fatal(err)
+			}
+		}
+	}
+}
+
+func benchLookupNTable(b *testing.B, n int, shortLen int) {
+	digests, err := createDigests(n)
+	if err != nil {
+		b.Fatal(err)
+	}
+	dset := &Set{entries: digestEntries(make([]*digestEntry, 0, n))}
+	for i := range digests {
+		if err := dset.Add(digests[i]); err != nil {
+			b.Fatal(err)
+		}
+	}
+	shorts := make([]string, 0, n)
+	for _, short := range ShortCodeTable(dset, shortLen) {
+		shorts = append(shorts, short)
+	}
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		if _, err = dset.Lookup(shorts[i%n]); err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func benchShortCodeNTable(b *testing.B, n int, shortLen int) {
+	digests, err := createDigests(n)
+	if err != nil {
+		b.Fatal(err)
+	}
+	dset := &Set{entries: digestEntries(make([]*digestEntry, 0, n))}
+	for i := range digests {
+		if err := dset.Add(digests[i]); err != nil {
+			b.Fatal(err)
+		}
+	}
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		ShortCodeTable(dset, shortLen)
+	}
+}
+
+func BenchmarkAdd10(b *testing.B) {
+	benchAddNTable(b, 10)
+}
+
+func BenchmarkAdd100(b *testing.B) {
+	benchAddNTable(b, 100)
+}
+
+func BenchmarkAdd1000(b *testing.B) {
+	benchAddNTable(b, 1000)
+}
+
+func BenchmarkLookup10(b *testing.B) {
+	benchLookupNTable(b, 10, 12)
+}
+
+func BenchmarkLookup100(b *testing.B) {
+	benchLookupNTable(b, 100, 12)
+}
+
+func BenchmarkLookup1000(b *testing.B) {
+	benchLookupNTable(b, 1000, 12)
+}
+
+func BenchmarkShortCode10(b *testing.B) {
+	benchShortCodeNTable(b, 10, 12)
+}
+func BenchmarkShortCode100(b *testing.B) {
+	benchShortCodeNTable(b, 100, 12)
+}
+func BenchmarkShortCode1000(b *testing.B) {
+	benchShortCodeNTable(b, 1000, 12)
+}

From 96bf78c30a95b949a667fd0d9297bb417fe8653e Mon Sep 17 00:00:00 2001
From: Stephen J Day <stephen.day@docker.com>
Date: Thu, 21 May 2015 18:44:08 -0700
Subject: [PATCH 02/10] Refactor specification of supported digests

To make the definition of supported digests more clear, we have refactored the
digest package to have a special Algorithm type. This represents the digest's
prefix and we associated various supported hash implementations through
function calls.

Signed-off-by: Stephen J Day <stephen.day@docker.com>
---
 digestset/set.go | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/digestset/set.go b/digestset/set.go
index 7a9a038..271d35d 100644
--- a/digestset/set.go
+++ b/digestset/set.go
@@ -42,17 +42,17 @@ func NewSet() *Set {
 // values or short values. This function does not test equality,
 // rather whether the second value could match against the first
 // value.
-func checkShortMatch(alg, hex, shortAlg, shortHex string) bool {
+func checkShortMatch(alg Algorithm, hex, shortAlg, shortHex string) bool {
 	if len(hex) == len(shortHex) {
 		if hex != shortHex {
 			return false
 		}
-		if len(shortAlg) > 0 && alg != shortAlg {
+		if len(shortAlg) > 0 && string(alg) != shortAlg {
 			return false
 		}
 	} else if !strings.HasPrefix(hex, shortHex) {
 		return false
-	} else if len(shortAlg) > 0 && alg != shortAlg {
+	} else if len(shortAlg) > 0 && string(alg) != shortAlg {
 		return false
 	}
 	return true
@@ -68,7 +68,7 @@ func (dst *Set) Lookup(d string) (Digest, error) {
 	}
 	var (
 		searchFunc func(int) bool
-		alg        string
+		alg        Algorithm
 		hex        string
 	)
 	dgst, err := ParseDigest(d)
@@ -88,13 +88,13 @@ func (dst *Set) Lookup(d string) (Digest, error) {
 		}
 	}
 	idx := sort.Search(len(dst.entries), searchFunc)
-	if idx == len(dst.entries) || !checkShortMatch(dst.entries[idx].alg, dst.entries[idx].val, alg, hex) {
+	if idx == len(dst.entries) || !checkShortMatch(dst.entries[idx].alg, dst.entries[idx].val, string(alg), hex) {
 		return "", ErrDigestNotFound
 	}
 	if dst.entries[idx].alg == alg && dst.entries[idx].val == hex {
 		return dst.entries[idx].digest, nil
 	}
-	if idx+1 < len(dst.entries) && checkShortMatch(dst.entries[idx+1].alg, dst.entries[idx+1].val, alg, hex) {
+	if idx+1 < len(dst.entries) && checkShortMatch(dst.entries[idx+1].alg, dst.entries[idx+1].val, string(alg), hex) {
 		return "", ErrDigestAmbiguous
 	}
 
@@ -172,7 +172,7 @@ func ShortCodeTable(dst *Set, length int) map[Digest]string {
 }
 
 type digestEntry struct {
-	alg    string
+	alg    Algorithm
 	val    string
 	digest Digest
 }

From 45599b9e08217367d71cf5830d2cc02a85cca0bc Mon Sep 17 00:00:00 2001
From: Derek McGowan <derek@mcgstyle.net>
Date: Tue, 29 Sep 2015 10:04:15 -0700
Subject: [PATCH 03/10] Add remove and list functions to digest set

Add mutex protection around set access

Signed-off-by: Derek McGowan <derek@mcgstyle.net>
---
 digestset/set.go      | 54 +++++++++++++++++++++++-
 digestset/set_test.go | 95 +++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 147 insertions(+), 2 deletions(-)

diff --git a/digestset/set.go b/digestset/set.go
index 271d35d..3fac41b 100644
--- a/digestset/set.go
+++ b/digestset/set.go
@@ -4,6 +4,7 @@ import (
 	"errors"
 	"sort"
 	"strings"
+	"sync"
 )
 
 var (
@@ -27,6 +28,7 @@ var (
 // the complete set of digests. To mitigate collisions, an
 // appropriately long short code should be used.
 type Set struct {
+	mutex   sync.RWMutex
 	entries digestEntries
 }
 
@@ -63,6 +65,8 @@ func checkShortMatch(alg Algorithm, hex, shortAlg, shortHex string) bool {
 // with an empty digest value. If multiple matches are found
 // ErrDigestAmbiguous will be returned with an empty digest value.
 func (dst *Set) Lookup(d string) (Digest, error) {
+	dst.mutex.RLock()
+	defer dst.mutex.RUnlock()
 	if len(dst.entries) == 0 {
 		return "", ErrDigestNotFound
 	}
@@ -101,13 +105,15 @@ func (dst *Set) Lookup(d string) (Digest, error) {
 	return dst.entries[idx].digest, nil
 }
 
-// Add adds the given digests to the set. An error will be returned
+// Add adds the given digest to the set. An error will be returned
 // if the given digest is invalid. If the digest already exists in the
-// table, this operation will be a no-op.
+// set, this operation will be a no-op.
 func (dst *Set) Add(d Digest) error {
 	if err := d.Validate(); err != nil {
 		return err
 	}
+	dst.mutex.Lock()
+	defer dst.mutex.Unlock()
 	entry := &digestEntry{alg: d.Algorithm(), val: d.Hex(), digest: d}
 	searchFunc := func(i int) bool {
 		if dst.entries[i].val == entry.val {
@@ -130,12 +136,56 @@ func (dst *Set) Add(d Digest) error {
 	return nil
 }
 
+// Remove removes the given digest from the set. An err will be
+// returned if the given digest is invalid. If the digest does
+// not exist in the set, this operation will be a no-op.
+func (dst *Set) Remove(d Digest) error {
+	if err := d.Validate(); err != nil {
+		return err
+	}
+	dst.mutex.Lock()
+	defer dst.mutex.Unlock()
+	entry := &digestEntry{alg: d.Algorithm(), val: d.Hex(), digest: d}
+	searchFunc := func(i int) bool {
+		if dst.entries[i].val == entry.val {
+			return dst.entries[i].alg >= entry.alg
+		}
+		return dst.entries[i].val >= entry.val
+	}
+	idx := sort.Search(len(dst.entries), searchFunc)
+	// Not found if idx is after or value at idx is not digest
+	if idx == len(dst.entries) || dst.entries[idx].digest != d {
+		return nil
+	}
+
+	entries := dst.entries
+	copy(entries[idx:], entries[idx+1:])
+	entries = entries[:len(entries)-1]
+	dst.entries = entries
+
+	return nil
+}
+
+// All returns all the digests in the set
+func (dst *Set) All() []Digest {
+	dst.mutex.RLock()
+	defer dst.mutex.RUnlock()
+	retValues := make([]Digest, len(dst.entries))
+	for i := range dst.entries {
+		retValues[i] = dst.entries[i].digest
+	}
+
+	return retValues
+}
+
 // ShortCodeTable returns a map of Digest to unique short codes. The
 // length represents the minimum value, the maximum length may be the
 // entire value of digest if uniqueness cannot be achieved without the
 // full value. This function will attempt to make short codes as short
 // as possible to be unique.
 func ShortCodeTable(dst *Set, length int) map[Digest]string {
+	dst.mutex.RLock()
+	defer dst.mutex.RUnlock()
 	m := make(map[Digest]string, len(dst.entries))
 	l := length
 	resetIdx := 0
diff --git a/digestset/set_test.go b/digestset/set_test.go
index faeba6d..0c0f650 100644
--- a/digestset/set_test.go
+++ b/digestset/set_test.go
@@ -125,6 +125,66 @@ func TestAddDuplication(t *testing.T) {
 	}
 }
 
+func TestRemove(t *testing.T) {
+	digests, err := createDigests(10)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	dset := NewSet()
+	for i := range digests {
+		if err := dset.Add(digests[i]); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	dgst, err := dset.Lookup(digests[0].String())
+	if err != nil {
+		t.Fatal(err)
+	}
+	if dgst != digests[0] {
+		t.Fatalf("Unexpected digest value:\n\tExpected: %s\n\tActual: %s", digests[0], dgst)
+	}
+
+	if err := dset.Remove(digests[0]); err != nil {
+		t.Fatal(err)
+	}
+
+	if _, err := dset.Lookup(digests[0].String()); err != ErrDigestNotFound {
+		t.Fatalf("Expected error %v when looking up removed digest, got %v", ErrDigestNotFound, err)
+	}
+}
+
+func TestAll(t *testing.T) {
+	digests, err := createDigests(100)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	dset := NewSet()
+	for i := range digests {
+		if err := dset.Add(digests[i]); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	all := map[Digest]struct{}{}
+	for _, dgst := range dset.All() {
+		all[dgst] = struct{}{}
+	}
+
+	if len(all) != len(digests) {
+		t.Fatalf("Unexpected number of unique digests found:\n\tExpected: %d\n\tActual: %d", len(digests), len(all))
+	}
+
+	for i, dgst := range digests {
+		if _, ok := all[dgst]; !ok {
+			t.Fatalf("Missing element at position %d: %s", i, dgst)
+		}
+	}
+
+}
+
 func assertEqualShort(t *testing.T, actual, expected string) {
 	if actual != expected {
 		t.Fatalf("Unexpected short value:\n\tExpected: %s\n\tActual: %s", expected, actual)
@@ -219,6 +279,29 @@ func benchLookupNTable(b *testing.B, n int, shortLen int) {
 	}
 }
 
+func benchRemoveNTable(b *testing.B, n int) {
+	digests, err := createDigests(n)
+	if err != nil {
+		b.Fatal(err)
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		dset := &Set{entries: digestEntries(make([]*digestEntry, 0, n))}
+		b.StopTimer()
+		for j := range digests {
+			if err = dset.Add(digests[j]); err != nil {
+				b.Fatal(err)
+			}
+		}
+		b.StartTimer()
+		for j := range digests {
+			if err = dset.Remove(digests[j]); err != nil {
+				b.Fatal(err)
+			}
+		}
+	}
+}
+
 func benchShortCodeNTable(b *testing.B, n int, shortLen int) {
 	digests, err := createDigests(n)
 	if err != nil {
@@ -249,6 +332,18 @@ func BenchmarkAdd1000(b *testing.B) {
 	benchAddNTable(b, 1000)
 }
 
+func BenchmarkRemove10(b *testing.B) {
+	benchRemoveNTable(b, 10)
+}
+
+func BenchmarkRemove100(b *testing.B) {
+	benchRemoveNTable(b, 100)
+}
+
+func BenchmarkRemove1000(b *testing.B) {
+	benchRemoveNTable(b, 1000)
+}
+
 func BenchmarkLookup10(b *testing.B) {
 	benchLookupNTable(b, 10, 12)
 }

From 6ac142d3167937cb46fb2e449fc4258ca9fded19 Mon Sep 17 00:00:00 2001
From: Tonis Tiigi <tonistiigi@gmail.com>
Date: Wed, 2 Dec 2015 15:57:47 -0800
Subject: [PATCH 04/10] Validate digest length on parsing

Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>
---
 digestset/set_test.go | 65 ++++++++++++++++++++++---------------------
 1 file changed, 33 insertions(+), 32 deletions(-)

diff --git a/digestset/set_test.go b/digestset/set_test.go
index 0c0f650..e9dab87 100644
--- a/digestset/set_test.go
+++ b/digestset/set_test.go
@@ -15,14 +15,14 @@ func assertEqualDigests(t *testing.T, d1, d2 Digest) {
 
 func TestLookup(t *testing.T) {
 	digests := []Digest{
-		"sha256:12345",
-		"sha256:1234",
-		"sha256:12346",
-		"sha256:54321",
-		"sha256:65431",
-		"sha256:64321",
-		"sha256:65421",
-		"sha256:65321",
+		"sha256:1234511111111111111111111111111111111111111111111111111111111111",
+		"sha256:1234111111111111111111111111111111111111111111111111111111111111",
+		"sha256:1234611111111111111111111111111111111111111111111111111111111111",
+		"sha256:5432111111111111111111111111111111111111111111111111111111111111",
+		"sha256:6543111111111111111111111111111111111111111111111111111111111111",
+		"sha256:6432111111111111111111111111111111111111111111111111111111111111",
+		"sha256:6542111111111111111111111111111111111111111111111111111111111111",
+		"sha256:6532111111111111111111111111111111111111111111111111111111111111",
 	}
 
 	dset := NewSet()
@@ -55,10 +55,12 @@ func TestLookup(t *testing.T) {
 	}
 
 	dgst, err = dset.Lookup("sha256:1234")
-	if err != nil {
+	if err == nil {
+		t.Fatal("Expected ambiguous error looking up: sha256:1234")
+	}
+	if err != ErrDigestAmbiguous {
 		t.Fatal(err)
 	}
-	assertEqualDigests(t, dgst, digests[1])
 
 	dgst, err = dset.Lookup("sha256:12345")
 	if err != nil {
@@ -87,14 +89,14 @@ func TestLookup(t *testing.T) {
 
 func TestAddDuplication(t *testing.T) {
 	digests := []Digest{
-		"sha256:1234",
-		"sha256:12345",
-		"sha256:12346",
-		"sha256:54321",
-		"sha256:65431",
-		"sha512:65431",
-		"sha512:65421",
-		"sha512:65321",
+		"sha256:1234111111111111111111111111111111111111111111111111111111111111",
+		"sha256:1234511111111111111111111111111111111111111111111111111111111111",
+		"sha256:1234611111111111111111111111111111111111111111111111111111111111",
+		"sha256:5432111111111111111111111111111111111111111111111111111111111111",
+		"sha256:6543111111111111111111111111111111111111111111111111111111111111",
+		"sha512:65431111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111",
+		"sha512:65421111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111",
+		"sha512:65321111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111",
 	}
 
 	dset := NewSet()
@@ -108,7 +110,7 @@ func TestAddDuplication(t *testing.T) {
 		t.Fatal("Invalid dset size")
 	}
 
-	if err := dset.Add(Digest("sha256:12345")); err != nil {
+	if err := dset.Add(Digest("sha256:1234511111111111111111111111111111111111111111111111111111111111")); err != nil {
 		t.Fatal(err)
 	}
 
@@ -116,7 +118,7 @@ func TestAddDuplication(t *testing.T) {
 		t.Fatal("Duplicate digest insert allowed")
 	}
 
-	if err := dset.Add(Digest("sha384:12345")); err != nil {
+	if err := dset.Add(Digest("sha384:123451111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111")); err != nil {
 		t.Fatal(err)
 	}
 
@@ -193,14 +195,14 @@ func assertEqualShort(t *testing.T, actual, expected string) {
 
 func TestShortCodeTable(t *testing.T) {
 	digests := []Digest{
-		"sha256:1234",
-		"sha256:12345",
-		"sha256:12346",
-		"sha256:54321",
-		"sha256:65431",
-		"sha256:64321",
-		"sha256:65421",
-		"sha256:65321",
+		"sha256:1234111111111111111111111111111111111111111111111111111111111111",
+		"sha256:1234511111111111111111111111111111111111111111111111111111111111",
+		"sha256:1234611111111111111111111111111111111111111111111111111111111111",
+		"sha256:5432111111111111111111111111111111111111111111111111111111111111",
+		"sha256:6543111111111111111111111111111111111111111111111111111111111111",
+		"sha256:6432111111111111111111111111111111111111111111111111111111111111",
+		"sha256:6542111111111111111111111111111111111111111111111111111111111111",
+		"sha256:6532111111111111111111111111111111111111111111111111111111111111",
 	}
 
 	dset := NewSet()
@@ -215,10 +217,9 @@ func TestShortCodeTable(t *testing.T) {
 	if len(dump) < len(digests) {
 		t.Fatalf("Error unexpected size: %d, expecting %d", len(dump), len(digests))
 	}
-
-	assertEqualShort(t, dump[digests[0]], "sha256:1234")
-	assertEqualShort(t, dump[digests[1]], "sha256:12345")
-	assertEqualShort(t, dump[digests[2]], "sha256:12346")
+	assertEqualShort(t, dump[digests[0]], "12341")
+	assertEqualShort(t, dump[digests[1]], "12345")
+	assertEqualShort(t, dump[digests[2]], "12346")
 	assertEqualShort(t, dump[digests[3]], "54")
 	assertEqualShort(t, dump[digests[4]], "6543")
 	assertEqualShort(t, dump[digests[5]], "64")

From 1cbb645d468c4a565467f3f4bf62107295811bcd Mon Sep 17 00:00:00 2001
From: Aaron Lehmann <aaron.lehmann@docker.com>
Date: Wed, 10 Feb 2016 16:26:29 -0800
Subject: [PATCH 05/10] Typo fixes in comments

Correct spelling of words in source code comments.

Signed-off-by: Aaron Lehmann <aaron.lehmann@docker.com>
---
 digestset/set.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/digestset/set.go b/digestset/set.go
index 3fac41b..4b9313c 100644
--- a/digestset/set.go
+++ b/digestset/set.go
@@ -22,7 +22,7 @@ var (
 // may be easily referenced by easily  referenced by a string
 // representation of the digest as well as short representation.
 // The uniqueness of the short representation is based on other
-// digests in the set. If digests are ommited from this set,
+// digests in the set. If digests are omitted from this set,
 // collisions in a larger set may not be detected, therefore it
 // is important to always do short representation lookups on
 // the complete set of digests. To mitigate collisions, an

From e0bfa0f7aae55caaad04afe89ab84908513b509b Mon Sep 17 00:00:00 2001
From: Stephen J Day <stephen.day@docker.com>
Date: Thu, 15 Dec 2016 14:58:54 -0800
Subject: [PATCH 06/10] digest: remove stuttering ParseDigest function

Signed-off-by: Stephen J Day <stephen.day@docker.com>
---
 digestset/set.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/digestset/set.go b/digestset/set.go
index 4b9313c..6074080 100644
--- a/digestset/set.go
+++ b/digestset/set.go
@@ -75,7 +75,7 @@ func (dst *Set) Lookup(d string) (Digest, error) {
 		alg        Algorithm
 		hex        string
 	)
-	dgst, err := ParseDigest(d)
+	dgst, err := Parse(d)
 	if err == ErrDigestInvalidFormat {
 		hex = d
 		searchFunc = func(i int) bool {

From 5dd3cbe30c37cedc1b2a993126979e2f75f2d7b9 Mon Sep 17 00:00:00 2001
From: Stephen J Day <stephen.day@docker.com>
Date: Fri, 16 Dec 2016 16:28:34 -0800
Subject: [PATCH 07/10] digest: migrate to opencontainers/go-digest

Signed-off-by: Stephen J Day <stephen.day@docker.com>
---
 digestset/set.go      | 30 ++++++++++++++++--------------
 digestset/set_test.go | 25 ++++++++++++++-----------
 2 files changed, 30 insertions(+), 25 deletions(-)

diff --git a/digestset/set.go b/digestset/set.go
index 6074080..71327dc 100644
--- a/digestset/set.go
+++ b/digestset/set.go
@@ -1,10 +1,12 @@
-package digest
+package digestset
 
 import (
 	"errors"
 	"sort"
 	"strings"
 	"sync"
+
+	digest "github.com/opencontainers/go-digest"
 )
 
 var (
@@ -44,7 +46,7 @@ func NewSet() *Set {
 // values or short values. This function does not test equality,
 // rather whether the second value could match against the first
 // value.
-func checkShortMatch(alg Algorithm, hex, shortAlg, shortHex string) bool {
+func checkShortMatch(alg digest.Algorithm, hex, shortAlg, shortHex string) bool {
 	if len(hex) == len(shortHex) {
 		if hex != shortHex {
 			return false
@@ -64,7 +66,7 @@ func checkShortMatch(alg Algorithm, hex, shortAlg, shortHex string) bool {
 // If no digests could be found ErrDigestNotFound will be returned
 // with an empty digest value. If multiple matches are found
 // ErrDigestAmbiguous will be returned with an empty digest value.
-func (dst *Set) Lookup(d string) (Digest, error) {
+func (dst *Set) Lookup(d string) (digest.Digest, error) {
 	dst.mutex.RLock()
 	defer dst.mutex.RUnlock()
 	if len(dst.entries) == 0 {
@@ -72,11 +74,11 @@ func (dst *Set) Lookup(d string) (Digest, error) {
 	}
 	var (
 		searchFunc func(int) bool
-		alg        Algorithm
+		alg        digest.Algorithm
 		hex        string
 	)
-	dgst, err := Parse(d)
-	if err == ErrDigestInvalidFormat {
+	dgst, err := digest.Parse(d)
+	if err == digest.ErrDigestInvalidFormat {
 		hex = d
 		searchFunc = func(i int) bool {
 			return dst.entries[i].val >= d
@@ -108,7 +110,7 @@ func (dst *Set) Lookup(d string) (Digest, error) {
 // Add adds the given digest to the set. An error will be returned
 // if the given digest is invalid. If the digest already exists in the
 // set, this operation will be a no-op.
-func (dst *Set) Add(d Digest) error {
+func (dst *Set) Add(d digest.Digest) error {
 	if err := d.Validate(); err != nil {
 		return err
 	}
@@ -139,7 +141,7 @@ func (dst *Set) Add(d Digest) error {
 // Remove removes the given digest from the set. An err will be
 // returned if the given digest is invalid. If the digest does
 // not exist in the set, this operation will be a no-op.
-func (dst *Set) Remove(d Digest) error {
+func (dst *Set) Remove(d digest.Digest) error {
 	if err := d.Validate(); err != nil {
 		return err
 	}
@@ -167,10 +169,10 @@ func (dst *Set) Remove(d Digest) error {
 }
 
 // All returns all the digests in the set
-func (dst *Set) All() []Digest {
+func (dst *Set) All() []digest.Digest {
 	dst.mutex.RLock()
 	defer dst.mutex.RUnlock()
-	retValues := make([]Digest, len(dst.entries))
+	retValues := make([]digest.Digest, len(dst.entries))
 	for i := range dst.entries {
 		retValues[i] = dst.entries[i].digest
 	}
@@ -183,10 +185,10 @@ func (dst *Set) All() []Digest {
 // entire value of digest if uniqueness cannot be achieved without the
 // full value. This function will attempt to make short codes as short
 // as possible to be unique.
-func ShortCodeTable(dst *Set, length int) map[Digest]string {
+func ShortCodeTable(dst *Set, length int) map[digest.Digest]string {
 	dst.mutex.RLock()
 	defer dst.mutex.RUnlock()
-	m := make(map[Digest]string, len(dst.entries))
+	m := make(map[digest.Digest]string, len(dst.entries))
 	l := length
 	resetIdx := 0
 	for i := 0; i < len(dst.entries); i++ {
@@ -222,9 +224,9 @@ func ShortCodeTable(dst *Set, length int) map[Digest]string {
 }
 
 type digestEntry struct {
-	alg    Algorithm
+	alg    digest.Algorithm
 	val    string
-	digest Digest
+	digest digest.Digest
 }
 
 type digestEntries []*digestEntry
diff --git a/digestset/set_test.go b/digestset/set_test.go
index e9dab87..89c5729 100644
--- a/digestset/set_test.go
+++ b/digestset/set_test.go
@@ -1,20 +1,23 @@
-package digest
+package digestset
 
 import (
 	"crypto/sha256"
+	_ "crypto/sha512"
 	"encoding/binary"
 	"math/rand"
 	"testing"
+
+	digest "github.com/opencontainers/go-digest"
 )
 
-func assertEqualDigests(t *testing.T, d1, d2 Digest) {
+func assertEqualDigests(t *testing.T, d1, d2 digest.Digest) {
 	if d1 != d2 {
 		t.Fatalf("Digests do not match:\n\tActual: %s\n\tExpected: %s", d1, d2)
 	}
 }
 
 func TestLookup(t *testing.T) {
-	digests := []Digest{
+	digests := []digest.Digest{
 		"sha256:1234511111111111111111111111111111111111111111111111111111111111",
 		"sha256:1234111111111111111111111111111111111111111111111111111111111111",
 		"sha256:1234611111111111111111111111111111111111111111111111111111111111",
@@ -88,7 +91,7 @@ func TestLookup(t *testing.T) {
 }
 
 func TestAddDuplication(t *testing.T) {
-	digests := []Digest{
+	digests := []digest.Digest{
 		"sha256:1234111111111111111111111111111111111111111111111111111111111111",
 		"sha256:1234511111111111111111111111111111111111111111111111111111111111",
 		"sha256:1234611111111111111111111111111111111111111111111111111111111111",
@@ -110,7 +113,7 @@ func TestAddDuplication(t *testing.T) {
 		t.Fatal("Invalid dset size")
 	}
 
-	if err := dset.Add(Digest("sha256:1234511111111111111111111111111111111111111111111111111111111111")); err != nil {
+	if err := dset.Add(digest.Digest("sha256:1234511111111111111111111111111111111111111111111111111111111111")); err != nil {
 		t.Fatal(err)
 	}
 
@@ -118,7 +121,7 @@ func TestAddDuplication(t *testing.T) {
 		t.Fatal("Duplicate digest insert allowed")
 	}
 
-	if err := dset.Add(Digest("sha384:123451111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111")); err != nil {
+	if err := dset.Add(digest.Digest("sha384:123451111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111")); err != nil {
 		t.Fatal(err)
 	}
 
@@ -170,7 +173,7 @@ func TestAll(t *testing.T) {
 		}
 	}
 
-	all := map[Digest]struct{}{}
+	all := map[digest.Digest]struct{}{}
 	for _, dgst := range dset.All() {
 		all[dgst] = struct{}{}
 	}
@@ -194,7 +197,7 @@ func assertEqualShort(t *testing.T, actual, expected string) {
 }
 
 func TestShortCodeTable(t *testing.T) {
-	digests := []Digest{
+	digests := []digest.Digest{
 		"sha256:1234111111111111111111111111111111111111111111111111111111111111",
 		"sha256:1234511111111111111111111111111111111111111111111111111111111111",
 		"sha256:1234611111111111111111111111111111111111111111111111111111111111",
@@ -227,15 +230,15 @@ func TestShortCodeTable(t *testing.T) {
 	assertEqualShort(t, dump[digests[7]], "653")
 }
 
-func createDigests(count int) ([]Digest, error) {
+func createDigests(count int) ([]digest.Digest, error) {
 	r := rand.New(rand.NewSource(25823))
-	digests := make([]Digest, count)
+	digests := make([]digest.Digest, count)
 	for i := range digests {
 		h := sha256.New()
 		if err := binary.Write(h, binary.BigEndian, r.Int63()); err != nil {
 			return nil, err
 		}
-		digests[i] = NewDigest("sha256", h)
+		digests[i] = digest.NewDigest("sha256", h)
 	}
 	return digests, nil
 }

From ecd7b3c77ddcf62299adba83edfecddb502df782 Mon Sep 17 00:00:00 2001
From: zhouhaibing089 <zhouhaibing089@gmail.com>
Date: Tue, 31 Oct 2017 16:33:36 +0800
Subject: [PATCH 08/10] digestset: refine some words on unit test

1. when lookup an entry which is missing, it should say NotFound.
2. when add duplicated entry, the entries size should be increased.
3. when add entry which has different algorithm, it should be allowed.

Signed-off-by: zhouhaibing089 <zhouhaibing089@gmail.com>
---
 digestset/set_test.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/digestset/set_test.go b/digestset/set_test.go
index 89c5729..67d46c6 100644
--- a/digestset/set_test.go
+++ b/digestset/set_test.go
@@ -51,7 +51,7 @@ func TestLookup(t *testing.T) {
 
 	dgst, err = dset.Lookup("9876")
 	if err == nil {
-		t.Fatal("Expected ambiguous error looking up: 9876")
+		t.Fatal("Expected not found error looking up: 9876")
 	}
 	if err != ErrDigestNotFound {
 		t.Fatal(err)
@@ -118,7 +118,7 @@ func TestAddDuplication(t *testing.T) {
 	}
 
 	if len(dset.entries) != 8 {
-		t.Fatal("Duplicate digest insert allowed")
+		t.Fatal("Duplicate digest insert should not increase entries size")
 	}
 
 	if err := dset.Add(digest.Digest("sha384:123451111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111")); err != nil {
@@ -126,7 +126,7 @@ func TestAddDuplication(t *testing.T) {
 	}
 
 	if len(dset.entries) != 9 {
-		t.Fatal("Insert with different algorithm not allowed")
+		t.Fatal("Insert with different algorithm should be allowed")
 	}
 }
 

From 132fb4765b646579f8a4b0adaef2bdda37684998 Mon Sep 17 00:00:00 2001
From: Derek McGowan <derek@mcgstyle.net>
Date: Mon, 6 Aug 2018 14:34:15 -0700
Subject: [PATCH 09/10] Enable static checks

Signed-off-by: Derek McGowan <derek@mcgstyle.net>
---
 digestset/set_test.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/digestset/set_test.go b/digestset/set_test.go
index 67d46c6..5f6d09d 100644
--- a/digestset/set_test.go
+++ b/digestset/set_test.go
@@ -41,7 +41,7 @@ func TestLookup(t *testing.T) {
 	}
 	assertEqualDigests(t, dgst, digests[3])
 
-	dgst, err = dset.Lookup("1234")
+	_, err = dset.Lookup("1234")
 	if err == nil {
 		t.Fatal("Expected ambiguous error looking up: 1234")
 	}
@@ -49,7 +49,7 @@ func TestLookup(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	dgst, err = dset.Lookup("9876")
+	_, err = dset.Lookup("9876")
 	if err == nil {
 		t.Fatal("Expected not found error looking up: 9876")
 	}
@@ -57,7 +57,7 @@ func TestLookup(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	dgst, err = dset.Lookup("sha256:1234")
+	_, err = dset.Lookup("sha256:1234")
 	if err == nil {
 		t.Fatal("Expected ambiguous error looking up: sha256:1234")
 	}

From 17eb78b6c7011994c9091cc2cb5b250dfb394b38 Mon Sep 17 00:00:00 2001
From: Derek McGowan <derek@mcg.dev>
Date: Sun, 10 May 2020 13:59:30 -0700
Subject: [PATCH 10/10] Add copyright to digestset files

Signed-off-by: Derek McGowan <derek@mcg.dev>
---
 digestset/set.go      | 15 +++++++++++++++
 digestset/set_test.go | 15 +++++++++++++++
 2 files changed, 30 insertions(+)

diff --git a/digestset/set.go b/digestset/set.go
index 71327dc..71f2418 100644
--- a/digestset/set.go
+++ b/digestset/set.go
@@ -1,3 +1,18 @@
+// Copyright 2020, 2020 OCI Contributors
+// Copyright 2017 Docker, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package digestset
 
 import (
diff --git a/digestset/set_test.go b/digestset/set_test.go
index 5f6d09d..7a01c24 100644
--- a/digestset/set_test.go
+++ b/digestset/set_test.go
@@ -1,3 +1,18 @@
+// Copyright 2020, 2020 OCI Contributors
+// Copyright 2017 Docker, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package digestset
 
 import (