Skip to content

Commit

Permalink
Add hash.XxHash
Browse files Browse the repository at this point in the history
Also move the non crypto hash funcs into this new package.

This is much faster than e.g. MD5, especially for larger inputs:

```
BenchmarkXxHash/xxHash_43-10         	 9611962	       114.2 ns/op	      56 B/op	       4 allocs/op
BenchmarkXxHash/xxHash_4300-10       	 3010317	       397.3 ns/op	      56 B/op	       4 allocs/op
BenchmarkXxHash/mdb5_43-10           	 5566405	       214.1 ns/op	     128 B/op	       4 allocs/op
BenchmarkXxHash/mdb5_4300-10         	  163681	      7432 ns/op	    4944 B/op	       4 allocs/op
```

Fixes gohugoio#12635
  • Loading branch information
bep committed Jul 3, 2024
1 parent 82af94d commit 8679a33
Show file tree
Hide file tree
Showing 6 changed files with 169 additions and 7 deletions.
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ require (
github.com/bep/overlayfs v0.9.2
github.com/bep/simplecobra v0.4.0
github.com/bep/tmc v0.5.1
github.com/cespare/xxhash/v2 v2.3.0
github.com/clbanning/mxj/v2 v2.7.0
github.com/cli/safeexec v1.0.1
github.com/disintegration/gift v1.2.1
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,8 @@ github.com/bep/tmc v0.5.1/go.mod h1:tGYHN8fS85aJPhDLgXETVKp+PR382OvFi2+q2GkGsq0=
github.com/bep/workers v1.0.0 h1:U+H8YmEaBCEaFZBst7GcRVEoqeRC9dzH2dWOwGmOchg=
github.com/bep/workers v1.0.0/go.mod h1:7kIESOB86HfR2379pwoMWNy8B50D7r99fRLUyPSNyCs=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
Expand Down
2 changes: 2 additions & 0 deletions tpl/crypto/crypto.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import (
"hash"
"hash/fnv"

"github.com/gohugoio/hugo/common/hugo"
"github.com/spf13/cast"
)

Expand Down Expand Up @@ -72,6 +73,7 @@ func (ns *Namespace) SHA256(v any) (string, error) {
// FNV32a hashes v using fnv32a algorithm.
// <docsmeta>{"newIn": "0.98.0" }</docsmeta>
func (ns *Namespace) FNV32a(v any) (int, error) {
hugo.Deprecate("crypto.FNV32a", "Use hash.FNV32a.", "v0.129.0")
conv, err := cast.ToStringE(v)
if err != nil {
return 0, err
Expand Down
7 changes: 0 additions & 7 deletions tpl/crypto/init.go
Original file line number Diff line number Diff line change
Expand Up @@ -53,13 +53,6 @@ func init() {
},
)

ns.AddMethodMapping(ctx.FNV32a,
nil,
[][2]string{
{`{{ crypto.FNV32a "Hugo Rocks!!" }}`, `1515779328`},
},
)

ns.AddMethodMapping(ctx.HMAC,
[]string{"hmac"},
[][2]string{
Expand Down
94 changes: 94 additions & 0 deletions tpl/hash/hash.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
// Copyright 2024 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package hash provides non-cryptographic hash functions for template use.
package hash

import (
"context"
"encoding/hex"
"hash/fnv"

"github.com/cespare/xxhash/v2"
"github.com/gohugoio/hugo/deps"
"github.com/gohugoio/hugo/tpl/internal"
"github.com/spf13/cast"
)

// New returns a new instance of the hash-namespaced template functions.
func New() *Namespace {
return &Namespace{}
}

// Namespace provides template functions for the "hash" namespace.
type Namespace struct{}

// XxHash returns the xxHash of the input string.
func (ns *Namespace) XxHash(v any) (string, error) {
conv, err := cast.ToStringE(v)
if err != nil {
return "", err
}

hasher := xxhash.New()

_, err = hasher.WriteString(conv)
if err != nil {
return "", err
}
hash := hasher.Sum(nil)
return hex.EncodeToString(hash), nil
}

// FNV32a hashes v using fnv32a algorithm.
// <docsmeta>{"newIn": "0.98.0" }</docsmeta>
func (ns *Namespace) FNV32a(v any) (int, error) {
conv, err := cast.ToStringE(v)
if err != nil {
return 0, err
}
algorithm := fnv.New32a()
algorithm.Write([]byte(conv))
return int(algorithm.Sum32()), nil
}

const name = "hash"

func init() {
f := func(d *deps.Deps) *internal.TemplateFuncsNamespace {
ctx := New()

ns := &internal.TemplateFuncsNamespace{
Name: name,
Context: func(cctx context.Context, args ...any) (any, error) { return ctx, nil },
}

ns.AddMethodMapping(ctx.XxHash,
[]string{"xxhash"},
[][2]string{
{`{{ hash.XxHash "The quick brown fox jumps over the lazy dog" }}`, `0b242d361fda71bc`},
},
)

ns.AddMethodMapping(ctx.FNV32a,
nil,
[][2]string{
{`{{ hash.FNV32a "Hugo Rocks!!" }}`, `1515779328`},
},
)

return ns
}

internal.AddTemplateFuncsNamespace(f)
}
70 changes: 70 additions & 0 deletions tpl/hash/hash_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
// Copyright 2024 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package hash provides non-cryptographic hash functions for template use.
package hash

import (
"crypto/md5"
"encoding/hex"
"fmt"
"strings"
"testing"

qt "github.com/frankban/quicktest"
"github.com/spf13/cast"
)

func TestXxHash(t *testing.T) {
t.Parallel()
c := qt.New(t)

ns := New()

h, err := ns.XxHash("The quick brown fox jumps over the lazy dog")
c.Assert(err, qt.IsNil)
// Facit: https://asecuritysite.com/encryption/xxhash?val=The%20quick%20brown%20fox%20jumps%20over%20the%20lazy%20dog
c.Assert(h, qt.Equals, "0b242d361fda71bc")
}

func BenchmarkXxHash(b *testing.B) {
const inputSmall = "The quick brown fox jumps over the lazy dog"
inputLarge := strings.Repeat(inputSmall, 100)

runBench := func(name, input string, b *testing.B, fn func(v any) (string, error)) {
b.Run(fmt.Sprintf("%s_%d", name, len(input)), func(b *testing.B) {
for i := 0; i < b.N; i++ {
_, _ = fn(input)
}
})
}

ns := New()
runBench("xxHash", inputSmall, b, ns.XxHash)
runBench("xxHash", inputLarge, b, ns.XxHash)

// Copied from the crypto tpl/crypto package,
// just to have something to compare the above with.
fn := func(v any) (string, error) {
conv, err := cast.ToStringE(v)
if err != nil {
return "", err
}

hash := md5.Sum([]byte(conv))
return hex.EncodeToString(hash[:]), nil
}

runBench("mdb5", inputSmall, b, fn)
runBench("mdb5", inputLarge, b, fn)
}

0 comments on commit 8679a33

Please sign in to comment.