main.go

package main

import (
	"flag"
	"fmt"
	"hash"
	"io"
	"log"
	"os"
	"runtime/pprof"
	"time"

	"code.cloudfoundry.org/bytefmt"
	//rollsum "github.com/chmduquesne/rollinghash/adler32"
	//rollsum "github.com/chmduquesne/rollinghash/buzhash32"
	rollsum "github.com/chmduquesne/rollinghash/buzhash64"
	//rollsum "github.com/chmduquesne/rollinghash/bozo32"
)

const (
	KiB = 1024
	MiB = 1024 * KiB
	GiB = 1024 * MiB

	clearscreen = "\033[2J\033[1;1H"
	clearline   = "\x1b[2K"
)

func genMasks() (res []uint64) {
	res = make([]uint64, 64)
	ones := ^uint64(0) // 0xffffffffffffffff
	for i := 0; i < 64; i++ {
		res[i] = ones >> uint(63-i)
	}
	return
}

// Gets the hash sum as a uint64
func sum64(h hash.Hash) (res uint64) {
	buf := make([]byte, 0, 8)
	s := h.Sum(buf)
	for _, b := range s {
		res <<= 8
		res |= uint64(b)
	}
	return
}

func main() {
	cpuprofile := flag.String("cpuprofile", "", "write cpu profile to file")
	dostats := flag.Bool("stats", false, "Do some stats about the rolling sum")
	size := flag.String("size", "256M", "How much data to read")
	flag.Parse()

	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			log.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	fileSize, err := bytefmt.ToBytes(*size)
	if err != nil {
		log.Fatal(err)
	}

	bufsize := 16 * MiB
	buf := make([]byte, bufsize)
	t := time.Now()

	f, err := os.Open("/dev/urandom")
	if err != nil {
		log.Fatal(err)
	}
	defer func() {
		if err := f.Close(); err != nil {
			log.Fatal(err)
		}
	}()

	io.ReadFull(f, buf)

	roll := rollsum.New()
	roll.Write(buf[:64])

	masks := genMasks()
	hits := make(map[uint64]uint64)
	for _, m := range masks {
		hits[m] = 0
	}

	n := uint64(0)
	k := 0
	for n < fileSize {
		if k >= bufsize {
			status := fmt.Sprintf("Byte count: %s", bytefmt.ByteSize(n))
			if *dostats {
				fmt.Printf(clearscreen)
				fmt.Println(status)
				for i, m := range masks {
					frequency := "NaN"
					if hits[m] != 0 {
						frequency = bytefmt.ByteSize(n / hits[m])
					}
					fmt.Printf("0x%016x (%02d bits): every %s\n", m, i+1, frequency)
				}
			} else {
				fmt.Printf(clearline)
				fmt.Printf(status)
				fmt.Printf("\r")
			}
			_, err := io.ReadFull(f, buf)
			if err != nil {
				panic(err)
			}
			k = 0
		}
		roll.Roll(buf[k])
		if *dostats {
			s := sum64(roll)
			for _, m := range masks {
				if s&m == m {
					hits[m] += 1
				} else {
					break
				}
			}
		}
		k++
		n++
	}
	duration := time.Since(t)
	fmt.Printf("Rolled %s of data in %v (%s/s).\n",
		bytefmt.ByteSize(n),
		duration,
		bytefmt.ByteSize(n*1e9/uint64(duration)),
	)
}