* Move modules/gzip to gitea.com/macaron/gzip * Fix vendorfor-closed-social
@ -1,131 +0,0 @@ | |||||
// Copyright 2019 The Gitea Authors. All rights reserved. | |||||
// Use of this source code is governed by a MIT-style | |||||
// license that can be found in the LICENSE file. | |||||
package gzip | |||||
import ( | |||||
"archive/zip" | |||||
"bytes" | |||||
"io/ioutil" | |||||
"net/http" | |||||
"net/http/httptest" | |||||
"testing" | |||||
"gitea.com/macaron/macaron" | |||||
gzipp "github.com/klauspost/compress/gzip" | |||||
"github.com/stretchr/testify/assert" | |||||
) | |||||
func setup(sampleResponse []byte) (*macaron.Macaron, *[]byte) { | |||||
m := macaron.New() | |||||
m.Use(Middleware()) | |||||
m.Get("/", func() *[]byte { return &sampleResponse }) | |||||
return m, &sampleResponse | |||||
} | |||||
func reqNoAcceptGzip(t *testing.T, m *macaron.Macaron, sampleResponse *[]byte) { | |||||
// Request without accept gzip: Should not gzip | |||||
resp := httptest.NewRecorder() | |||||
req, err := http.NewRequest("GET", "/", nil) | |||||
assert.NoError(t, err) | |||||
m.ServeHTTP(resp, req) | |||||
_, ok := resp.HeaderMap[contentEncodingHeader] | |||||
assert.False(t, ok) | |||||
contentEncoding := resp.Header().Get(contentEncodingHeader) | |||||
assert.NotContains(t, contentEncoding, "gzip") | |||||
result := resp.Body.Bytes() | |||||
assert.Equal(t, *sampleResponse, result) | |||||
} | |||||
func reqAcceptGzip(t *testing.T, m *macaron.Macaron, sampleResponse *[]byte, expectGzip bool) { | |||||
// Request without accept gzip: Should not gzip | |||||
resp := httptest.NewRecorder() | |||||
req, err := http.NewRequest("GET", "/", nil) | |||||
assert.NoError(t, err) | |||||
req.Header.Set(acceptEncodingHeader, "gzip") | |||||
m.ServeHTTP(resp, req) | |||||
_, ok := resp.HeaderMap[contentEncodingHeader] | |||||
assert.Equal(t, ok, expectGzip) | |||||
contentEncoding := resp.Header().Get(contentEncodingHeader) | |||||
if expectGzip { | |||||
assert.Contains(t, contentEncoding, "gzip") | |||||
gzippReader, err := gzipp.NewReader(resp.Body) | |||||
assert.NoError(t, err) | |||||
result, err := ioutil.ReadAll(gzippReader) | |||||
assert.NoError(t, err) | |||||
assert.Equal(t, *sampleResponse, result) | |||||
} else { | |||||
assert.NotContains(t, contentEncoding, "gzip") | |||||
result := resp.Body.Bytes() | |||||
assert.Equal(t, *sampleResponse, result) | |||||
} | |||||
} | |||||
func TestMiddlewareSmall(t *testing.T) { | |||||
m, sampleResponse := setup([]byte("Small response")) | |||||
reqNoAcceptGzip(t, m, sampleResponse) | |||||
reqAcceptGzip(t, m, sampleResponse, false) | |||||
} | |||||
func TestMiddlewareLarge(t *testing.T) { | |||||
b := make([]byte, MinSize+1) | |||||
for i := range b { | |||||
b[i] = byte(i % 256) | |||||
} | |||||
m, sampleResponse := setup(b) | |||||
reqNoAcceptGzip(t, m, sampleResponse) | |||||
// This should be gzipped as we accept gzip | |||||
reqAcceptGzip(t, m, sampleResponse, true) | |||||
} | |||||
func TestMiddlewareGzip(t *testing.T) { | |||||
b := make([]byte, MinSize*10) | |||||
for i := range b { | |||||
b[i] = byte(i % 256) | |||||
} | |||||
outputBuffer := bytes.NewBuffer([]byte{}) | |||||
gzippWriter := gzipp.NewWriter(outputBuffer) | |||||
gzippWriter.Write(b) | |||||
gzippWriter.Flush() | |||||
gzippWriter.Close() | |||||
output := outputBuffer.Bytes() | |||||
m, sampleResponse := setup(output) | |||||
reqNoAcceptGzip(t, m, sampleResponse) | |||||
// This should not be gzipped even though we accept gzip | |||||
reqAcceptGzip(t, m, sampleResponse, false) | |||||
} | |||||
func TestMiddlewareZip(t *testing.T) { | |||||
b := make([]byte, MinSize*10) | |||||
for i := range b { | |||||
b[i] = byte(i % 256) | |||||
} | |||||
outputBuffer := bytes.NewBuffer([]byte{}) | |||||
zipWriter := zip.NewWriter(outputBuffer) | |||||
fileWriter, err := zipWriter.Create("default") | |||||
assert.NoError(t, err) | |||||
fileWriter.Write(b) | |||||
//fileWriter.Close() | |||||
zipWriter.Close() | |||||
output := outputBuffer.Bytes() | |||||
m, sampleResponse := setup(output) | |||||
reqNoAcceptGzip(t, m, sampleResponse) | |||||
// This should not be gzipped even though we accept gzip | |||||
reqAcceptGzip(t, m, sampleResponse, false) | |||||
} |
@ -0,0 +1,9 @@ | |||||
module gitea.com/macaron/gzip | |||||
go 1.12 | |||||
require ( | |||||
gitea.com/macaron/macaron v1.3.3-0.20190821202302-9646c0587edb | |||||
github.com/klauspost/compress v1.9.2 | |||||
github.com/stretchr/testify v1.4.0 | |||||
) |
@ -0,0 +1,42 @@ | |||||
gitea.com/macaron/inject v0.0.0-20190803172902-8375ba841591 h1:UbCTjPcLrNxR9LzKDjQBMT2zoxZuEnca1pZCpgeMuhQ= | |||||
gitea.com/macaron/inject v0.0.0-20190803172902-8375ba841591/go.mod h1:h6E4kLao1Yko6DOU6QDnQPcuoNzvbZqzj2mtPcEn1aM= | |||||
gitea.com/macaron/macaron v1.3.3-0.20190821202302-9646c0587edb h1:amL0md6orTj1tXY16ANzVU9FmzQB+W7aJwp8pVDbrmA= | |||||
gitea.com/macaron/macaron v1.3.3-0.20190821202302-9646c0587edb/go.mod h1:0coI+mSPSwbsyAbOuFllVS38awuk9mevhLD52l50Gjs= | |||||
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= | |||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= | |||||
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= | |||||
github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e h1:JKmoR8x90Iww1ks85zJ1lfDGgIiMDuIptTOhJq+zKyg= | |||||
github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= | |||||
github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= | |||||
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= | |||||
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= | |||||
github.com/klauspost/compress v1.9.2 h1:LfVyl+ZlLlLDeQ/d2AqfGIIH4qEDu0Ed2S5GyhCWIWY= | |||||
github.com/klauspost/compress v1.9.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= | |||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= | |||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= | |||||
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= | |||||
github.com/smartystreets/assertions v0.0.0-20190116191733-b6c0e53d7304 h1:Jpy1PXuP99tXNrhbq2BaPz9B+jNAvH1JPQQpG/9GCXY= | |||||
github.com/smartystreets/assertions v0.0.0-20190116191733-b6c0e53d7304/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= | |||||
github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s= | |||||
github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 h1:WN9BUFbdyOsSH/XohnWpXOlq9NBD5sGAB2FciQMUEe8= | |||||
github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= | |||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= | |||||
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= | |||||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= | |||||
github.com/unknwon/com v0.0.0-20190804042917-757f69c95f3e h1:GSGeB9EAKY2spCABz6xOX5DbxZEXolK+nBSvmsQwRjM= | |||||
github.com/unknwon/com v0.0.0-20190804042917-757f69c95f3e/go.mod h1:tOOxU81rwgoCLoOVVPHb6T/wt8HZygqH5id+GNnlCXM= | |||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= | |||||
golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 h1:HuIa8hRrWRSrqYzx1qI49NNxhdi2PrY7gxVSq1JjLDc= | |||||
golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= | |||||
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= | |||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= | |||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= | |||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= | |||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= | |||||
golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= | |||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= | |||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= | |||||
gopkg.in/ini.v1 v1.44.0 h1:YRJzTUp0kSYWUVFF5XAbDFfyiqwsl0Vb9R8TVP5eRi0= | |||||
gopkg.in/ini.v1 v1.44.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= | |||||
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= | |||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= |
@ -1,32 +0,0 @@ | |||||
// Copyright 2012 The Go Authors. All rights reserved. | |||||
// Use of this source code is governed by a BSD-style | |||||
// license that can be found in the LICENSE file. | |||||
package flate | |||||
// forwardCopy is like the built-in copy function except that it always goes | |||||
// forward from the start, even if the dst and src overlap. | |||||
// It is equivalent to: | |||||
// for i := 0; i < n; i++ { | |||||
// mem[dst+i] = mem[src+i] | |||||
// } | |||||
func forwardCopy(mem []byte, dst, src, n int) { | |||||
if dst <= src { | |||||
copy(mem[dst:dst+n], mem[src:src+n]) | |||||
return | |||||
} | |||||
for { | |||||
if dst >= src+n { | |||||
copy(mem[dst:dst+n], mem[src:src+n]) | |||||
return | |||||
} | |||||
// There is some forward overlap. The destination | |||||
// will be filled with a repeated pattern of mem[src:src+k]. | |||||
// We copy one instance of the pattern here, then repeat. | |||||
// Each time around this loop k will double. | |||||
k := dst - src | |||||
copy(mem[dst:dst+k], mem[src:src+k]) | |||||
n -= k | |||||
dst += k | |||||
} | |||||
} |
@ -1,41 +0,0 @@ | |||||
//+build !noasm | |||||
//+build !appengine | |||||
// Copyright 2015, Klaus Post, see LICENSE for details. | |||||
package flate | |||||
import ( | |||||
"github.com/klauspost/cpuid" | |||||
) | |||||
// crc32sse returns a hash for the first 4 bytes of the slice | |||||
// len(a) must be >= 4. | |||||
//go:noescape | |||||
func crc32sse(a []byte) uint32 | |||||
// crc32sseAll calculates hashes for each 4-byte set in a. | |||||
// dst must be east len(a) - 4 in size. | |||||
// The size is not checked by the assembly. | |||||
//go:noescape | |||||
func crc32sseAll(a []byte, dst []uint32) | |||||
// matchLenSSE4 returns the number of matching bytes in a and b | |||||
// up to length 'max'. Both slices must be at least 'max' | |||||
// bytes in size. | |||||
// | |||||
// TODO: drop the "SSE4" name, since it doesn't use any SSE instructions. | |||||
// | |||||
//go:noescape | |||||
func matchLenSSE4(a, b []byte, max int) int | |||||
// histogram accumulates a histogram of b in h. | |||||
// h must be at least 256 entries in length, | |||||
// and must be cleared before calling this function. | |||||
//go:noescape | |||||
func histogram(b []byte, h []int32) | |||||
// Detect SSE 4.2 feature. | |||||
func init() { | |||||
useSSE42 = cpuid.CPU.SSE42() | |||||
} |
@ -1,213 +0,0 @@ | |||||
//+build !noasm | |||||
//+build !appengine | |||||
// Copyright 2015, Klaus Post, see LICENSE for details. | |||||
// func crc32sse(a []byte) uint32 | |||||
TEXT ·crc32sse(SB), 4, $0 | |||||
MOVQ a+0(FP), R10 | |||||
XORQ BX, BX | |||||
// CRC32 dword (R10), EBX | |||||
BYTE $0xF2; BYTE $0x41; BYTE $0x0f | |||||
BYTE $0x38; BYTE $0xf1; BYTE $0x1a | |||||
MOVL BX, ret+24(FP) | |||||
RET | |||||
// func crc32sseAll(a []byte, dst []uint32) | |||||
TEXT ·crc32sseAll(SB), 4, $0 | |||||
MOVQ a+0(FP), R8 // R8: src | |||||
MOVQ a_len+8(FP), R10 // input length | |||||
MOVQ dst+24(FP), R9 // R9: dst | |||||
SUBQ $4, R10 | |||||
JS end | |||||
JZ one_crc | |||||
MOVQ R10, R13 | |||||
SHRQ $2, R10 // len/4 | |||||
ANDQ $3, R13 // len&3 | |||||
XORQ BX, BX | |||||
ADDQ $1, R13 | |||||
TESTQ R10, R10 | |||||
JZ rem_loop | |||||
crc_loop: | |||||
MOVQ (R8), R11 | |||||
XORQ BX, BX | |||||
XORQ DX, DX | |||||
XORQ DI, DI | |||||
MOVQ R11, R12 | |||||
SHRQ $8, R11 | |||||
MOVQ R12, AX | |||||
MOVQ R11, CX | |||||
SHRQ $16, R12 | |||||
SHRQ $16, R11 | |||||
MOVQ R12, SI | |||||
// CRC32 EAX, EBX | |||||
BYTE $0xF2; BYTE $0x0f | |||||
BYTE $0x38; BYTE $0xf1; BYTE $0xd8 | |||||
// CRC32 ECX, EDX | |||||
BYTE $0xF2; BYTE $0x0f | |||||
BYTE $0x38; BYTE $0xf1; BYTE $0xd1 | |||||
// CRC32 ESI, EDI | |||||
BYTE $0xF2; BYTE $0x0f | |||||
BYTE $0x38; BYTE $0xf1; BYTE $0xfe | |||||
MOVL BX, (R9) | |||||
MOVL DX, 4(R9) | |||||
MOVL DI, 8(R9) | |||||
XORQ BX, BX | |||||
MOVL R11, AX | |||||
// CRC32 EAX, EBX | |||||
BYTE $0xF2; BYTE $0x0f | |||||
BYTE $0x38; BYTE $0xf1; BYTE $0xd8 | |||||
MOVL BX, 12(R9) | |||||
ADDQ $16, R9 | |||||
ADDQ $4, R8 | |||||
XORQ BX, BX | |||||
SUBQ $1, R10 | |||||
JNZ crc_loop | |||||
rem_loop: | |||||
MOVL (R8), AX | |||||
// CRC32 EAX, EBX | |||||
BYTE $0xF2; BYTE $0x0f | |||||
BYTE $0x38; BYTE $0xf1; BYTE $0xd8 | |||||
MOVL BX, (R9) | |||||
ADDQ $4, R9 | |||||
ADDQ $1, R8 | |||||
XORQ BX, BX | |||||
SUBQ $1, R13 | |||||
JNZ rem_loop | |||||
end: | |||||
RET | |||||
one_crc: | |||||
MOVQ $1, R13 | |||||
XORQ BX, BX | |||||
JMP rem_loop | |||||
// func matchLenSSE4(a, b []byte, max int) int | |||||
TEXT ·matchLenSSE4(SB), 4, $0 | |||||
MOVQ a_base+0(FP), SI | |||||
MOVQ b_base+24(FP), DI | |||||
MOVQ DI, DX | |||||
MOVQ max+48(FP), CX | |||||
cmp8: | |||||
// As long as we are 8 or more bytes before the end of max, we can load and | |||||
// compare 8 bytes at a time. If those 8 bytes are equal, repeat. | |||||
CMPQ CX, $8 | |||||
JLT cmp1 | |||||
MOVQ (SI), AX | |||||
MOVQ (DI), BX | |||||
CMPQ AX, BX | |||||
JNE bsf | |||||
ADDQ $8, SI | |||||
ADDQ $8, DI | |||||
SUBQ $8, CX | |||||
JMP cmp8 | |||||
bsf: | |||||
// If those 8 bytes were not equal, XOR the two 8 byte values, and return | |||||
// the index of the first byte that differs. The BSF instruction finds the | |||||
// least significant 1 bit, the amd64 architecture is little-endian, and | |||||
// the shift by 3 converts a bit index to a byte index. | |||||
XORQ AX, BX | |||||
BSFQ BX, BX | |||||
SHRQ $3, BX | |||||
ADDQ BX, DI | |||||
// Subtract off &b[0] to convert from &b[ret] to ret, and return. | |||||
SUBQ DX, DI | |||||
MOVQ DI, ret+56(FP) | |||||
RET | |||||
cmp1: | |||||
// In the slices' tail, compare 1 byte at a time. | |||||
CMPQ CX, $0 | |||||
JEQ matchLenEnd | |||||
MOVB (SI), AX | |||||
MOVB (DI), BX | |||||
CMPB AX, BX | |||||
JNE matchLenEnd | |||||
ADDQ $1, SI | |||||
ADDQ $1, DI | |||||
SUBQ $1, CX | |||||
JMP cmp1 | |||||
matchLenEnd: | |||||
// Subtract off &b[0] to convert from &b[ret] to ret, and return. | |||||
SUBQ DX, DI | |||||
MOVQ DI, ret+56(FP) | |||||
RET | |||||
// func histogram(b []byte, h []int32) | |||||
TEXT ·histogram(SB), 4, $0 | |||||
MOVQ b+0(FP), SI // SI: &b | |||||
MOVQ b_len+8(FP), R9 // R9: len(b) | |||||
MOVQ h+24(FP), DI // DI: Histogram | |||||
MOVQ R9, R8 | |||||
SHRQ $3, R8 | |||||
JZ hist1 | |||||
XORQ R11, R11 | |||||
loop_hist8: | |||||
MOVQ (SI), R10 | |||||
MOVB R10, R11 | |||||
INCL (DI)(R11*4) | |||||
SHRQ $8, R10 | |||||
MOVB R10, R11 | |||||
INCL (DI)(R11*4) | |||||
SHRQ $8, R10 | |||||
MOVB R10, R11 | |||||
INCL (DI)(R11*4) | |||||
SHRQ $8, R10 | |||||
MOVB R10, R11 | |||||
INCL (DI)(R11*4) | |||||
SHRQ $8, R10 | |||||
MOVB R10, R11 | |||||
INCL (DI)(R11*4) | |||||
SHRQ $8, R10 | |||||
MOVB R10, R11 | |||||
INCL (DI)(R11*4) | |||||
SHRQ $8, R10 | |||||
MOVB R10, R11 | |||||
INCL (DI)(R11*4) | |||||
SHRQ $8, R10 | |||||
INCL (DI)(R10*4) | |||||
ADDQ $8, SI | |||||
DECQ R8 | |||||
JNZ loop_hist8 | |||||
hist1: | |||||
ANDQ $7, R9 | |||||
JZ end_hist | |||||
XORQ R10, R10 | |||||
loop_hist1: | |||||
MOVB (SI), R10 | |||||
INCL (DI)(R10*4) | |||||
INCQ SI | |||||
DECQ R9 | |||||
JNZ loop_hist1 | |||||
end_hist: | |||||
RET |
@ -1,35 +0,0 @@ | |||||
//+build !amd64 noasm appengine | |||||
// Copyright 2015, Klaus Post, see LICENSE for details. | |||||
package flate | |||||
func init() { | |||||
useSSE42 = false | |||||
} | |||||
// crc32sse should never be called. | |||||
func crc32sse(a []byte) uint32 { | |||||
panic("no assembler") | |||||
} | |||||
// crc32sseAll should never be called. | |||||
func crc32sseAll(a []byte, dst []uint32) { | |||||
panic("no assembler") | |||||
} | |||||
// matchLenSSE4 should never be called. | |||||
func matchLenSSE4(a, b []byte, max int) int { | |||||
panic("no assembler") | |||||
return 0 | |||||
} | |||||
// histogram accumulates a histogram of b in h. | |||||
// | |||||
// len(h) must be >= 256, and h's elements must be all zeroes. | |||||
func histogram(b []byte, h []int32) { | |||||
h = h[:256] | |||||
for _, t := range b { | |||||
h[t]++ | |||||
} | |||||
} |
@ -0,0 +1,257 @@ | |||||
// Copyright 2011 The Snappy-Go Authors. All rights reserved. | |||||
// Modified for deflate by Klaus Post (c) 2015. | |||||
// Use of this source code is governed by a BSD-style | |||||
// license that can be found in the LICENSE file. | |||||
package flate | |||||
import ( | |||||
"fmt" | |||||
"math/bits" | |||||
) | |||||
type fastEnc interface { | |||||
Encode(dst *tokens, src []byte) | |||||
Reset() | |||||
} | |||||
func newFastEnc(level int) fastEnc { | |||||
switch level { | |||||
case 1: | |||||
return &fastEncL1{fastGen: fastGen{cur: maxStoreBlockSize}} | |||||
case 2: | |||||
return &fastEncL2{fastGen: fastGen{cur: maxStoreBlockSize}} | |||||
case 3: | |||||
return &fastEncL3{fastGen: fastGen{cur: maxStoreBlockSize}} | |||||
case 4: | |||||
return &fastEncL4{fastGen: fastGen{cur: maxStoreBlockSize}} | |||||
case 5: | |||||
return &fastEncL5{fastGen: fastGen{cur: maxStoreBlockSize}} | |||||
case 6: | |||||
return &fastEncL6{fastGen: fastGen{cur: maxStoreBlockSize}} | |||||
default: | |||||
panic("invalid level specified") | |||||
} | |||||
} | |||||
const ( | |||||
tableBits = 16 // Bits used in the table | |||||
tableSize = 1 << tableBits // Size of the table | |||||
tableShift = 32 - tableBits // Right-shift to get the tableBits most significant bits of a uint32. | |||||
baseMatchOffset = 1 // The smallest match offset | |||||
baseMatchLength = 3 // The smallest match length per the RFC section 3.2.5 | |||||
maxMatchOffset = 1 << 15 // The largest match offset | |||||
bTableBits = 18 // Bits used in the big tables | |||||
bTableSize = 1 << bTableBits // Size of the table | |||||
allocHistory = maxMatchOffset * 10 // Size to preallocate for history. | |||||
bufferReset = (1 << 31) - allocHistory - maxStoreBlockSize // Reset the buffer offset when reaching this. | |||||
) | |||||
const ( | |||||
prime3bytes = 506832829 | |||||
prime4bytes = 2654435761 | |||||
prime5bytes = 889523592379 | |||||
prime6bytes = 227718039650203 | |||||
prime7bytes = 58295818150454627 | |||||
prime8bytes = 0xcf1bbcdcb7a56463 | |||||
) | |||||
func load32(b []byte, i int) uint32 { | |||||
// Help the compiler eliminate bounds checks on the read so it can be done in a single read. | |||||
b = b[i:] | |||||
b = b[:4] | |||||
return uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24 | |||||
} | |||||
func load64(b []byte, i int) uint64 { | |||||
// Help the compiler eliminate bounds checks on the read so it can be done in a single read. | |||||
b = b[i:] | |||||
b = b[:8] | |||||
return uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 | | |||||
uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56 | |||||
} | |||||
func load3232(b []byte, i int32) uint32 { | |||||
// Help the compiler eliminate bounds checks on the read so it can be done in a single read. | |||||
b = b[i:] | |||||
b = b[:4] | |||||
return uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24 | |||||
} | |||||
func load6432(b []byte, i int32) uint64 { | |||||
// Help the compiler eliminate bounds checks on the read so it can be done in a single read. | |||||
b = b[i:] | |||||
b = b[:8] | |||||
return uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 | | |||||
uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56 | |||||
} | |||||
func hash(u uint32) uint32 { | |||||
return (u * 0x1e35a7bd) >> tableShift | |||||
} | |||||
type tableEntry struct { | |||||
val uint32 | |||||
offset int32 | |||||
} | |||||
// fastGen maintains the table for matches, | |||||
// and the previous byte block for level 2. | |||||
// This is the generic implementation. | |||||
type fastGen struct { | |||||
hist []byte | |||||
cur int32 | |||||
} | |||||
func (e *fastGen) addBlock(src []byte) int32 { | |||||
// check if we have space already | |||||
if len(e.hist)+len(src) > cap(e.hist) { | |||||
if cap(e.hist) == 0 { | |||||
e.hist = make([]byte, 0, allocHistory) | |||||
} else { | |||||
if cap(e.hist) < maxMatchOffset*2 { | |||||
panic("unexpected buffer size") | |||||
} | |||||
// Move down | |||||
offset := int32(len(e.hist)) - maxMatchOffset | |||||
copy(e.hist[0:maxMatchOffset], e.hist[offset:]) | |||||
e.cur += offset | |||||
e.hist = e.hist[:maxMatchOffset] | |||||
} | |||||
} | |||||
s := int32(len(e.hist)) | |||||
e.hist = append(e.hist, src...) | |||||
return s | |||||
} | |||||
// hash4 returns the hash of u to fit in a hash table with h bits. | |||||
// Preferably h should be a constant and should always be <32. | |||||
func hash4u(u uint32, h uint8) uint32 { | |||||
return (u * prime4bytes) >> ((32 - h) & 31) | |||||
} | |||||
type tableEntryPrev struct { | |||||
Cur tableEntry | |||||
Prev tableEntry | |||||
} | |||||
// hash4x64 returns the hash of the lowest 4 bytes of u to fit in a hash table with h bits. | |||||
// Preferably h should be a constant and should always be <32. | |||||
func hash4x64(u uint64, h uint8) uint32 { | |||||
return (uint32(u) * prime4bytes) >> ((32 - h) & 31) | |||||
} | |||||
// hash7 returns the hash of the lowest 7 bytes of u to fit in a hash table with h bits. | |||||
// Preferably h should be a constant and should always be <64. | |||||
func hash7(u uint64, h uint8) uint32 { | |||||
return uint32(((u << (64 - 56)) * prime7bytes) >> ((64 - h) & 63)) | |||||
} | |||||
// hash8 returns the hash of u to fit in a hash table with h bits. | |||||
// Preferably h should be a constant and should always be <64. | |||||
func hash8(u uint64, h uint8) uint32 { | |||||
return uint32((u * prime8bytes) >> ((64 - h) & 63)) | |||||
} | |||||
// hash6 returns the hash of the lowest 6 bytes of u to fit in a hash table with h bits. | |||||
// Preferably h should be a constant and should always be <64. | |||||
func hash6(u uint64, h uint8) uint32 { | |||||
return uint32(((u << (64 - 48)) * prime6bytes) >> ((64 - h) & 63)) | |||||
} | |||||
// matchlen will return the match length between offsets and t in src. | |||||
// The maximum length returned is maxMatchLength - 4. | |||||
// It is assumed that s > t, that t >=0 and s < len(src). | |||||
func (e *fastGen) matchlen(s, t int32, src []byte) int32 { | |||||
if debugDecode { | |||||
if t >= s { | |||||
panic(fmt.Sprint("t >=s:", t, s)) | |||||
} | |||||
if int(s) >= len(src) { | |||||
panic(fmt.Sprint("s >= len(src):", s, len(src))) | |||||
} | |||||
if t < 0 { | |||||
panic(fmt.Sprint("t < 0:", t)) | |||||
} | |||||
if s-t > maxMatchOffset { | |||||
panic(fmt.Sprint(s, "-", t, "(", s-t, ") > maxMatchLength (", maxMatchOffset, ")")) | |||||
} | |||||
} | |||||
s1 := int(s) + maxMatchLength - 4 | |||||
if s1 > len(src) { | |||||
s1 = len(src) | |||||
} | |||||
// Extend the match to be as long as possible. | |||||
return int32(matchLen(src[s:s1], src[t:])) | |||||
} | |||||
// matchlenLong will return the match length between offsets and t in src. | |||||
// It is assumed that s > t, that t >=0 and s < len(src). | |||||
func (e *fastGen) matchlenLong(s, t int32, src []byte) int32 { | |||||
if debugDecode { | |||||
if t >= s { | |||||
panic(fmt.Sprint("t >=s:", t, s)) | |||||
} | |||||
if int(s) >= len(src) { | |||||
panic(fmt.Sprint("s >= len(src):", s, len(src))) | |||||
} | |||||
if t < 0 { | |||||
panic(fmt.Sprint("t < 0:", t)) | |||||
} | |||||
if s-t > maxMatchOffset { | |||||
panic(fmt.Sprint(s, "-", t, "(", s-t, ") > maxMatchLength (", maxMatchOffset, ")")) | |||||
} | |||||
} | |||||
// Extend the match to be as long as possible. | |||||
return int32(matchLen(src[s:], src[t:])) | |||||
} | |||||
// Reset the encoding table. | |||||
func (e *fastGen) Reset() { | |||||
if cap(e.hist) < int(maxMatchOffset*8) { | |||||
l := maxMatchOffset * 8 | |||||
// Make it at least 1MB. | |||||
if l < 1<<20 { | |||||
l = 1 << 20 | |||||
} | |||||
e.hist = make([]byte, 0, l) | |||||
} | |||||
// We offset current position so everything will be out of reach | |||||
e.cur += maxMatchOffset + int32(len(e.hist)) | |||||
e.hist = e.hist[:0] | |||||
} | |||||
// matchLen returns the maximum length. | |||||
// 'a' must be the shortest of the two. | |||||
func matchLen(a, b []byte) int { | |||||
b = b[:len(a)] | |||||
var checked int | |||||
if len(a) > 4 { | |||||
// Try 4 bytes first | |||||
if diff := load32(a, 0) ^ load32(b, 0); diff != 0 { | |||||
return bits.TrailingZeros32(diff) >> 3 | |||||
} | |||||
// Switch to 8 byte matching. | |||||
checked = 4 | |||||
a = a[4:] | |||||
b = b[4:] | |||||
for len(a) >= 8 { | |||||
b = b[:len(a)] | |||||
if diff := load64(a, 0) ^ load64(b, 0); diff != 0 { | |||||
return checked + (bits.TrailingZeros64(diff) >> 3) | |||||
} | |||||
checked += 8 | |||||
a = a[8:] | |||||
b = b[8:] | |||||
} | |||||
} | |||||
b = b[:len(a)] | |||||
for i := range a { | |||||
if a[i] != b[i] { | |||||
return int(i) + checked | |||||
} | |||||
} | |||||
return len(a) + checked | |||||
} |
@ -0,0 +1,174 @@ | |||||
package flate | |||||
// fastGen maintains the table for matches, | |||||
// and the previous byte block for level 2. | |||||
// This is the generic implementation. | |||||
type fastEncL1 struct { | |||||
fastGen | |||||
table [tableSize]tableEntry | |||||
} | |||||
// EncodeL1 uses a similar algorithm to level 1 | |||||
func (e *fastEncL1) Encode(dst *tokens, src []byte) { | |||||
const ( | |||||
inputMargin = 12 - 1 | |||||
minNonLiteralBlockSize = 1 + 1 + inputMargin | |||||
) | |||||
// Protect against e.cur wraparound. | |||||
for e.cur >= bufferReset { | |||||
if len(e.hist) == 0 { | |||||
for i := range e.table[:] { | |||||
e.table[i] = tableEntry{} | |||||
} | |||||
e.cur = maxMatchOffset | |||||
break | |||||
} | |||||
// Shift down everything in the table that isn't already too far away. | |||||
minOff := e.cur + int32(len(e.hist)) - maxMatchOffset | |||||
for i := range e.table[:] { | |||||
v := e.table[i].offset | |||||
if v <= minOff { | |||||
v = 0 | |||||
} else { | |||||
v = v - e.cur + maxMatchOffset | |||||
} | |||||
e.table[i].offset = v | |||||
} | |||||
e.cur = maxMatchOffset | |||||
} | |||||
s := e.addBlock(src) | |||||
// This check isn't in the Snappy implementation, but there, the caller | |||||
// instead of the callee handles this case. | |||||
if len(src) < minNonLiteralBlockSize { | |||||
// We do not fill the token table. | |||||
// This will be picked up by caller. | |||||
dst.n = uint16(len(src)) | |||||
return | |||||
} | |||||
// Override src | |||||
src = e.hist | |||||
nextEmit := s | |||||
// sLimit is when to stop looking for offset/length copies. The inputMargin | |||||
// lets us use a fast path for emitLiteral in the main loop, while we are | |||||
// looking for copies. | |||||
sLimit := int32(len(src) - inputMargin) | |||||
// nextEmit is where in src the next emitLiteral should start from. | |||||
cv := load3232(src, s) | |||||
for { | |||||
const skipLog = 5 | |||||
const doEvery = 2 | |||||
nextS := s | |||||
var candidate tableEntry | |||||
for { | |||||
nextHash := hash(cv) | |||||
candidate = e.table[nextHash] | |||||
nextS = s + doEvery + (s-nextEmit)>>skipLog | |||||
if nextS > sLimit { | |||||
goto emitRemainder | |||||
} | |||||
now := load6432(src, nextS) | |||||
e.table[nextHash] = tableEntry{offset: s + e.cur, val: cv} | |||||
nextHash = hash(uint32(now)) | |||||
offset := s - (candidate.offset - e.cur) | |||||
if offset < maxMatchOffset && cv == candidate.val { | |||||
e.table[nextHash] = tableEntry{offset: nextS + e.cur, val: uint32(now)} | |||||
break | |||||
} | |||||
// Do one right away... | |||||
cv = uint32(now) | |||||
s = nextS | |||||
nextS++ | |||||
candidate = e.table[nextHash] | |||||
now >>= 8 | |||||
e.table[nextHash] = tableEntry{offset: s + e.cur, val: cv} | |||||
offset = s - (candidate.offset - e.cur) | |||||
if offset < maxMatchOffset && cv == candidate.val { | |||||
e.table[nextHash] = tableEntry{offset: nextS + e.cur, val: uint32(now)} | |||||
break | |||||
} | |||||
cv = uint32(now) | |||||
s = nextS | |||||
} | |||||
// A 4-byte match has been found. We'll later see if more than 4 bytes | |||||
// match. But, prior to the match, src[nextEmit:s] are unmatched. Emit | |||||
// them as literal bytes. | |||||
for { | |||||
// Invariant: we have a 4-byte match at s, and no need to emit any | |||||
// literal bytes prior to s. | |||||
// Extend the 4-byte match as long as possible. | |||||
t := candidate.offset - e.cur | |||||
l := e.matchlenLong(s+4, t+4, src) + 4 | |||||
// Extend backwards | |||||
for t > 0 && s > nextEmit && src[t-1] == src[s-1] { | |||||
s-- | |||||
t-- | |||||
l++ | |||||
} | |||||
if nextEmit < s { | |||||
emitLiteral(dst, src[nextEmit:s]) | |||||
} | |||||
// Save the match found | |||||
dst.AddMatchLong(l, uint32(s-t-baseMatchOffset)) | |||||
s += l | |||||
nextEmit = s | |||||
if nextS >= s { | |||||
s = nextS + 1 | |||||
} | |||||
if s >= sLimit { | |||||
// Index first pair after match end. | |||||
if int(s+l+4) < len(src) { | |||||
cv := load3232(src, s) | |||||
e.table[hash(cv)] = tableEntry{offset: s + e.cur, val: cv} | |||||
} | |||||
goto emitRemainder | |||||
} | |||||
// We could immediately start working at s now, but to improve | |||||
// compression we first update the hash table at s-2 and at s. If | |||||
// another emitCopy is not our next move, also calculate nextHash | |||||
// at s+1. At least on GOARCH=amd64, these three hash calculations | |||||
// are faster as one load64 call (with some shifts) instead of | |||||
// three load32 calls. | |||||
x := load6432(src, s-2) | |||||
o := e.cur + s - 2 | |||||
prevHash := hash(uint32(x)) | |||||
e.table[prevHash] = tableEntry{offset: o, val: uint32(x)} | |||||
x >>= 16 | |||||
currHash := hash(uint32(x)) | |||||
candidate = e.table[currHash] | |||||
e.table[currHash] = tableEntry{offset: o + 2, val: uint32(x)} | |||||
offset := s - (candidate.offset - e.cur) | |||||
if offset > maxMatchOffset || uint32(x) != candidate.val { | |||||
cv = uint32(x >> 8) | |||||
s++ | |||||
break | |||||
} | |||||
} | |||||
} | |||||
emitRemainder: | |||||
if int(nextEmit) < len(src) { | |||||
// If nothing was added, don't encode literals. | |||||
if dst.n == 0 { | |||||
return | |||||
} | |||||
emitLiteral(dst, src[nextEmit:]) | |||||
} | |||||
} |
@ -0,0 +1,199 @@ | |||||
package flate | |||||
// fastGen maintains the table for matches, | |||||
// and the previous byte block for level 2. | |||||
// This is the generic implementation. | |||||
type fastEncL2 struct { | |||||
fastGen | |||||
table [bTableSize]tableEntry | |||||
} | |||||
// EncodeL2 uses a similar algorithm to level 1, but is capable | |||||
// of matching across blocks giving better compression at a small slowdown. | |||||
func (e *fastEncL2) Encode(dst *tokens, src []byte) { | |||||
const ( | |||||
inputMargin = 12 - 1 | |||||
minNonLiteralBlockSize = 1 + 1 + inputMargin | |||||
) | |||||
// Protect against e.cur wraparound. | |||||
for e.cur >= bufferReset { | |||||
if len(e.hist) == 0 { | |||||
for i := range e.table[:] { | |||||
e.table[i] = tableEntry{} | |||||
} | |||||
e.cur = maxMatchOffset | |||||
break | |||||
} | |||||
// Shift down everything in the table that isn't already too far away. | |||||
minOff := e.cur + int32(len(e.hist)) - maxMatchOffset | |||||
for i := range e.table[:] { | |||||
v := e.table[i].offset | |||||
if v <= minOff { | |||||
v = 0 | |||||
} else { | |||||
v = v - e.cur + maxMatchOffset | |||||
} | |||||
e.table[i].offset = v | |||||
} | |||||
e.cur = maxMatchOffset | |||||
} | |||||
s := e.addBlock(src) | |||||
// This check isn't in the Snappy implementation, but there, the caller | |||||
// instead of the callee handles this case. | |||||
if len(src) < minNonLiteralBlockSize { | |||||
// We do not fill the token table. | |||||
// This will be picked up by caller. | |||||
dst.n = uint16(len(src)) | |||||
return | |||||
} | |||||
// Override src | |||||
src = e.hist | |||||
nextEmit := s | |||||
// sLimit is when to stop looking for offset/length copies. The inputMargin | |||||
// lets us use a fast path for emitLiteral in the main loop, while we are | |||||
// looking for copies. | |||||
sLimit := int32(len(src) - inputMargin) | |||||
// nextEmit is where in src the next emitLiteral should start from. | |||||
cv := load3232(src, s) | |||||
for { | |||||
// When should we start skipping if we haven't found matches in a long while. | |||||
const skipLog = 5 | |||||
const doEvery = 2 | |||||
nextS := s | |||||
var candidate tableEntry | |||||
for { | |||||
nextHash := hash4u(cv, bTableBits) | |||||
s = nextS | |||||
nextS = s + doEvery + (s-nextEmit)>>skipLog | |||||
if nextS > sLimit { | |||||
goto emitRemainder | |||||
} | |||||
candidate = e.table[nextHash] | |||||
now := load6432(src, nextS) | |||||
e.table[nextHash] = tableEntry{offset: s + e.cur, val: cv} | |||||
nextHash = hash4u(uint32(now), bTableBits) | |||||
offset := s - (candidate.offset - e.cur) | |||||
if offset < maxMatchOffset && cv == candidate.val { | |||||
e.table[nextHash] = tableEntry{offset: nextS + e.cur, val: uint32(now)} | |||||
break | |||||
} | |||||
// Do one right away... | |||||
cv = uint32(now) | |||||
s = nextS | |||||
nextS++ | |||||
candidate = e.table[nextHash] | |||||
now >>= 8 | |||||
e.table[nextHash] = tableEntry{offset: s + e.cur, val: cv} | |||||
offset = s - (candidate.offset - e.cur) | |||||
if offset < maxMatchOffset && cv == candidate.val { | |||||
break | |||||
} | |||||
cv = uint32(now) | |||||
} | |||||
// A 4-byte match has been found. We'll later see if more than 4 bytes | |||||
// match. But, prior to the match, src[nextEmit:s] are unmatched. Emit | |||||
// them as literal bytes. | |||||
// Call emitCopy, and then see if another emitCopy could be our next | |||||
// move. Repeat until we find no match for the input immediately after | |||||
// what was consumed by the last emitCopy call. | |||||
// | |||||
// If we exit this loop normally then we need to call emitLiteral next, | |||||
// though we don't yet know how big the literal will be. We handle that | |||||
// by proceeding to the next iteration of the main loop. We also can | |||||
// exit this loop via goto if we get close to exhausting the input. | |||||
for { | |||||
// Invariant: we have a 4-byte match at s, and no need to emit any | |||||
// literal bytes prior to s. | |||||
// Extend the 4-byte match as long as possible. | |||||
t := candidate.offset - e.cur | |||||
l := e.matchlenLong(s+4, t+4, src) + 4 | |||||
// Extend backwards | |||||
for t > 0 && s > nextEmit && src[t-1] == src[s-1] { | |||||
s-- | |||||
t-- | |||||
l++ | |||||
} | |||||
if nextEmit < s { | |||||
emitLiteral(dst, src[nextEmit:s]) | |||||
} | |||||
dst.AddMatchLong(l, uint32(s-t-baseMatchOffset)) | |||||
s += l | |||||
nextEmit = s | |||||
if nextS >= s { | |||||
s = nextS + 1 | |||||
} | |||||
if s >= sLimit { | |||||
// Index first pair after match end. | |||||
if int(s+l+4) < len(src) { | |||||
cv := load3232(src, s) | |||||
e.table[hash4u(cv, bTableBits)] = tableEntry{offset: s + e.cur, val: cv} | |||||
} | |||||
goto emitRemainder | |||||
} | |||||
// Store every second hash in-between, but offset by 1. | |||||
for i := s - l + 2; i < s-5; i += 7 { | |||||
x := load6432(src, int32(i)) | |||||
nextHash := hash4u(uint32(x), bTableBits) | |||||
e.table[nextHash] = tableEntry{offset: e.cur + i, val: uint32(x)} | |||||
// Skip one | |||||
x >>= 16 | |||||
nextHash = hash4u(uint32(x), bTableBits) | |||||
e.table[nextHash] = tableEntry{offset: e.cur + i + 2, val: uint32(x)} | |||||
// Skip one | |||||
x >>= 16 | |||||
nextHash = hash4u(uint32(x), bTableBits) | |||||
e.table[nextHash] = tableEntry{offset: e.cur + i + 4, val: uint32(x)} | |||||
} | |||||
// We could immediately start working at s now, but to improve | |||||
// compression we first update the hash table at s-2 to s. If | |||||
// another emitCopy is not our next move, also calculate nextHash | |||||
// at s+1. At least on GOARCH=amd64, these three hash calculations | |||||
// are faster as one load64 call (with some shifts) instead of | |||||
// three load32 calls. | |||||
x := load6432(src, s-2) | |||||
o := e.cur + s - 2 | |||||
prevHash := hash4u(uint32(x), bTableBits) | |||||
prevHash2 := hash4u(uint32(x>>8), bTableBits) | |||||
e.table[prevHash] = tableEntry{offset: o, val: uint32(x)} | |||||
e.table[prevHash2] = tableEntry{offset: o + 1, val: uint32(x >> 8)} | |||||
currHash := hash4u(uint32(x>>16), bTableBits) | |||||
candidate = e.table[currHash] | |||||
e.table[currHash] = tableEntry{offset: o + 2, val: uint32(x >> 16)} | |||||
offset := s - (candidate.offset - e.cur) | |||||
if offset > maxMatchOffset || uint32(x>>16) != candidate.val { | |||||
cv = uint32(x >> 24) | |||||
s++ | |||||
break | |||||
} | |||||
} | |||||
} | |||||
emitRemainder: | |||||
if int(nextEmit) < len(src) { | |||||
// If nothing was added, don't encode literals. | |||||
if dst.n == 0 { | |||||
return | |||||
} | |||||
emitLiteral(dst, src[nextEmit:]) | |||||
} | |||||
} |
@ -0,0 +1,225 @@ | |||||
package flate | |||||
// fastEncL3 | |||||
type fastEncL3 struct { | |||||
fastGen | |||||
table [tableSize]tableEntryPrev | |||||
} | |||||
// Encode uses a similar algorithm to level 2, will check up to two candidates. | |||||
func (e *fastEncL3) Encode(dst *tokens, src []byte) { | |||||
const ( | |||||
inputMargin = 8 - 1 | |||||
minNonLiteralBlockSize = 1 + 1 + inputMargin | |||||
) | |||||
// Protect against e.cur wraparound. | |||||
for e.cur >= bufferReset { | |||||
if len(e.hist) == 0 { | |||||
for i := range e.table[:] { | |||||
e.table[i] = tableEntryPrev{} | |||||
} | |||||
e.cur = maxMatchOffset | |||||
break | |||||
} | |||||
// Shift down everything in the table that isn't already too far away. | |||||
minOff := e.cur + int32(len(e.hist)) - maxMatchOffset | |||||
for i := range e.table[:] { | |||||
v := e.table[i] | |||||
if v.Cur.offset <= minOff { | |||||
v.Cur.offset = 0 | |||||
} else { | |||||
v.Cur.offset = v.Cur.offset - e.cur + maxMatchOffset | |||||
} | |||||
if v.Prev.offset <= minOff { | |||||
v.Prev.offset = 0 | |||||
} else { | |||||
v.Prev.offset = v.Prev.offset - e.cur + maxMatchOffset | |||||
} | |||||
e.table[i] = v | |||||
} | |||||
e.cur = maxMatchOffset | |||||
} | |||||
s := e.addBlock(src) | |||||
// Skip if too small. | |||||
if len(src) < minNonLiteralBlockSize { | |||||
// We do not fill the token table. | |||||
// This will be picked up by caller. | |||||
dst.n = uint16(len(src)) | |||||
return | |||||
} | |||||
// Override src | |||||
src = e.hist | |||||
nextEmit := s | |||||
// sLimit is when to stop looking for offset/length copies. The inputMargin | |||||
// lets us use a fast path for emitLiteral in the main loop, while we are | |||||
// looking for copies. | |||||
sLimit := int32(len(src) - inputMargin) | |||||
// nextEmit is where in src the next emitLiteral should start from. | |||||
cv := load3232(src, s) | |||||
for { | |||||
const skipLog = 6 | |||||
nextS := s | |||||
var candidate tableEntry | |||||
for { | |||||
nextHash := hash(cv) | |||||
s = nextS | |||||
nextS = s + 1 + (s-nextEmit)>>skipLog | |||||
if nextS > sLimit { | |||||
goto emitRemainder | |||||
} | |||||
candidates := e.table[nextHash] | |||||
now := load3232(src, nextS) | |||||
e.table[nextHash] = tableEntryPrev{Prev: candidates.Cur, Cur: tableEntry{offset: s + e.cur, val: cv}} | |||||
// Check both candidates | |||||
candidate = candidates.Cur | |||||
offset := s - (candidate.offset - e.cur) | |||||
if cv == candidate.val { | |||||
if offset > maxMatchOffset { | |||||
cv = now | |||||
// Previous will also be invalid, we have nothing. | |||||
continue | |||||
} | |||||
o2 := s - (candidates.Prev.offset - e.cur) | |||||
if cv != candidates.Prev.val || o2 > maxMatchOffset { | |||||
break | |||||
} | |||||
// Both match and are valid, pick longest. | |||||
l1, l2 := matchLen(src[s+4:], src[s-offset+4:]), matchLen(src[s+4:], src[s-o2+4:]) | |||||
if l2 > l1 { | |||||
candidate = candidates.Prev | |||||
} | |||||
break | |||||
} else { | |||||
// We only check if value mismatches. | |||||
// Offset will always be invalid in other cases. | |||||
candidate = candidates.Prev | |||||
if cv == candidate.val { | |||||
offset := s - (candidate.offset - e.cur) | |||||
if offset <= maxMatchOffset { | |||||
break | |||||
} | |||||
} | |||||
} | |||||
cv = now | |||||
} | |||||
// Call emitCopy, and then see if another emitCopy could be our next | |||||
// move. Repeat until we find no match for the input immediately after | |||||
// what was consumed by the last emitCopy call. | |||||
// | |||||
// If we exit this loop normally then we need to call emitLiteral next, | |||||
// though we don't yet know how big the literal will be. We handle that | |||||
// by proceeding to the next iteration of the main loop. We also can | |||||
// exit this loop via goto if we get close to exhausting the input. | |||||
for { | |||||
// Invariant: we have a 4-byte match at s, and no need to emit any | |||||
// literal bytes prior to s. | |||||
// Extend the 4-byte match as long as possible. | |||||
// | |||||
t := candidate.offset - e.cur | |||||
l := e.matchlenLong(s+4, t+4, src) + 4 | |||||
// Extend backwards | |||||
for t > 0 && s > nextEmit && src[t-1] == src[s-1] { | |||||
s-- | |||||
t-- | |||||
l++ | |||||
} | |||||
if nextEmit < s { | |||||
emitLiteral(dst, src[nextEmit:s]) | |||||
} | |||||
dst.AddMatchLong(l, uint32(s-t-baseMatchOffset)) | |||||
s += l | |||||
nextEmit = s | |||||
if nextS >= s { | |||||
s = nextS + 1 | |||||
} | |||||
if s >= sLimit { | |||||
t += l | |||||
// Index first pair after match end. | |||||
if int(t+4) < len(src) && t > 0 { | |||||
cv := load3232(src, t) | |||||
nextHash := hash(cv) | |||||
e.table[nextHash] = tableEntryPrev{ | |||||
Prev: e.table[nextHash].Cur, | |||||
Cur: tableEntry{offset: e.cur + t, val: cv}, | |||||
} | |||||
} | |||||
goto emitRemainder | |||||
} | |||||
// We could immediately start working at s now, but to improve | |||||
// compression we first update the hash table at s-3 to s. | |||||
x := load6432(src, s-3) | |||||
prevHash := hash(uint32(x)) | |||||
e.table[prevHash] = tableEntryPrev{ | |||||
Prev: e.table[prevHash].Cur, | |||||
Cur: tableEntry{offset: e.cur + s - 3, val: uint32(x)}, | |||||
} | |||||
x >>= 8 | |||||
prevHash = hash(uint32(x)) | |||||
e.table[prevHash] = tableEntryPrev{ | |||||
Prev: e.table[prevHash].Cur, | |||||
Cur: tableEntry{offset: e.cur + s - 2, val: uint32(x)}, | |||||
} | |||||
x >>= 8 | |||||
prevHash = hash(uint32(x)) | |||||
e.table[prevHash] = tableEntryPrev{ | |||||
Prev: e.table[prevHash].Cur, | |||||
Cur: tableEntry{offset: e.cur + s - 1, val: uint32(x)}, | |||||
} | |||||
x >>= 8 | |||||
currHash := hash(uint32(x)) | |||||
candidates := e.table[currHash] | |||||
cv = uint32(x) | |||||
e.table[currHash] = tableEntryPrev{ | |||||
Prev: candidates.Cur, | |||||
Cur: tableEntry{offset: s + e.cur, val: cv}, | |||||
} | |||||
// Check both candidates | |||||
candidate = candidates.Cur | |||||
if cv == candidate.val { | |||||
offset := s - (candidate.offset - e.cur) | |||||
if offset <= maxMatchOffset { | |||||
continue | |||||
} | |||||
} else { | |||||
// We only check if value mismatches. | |||||
// Offset will always be invalid in other cases. | |||||
candidate = candidates.Prev | |||||
if cv == candidate.val { | |||||
offset := s - (candidate.offset - e.cur) | |||||
if offset <= maxMatchOffset { | |||||
continue | |||||
} | |||||
} | |||||
} | |||||
cv = uint32(x >> 8) | |||||
s++ | |||||
break | |||||
} | |||||
} | |||||
emitRemainder: | |||||
if int(nextEmit) < len(src) { | |||||
// If nothing was added, don't encode literals. | |||||
if dst.n == 0 { | |||||
return | |||||
} | |||||
emitLiteral(dst, src[nextEmit:]) | |||||
} | |||||
} |
@ -0,0 +1,210 @@ | |||||
package flate | |||||
import "fmt" | |||||
type fastEncL4 struct { | |||||
fastGen | |||||
table [tableSize]tableEntry | |||||
bTable [tableSize]tableEntry | |||||
} | |||||
func (e *fastEncL4) Encode(dst *tokens, src []byte) { | |||||
const ( | |||||
inputMargin = 12 - 1 | |||||
minNonLiteralBlockSize = 1 + 1 + inputMargin | |||||
) | |||||
// Protect against e.cur wraparound. | |||||
for e.cur >= bufferReset { | |||||
if len(e.hist) == 0 { | |||||
for i := range e.table[:] { | |||||
e.table[i] = tableEntry{} | |||||
} | |||||
for i := range e.bTable[:] { | |||||
e.bTable[i] = tableEntry{} | |||||
} | |||||
e.cur = maxMatchOffset | |||||
break | |||||
} | |||||
// Shift down everything in the table that isn't already too far away. | |||||
minOff := e.cur + int32(len(e.hist)) - maxMatchOffset | |||||
for i := range e.table[:] { | |||||
v := e.table[i].offset | |||||
if v <= minOff { | |||||
v = 0 | |||||
} else { | |||||
v = v - e.cur + maxMatchOffset | |||||
} | |||||
e.table[i].offset = v | |||||
} | |||||
for i := range e.bTable[:] { | |||||
v := e.bTable[i].offset | |||||
if v <= minOff { | |||||
v = 0 | |||||
} else { | |||||
v = v - e.cur + maxMatchOffset | |||||
} | |||||
e.bTable[i].offset = v | |||||
} | |||||
e.cur = maxMatchOffset | |||||
} | |||||
s := e.addBlock(src) | |||||
// This check isn't in the Snappy implementation, but there, the caller | |||||
// instead of the callee handles this case. | |||||
if len(src) < minNonLiteralBlockSize { | |||||
// We do not fill the token table. | |||||
// This will be picked up by caller. | |||||
dst.n = uint16(len(src)) | |||||
return | |||||
} | |||||
// Override src | |||||
src = e.hist | |||||
nextEmit := s | |||||
// sLimit is when to stop looking for offset/length copies. The inputMargin | |||||
// lets us use a fast path for emitLiteral in the main loop, while we are | |||||
// looking for copies. | |||||
sLimit := int32(len(src) - inputMargin) | |||||
// nextEmit is where in src the next emitLiteral should start from. | |||||
cv := load6432(src, s) | |||||
for { | |||||
const skipLog = 6 | |||||
const doEvery = 1 | |||||
nextS := s | |||||
var t int32 | |||||
for { | |||||
nextHashS := hash4x64(cv, tableBits) | |||||
nextHashL := hash7(cv, tableBits) | |||||
s = nextS | |||||
nextS = s + doEvery + (s-nextEmit)>>skipLog | |||||
if nextS > sLimit { | |||||
goto emitRemainder | |||||
} | |||||
// Fetch a short+long candidate | |||||
sCandidate := e.table[nextHashS] | |||||
lCandidate := e.bTable[nextHashL] | |||||
next := load6432(src, nextS) | |||||
entry := tableEntry{offset: s + e.cur, val: uint32(cv)} | |||||
e.table[nextHashS] = entry | |||||
e.bTable[nextHashL] = entry | |||||
t = lCandidate.offset - e.cur | |||||
if s-t < maxMatchOffset && uint32(cv) == lCandidate.val { | |||||
// We got a long match. Use that. | |||||
break | |||||
} | |||||
t = sCandidate.offset - e.cur | |||||
if s-t < maxMatchOffset && uint32(cv) == sCandidate.val { | |||||
// Found a 4 match... | |||||
lCandidate = e.bTable[hash7(next, tableBits)] | |||||
// If the next long is a candidate, check if we should use that instead... | |||||
lOff := nextS - (lCandidate.offset - e.cur) | |||||
if lOff < maxMatchOffset && lCandidate.val == uint32(next) { | |||||
l1, l2 := matchLen(src[s+4:], src[t+4:]), matchLen(src[nextS+4:], src[nextS-lOff+4:]) | |||||
if l2 > l1 { | |||||
s = nextS | |||||
t = lCandidate.offset - e.cur | |||||
} | |||||
} | |||||
break | |||||
} | |||||
cv = next | |||||
} | |||||
// A 4-byte match has been found. We'll later see if more than 4 bytes | |||||
// match. But, prior to the match, src[nextEmit:s] are unmatched. Emit | |||||
// them as literal bytes. | |||||
// Extend the 4-byte match as long as possible. | |||||
l := e.matchlenLong(s+4, t+4, src) + 4 | |||||
// Extend backwards | |||||
for t > 0 && s > nextEmit && src[t-1] == src[s-1] { | |||||
s-- | |||||
t-- | |||||
l++ | |||||
} | |||||
if nextEmit < s { | |||||
emitLiteral(dst, src[nextEmit:s]) | |||||
} | |||||
if false { | |||||
if t >= s { | |||||
panic("s-t") | |||||
} | |||||
if (s - t) > maxMatchOffset { | |||||
panic(fmt.Sprintln("mmo", t)) | |||||
} | |||||
if l < baseMatchLength { | |||||
panic("bml") | |||||
} | |||||
} | |||||
dst.AddMatchLong(l, uint32(s-t-baseMatchOffset)) | |||||
s += l | |||||
nextEmit = s | |||||
if nextS >= s { | |||||
s = nextS + 1 | |||||
} | |||||
if s >= sLimit { | |||||
// Index first pair after match end. | |||||
if int(s+8) < len(src) { | |||||
cv := load6432(src, s) | |||||
e.table[hash4x64(cv, tableBits)] = tableEntry{offset: s + e.cur, val: uint32(cv)} | |||||
e.bTable[hash7(cv, tableBits)] = tableEntry{offset: s + e.cur, val: uint32(cv)} | |||||
} | |||||
goto emitRemainder | |||||
} | |||||
// Store every 3rd hash in-between | |||||
if true { | |||||
i := nextS | |||||
if i < s-1 { | |||||
cv := load6432(src, i) | |||||
t := tableEntry{offset: i + e.cur, val: uint32(cv)} | |||||
t2 := tableEntry{val: uint32(cv >> 8), offset: t.offset + 1} | |||||
e.bTable[hash7(cv, tableBits)] = t | |||||
e.bTable[hash7(cv>>8, tableBits)] = t2 | |||||
e.table[hash4u(t2.val, tableBits)] = t2 | |||||
i += 3 | |||||
for ; i < s-1; i += 3 { | |||||
cv := load6432(src, i) | |||||
t := tableEntry{offset: i + e.cur, val: uint32(cv)} | |||||
t2 := tableEntry{val: uint32(cv >> 8), offset: t.offset + 1} | |||||
e.bTable[hash7(cv, tableBits)] = t | |||||
e.bTable[hash7(cv>>8, tableBits)] = t2 | |||||
e.table[hash4u(t2.val, tableBits)] = t2 | |||||
} | |||||
} | |||||
} | |||||
// We could immediately start working at s now, but to improve | |||||
// compression we first update the hash table at s-1 and at s. | |||||
x := load6432(src, s-1) | |||||
o := e.cur + s - 1 | |||||
prevHashS := hash4x64(x, tableBits) | |||||
prevHashL := hash7(x, tableBits) | |||||
e.table[prevHashS] = tableEntry{offset: o, val: uint32(x)} | |||||
e.bTable[prevHashL] = tableEntry{offset: o, val: uint32(x)} | |||||
cv = x >> 8 | |||||
} | |||||
emitRemainder: | |||||
if int(nextEmit) < len(src) { | |||||
// If nothing was added, don't encode literals. | |||||
if dst.n == 0 { | |||||
return | |||||
} | |||||
emitLiteral(dst, src[nextEmit:]) | |||||
} | |||||
} |
@ -0,0 +1,276 @@ | |||||
package flate | |||||
import "fmt" | |||||
type fastEncL5 struct { | |||||
fastGen | |||||
table [tableSize]tableEntry | |||||
bTable [tableSize]tableEntryPrev | |||||
} | |||||
func (e *fastEncL5) Encode(dst *tokens, src []byte) { | |||||
const ( | |||||
inputMargin = 12 - 1 | |||||
minNonLiteralBlockSize = 1 + 1 + inputMargin | |||||
) | |||||
// Protect against e.cur wraparound. | |||||
for e.cur >= bufferReset { | |||||
if len(e.hist) == 0 { | |||||
for i := range e.table[:] { | |||||
e.table[i] = tableEntry{} | |||||
} | |||||
for i := range e.bTable[:] { | |||||
e.bTable[i] = tableEntryPrev{} | |||||
} | |||||
e.cur = maxMatchOffset | |||||
break | |||||
} | |||||
// Shift down everything in the table that isn't already too far away. | |||||
minOff := e.cur + int32(len(e.hist)) - maxMatchOffset | |||||
for i := range e.table[:] { | |||||
v := e.table[i].offset | |||||
if v <= minOff { | |||||
v = 0 | |||||
} else { | |||||
v = v - e.cur + maxMatchOffset | |||||
} | |||||
e.table[i].offset = v | |||||
} | |||||
for i := range e.bTable[:] { | |||||
v := e.bTable[i] | |||||
if v.Cur.offset <= minOff { | |||||
v.Cur.offset = 0 | |||||
v.Prev.offset = 0 | |||||
} else { | |||||
v.Cur.offset = v.Cur.offset - e.cur + maxMatchOffset | |||||
if v.Prev.offset <= minOff { | |||||
v.Prev.offset = 0 | |||||
} else { | |||||
v.Prev.offset = v.Prev.offset - e.cur + maxMatchOffset | |||||
} | |||||
} | |||||
e.bTable[i] = v | |||||
} | |||||
e.cur = maxMatchOffset | |||||
} | |||||
s := e.addBlock(src) | |||||
// This check isn't in the Snappy implementation, but there, the caller | |||||
// instead of the callee handles this case. | |||||
if len(src) < minNonLiteralBlockSize { | |||||
// We do not fill the token table. | |||||
// This will be picked up by caller. | |||||
dst.n = uint16(len(src)) | |||||
return | |||||
} | |||||
// Override src | |||||
src = e.hist | |||||
nextEmit := s | |||||
// sLimit is when to stop looking for offset/length copies. The inputMargin | |||||
// lets us use a fast path for emitLiteral in the main loop, while we are | |||||
// looking for copies. | |||||
sLimit := int32(len(src) - inputMargin) | |||||
// nextEmit is where in src the next emitLiteral should start from. | |||||
cv := load6432(src, s) | |||||
for { | |||||
const skipLog = 6 | |||||
const doEvery = 1 | |||||
nextS := s | |||||
var l int32 | |||||
var t int32 | |||||
for { | |||||
nextHashS := hash4x64(cv, tableBits) | |||||
nextHashL := hash7(cv, tableBits) | |||||
s = nextS | |||||
nextS = s + doEvery + (s-nextEmit)>>skipLog | |||||
if nextS > sLimit { | |||||
goto emitRemainder | |||||
} | |||||
// Fetch a short+long candidate | |||||
sCandidate := e.table[nextHashS] | |||||
lCandidate := e.bTable[nextHashL] | |||||
next := load6432(src, nextS) | |||||
entry := tableEntry{offset: s + e.cur, val: uint32(cv)} | |||||
e.table[nextHashS] = entry | |||||
eLong := &e.bTable[nextHashL] | |||||
eLong.Cur, eLong.Prev = entry, eLong.Cur | |||||
nextHashS = hash4x64(next, tableBits) | |||||
nextHashL = hash7(next, tableBits) | |||||
t = lCandidate.Cur.offset - e.cur | |||||
if s-t < maxMatchOffset { | |||||
if uint32(cv) == lCandidate.Cur.val { | |||||
// Store the next match | |||||
e.table[nextHashS] = tableEntry{offset: nextS + e.cur, val: uint32(next)} | |||||
eLong := &e.bTable[nextHashL] | |||||
eLong.Cur, eLong.Prev = tableEntry{offset: nextS + e.cur, val: uint32(next)}, eLong.Cur | |||||
t2 := lCandidate.Prev.offset - e.cur | |||||
if s-t2 < maxMatchOffset && uint32(cv) == lCandidate.Prev.val { | |||||
l = e.matchlen(s+4, t+4, src) + 4 | |||||
ml1 := e.matchlen(s+4, t2+4, src) + 4 | |||||
if ml1 > l { | |||||
t = t2 | |||||
l = ml1 | |||||
break | |||||
} | |||||
} | |||||
break | |||||
} | |||||
t = lCandidate.Prev.offset - e.cur | |||||
if s-t < maxMatchOffset && uint32(cv) == lCandidate.Prev.val { | |||||
// Store the next match | |||||
e.table[nextHashS] = tableEntry{offset: nextS + e.cur, val: uint32(next)} | |||||
eLong := &e.bTable[nextHashL] | |||||
eLong.Cur, eLong.Prev = tableEntry{offset: nextS + e.cur, val: uint32(next)}, eLong.Cur | |||||
break | |||||
} | |||||
} | |||||
t = sCandidate.offset - e.cur | |||||
if s-t < maxMatchOffset && uint32(cv) == sCandidate.val { | |||||
// Found a 4 match... | |||||
l = e.matchlen(s+4, t+4, src) + 4 | |||||
lCandidate = e.bTable[nextHashL] | |||||
// Store the next match | |||||
e.table[nextHashS] = tableEntry{offset: nextS + e.cur, val: uint32(next)} | |||||
eLong := &e.bTable[nextHashL] | |||||
eLong.Cur, eLong.Prev = tableEntry{offset: nextS + e.cur, val: uint32(next)}, eLong.Cur | |||||
// If the next long is a candidate, use that... | |||||
t2 := lCandidate.Cur.offset - e.cur | |||||
if nextS-t2 < maxMatchOffset { | |||||
if lCandidate.Cur.val == uint32(next) { | |||||
ml := e.matchlen(nextS+4, t2+4, src) + 4 | |||||
if ml > l { | |||||
t = t2 | |||||
s = nextS | |||||
l = ml | |||||
break | |||||
} | |||||
} | |||||
// If the previous long is a candidate, use that... | |||||
t2 = lCandidate.Prev.offset - e.cur | |||||
if nextS-t2 < maxMatchOffset && lCandidate.Prev.val == uint32(next) { | |||||
ml := e.matchlen(nextS+4, t2+4, src) + 4 | |||||
if ml > l { | |||||
t = t2 | |||||
s = nextS | |||||
l = ml | |||||
break | |||||
} | |||||
} | |||||
} | |||||
break | |||||
} | |||||
cv = next | |||||
} | |||||
// A 4-byte match has been found. We'll later see if more than 4 bytes | |||||
// match. But, prior to the match, src[nextEmit:s] are unmatched. Emit | |||||
// them as literal bytes. | |||||
// Extend the 4-byte match as long as possible. | |||||
if l == 0 { | |||||
l = e.matchlenLong(s+4, t+4, src) + 4 | |||||
} else if l == maxMatchLength { | |||||
l += e.matchlenLong(s+l, t+l, src) | |||||
} | |||||
// Extend backwards | |||||
for t > 0 && s > nextEmit && src[t-1] == src[s-1] { | |||||
s-- | |||||
t-- | |||||
l++ | |||||
} | |||||
if nextEmit < s { | |||||
emitLiteral(dst, src[nextEmit:s]) | |||||
} | |||||
if false { | |||||
if t >= s { | |||||
panic(fmt.Sprintln("s-t", s, t)) | |||||
} | |||||
if (s - t) > maxMatchOffset { | |||||
panic(fmt.Sprintln("mmo", s-t)) | |||||
} | |||||
if l < baseMatchLength { | |||||
panic("bml") | |||||
} | |||||
} | |||||
dst.AddMatchLong(l, uint32(s-t-baseMatchOffset)) | |||||
s += l | |||||
nextEmit = s | |||||
if nextS >= s { | |||||
s = nextS + 1 | |||||
} | |||||
if s >= sLimit { | |||||
goto emitRemainder | |||||
} | |||||
// Store every 3rd hash in-between. | |||||
if true { | |||||
const hashEvery = 3 | |||||
i := s - l + 1 | |||||
if i < s-1 { | |||||
cv := load6432(src, i) | |||||
t := tableEntry{offset: i + e.cur, val: uint32(cv)} | |||||
e.table[hash4x64(cv, tableBits)] = t | |||||
eLong := &e.bTable[hash7(cv, tableBits)] | |||||
eLong.Cur, eLong.Prev = t, eLong.Cur | |||||
// Do an long at i+1 | |||||
cv >>= 8 | |||||
t = tableEntry{offset: t.offset + 1, val: uint32(cv)} | |||||
eLong = &e.bTable[hash7(cv, tableBits)] | |||||
eLong.Cur, eLong.Prev = t, eLong.Cur | |||||
// We only have enough bits for a short entry at i+2 | |||||
cv >>= 8 | |||||
t = tableEntry{offset: t.offset + 1, val: uint32(cv)} | |||||
e.table[hash4x64(cv, tableBits)] = t | |||||
// Skip one - otherwise we risk hitting 's' | |||||
i += 4 | |||||
for ; i < s-1; i += hashEvery { | |||||
cv := load6432(src, i) | |||||
t := tableEntry{offset: i + e.cur, val: uint32(cv)} | |||||
t2 := tableEntry{offset: t.offset + 1, val: uint32(cv >> 8)} | |||||
eLong := &e.bTable[hash7(cv, tableBits)] | |||||
eLong.Cur, eLong.Prev = t, eLong.Cur | |||||
e.table[hash4u(t2.val, tableBits)] = t2 | |||||
} | |||||
} | |||||
} | |||||
// We could immediately start working at s now, but to improve | |||||
// compression we first update the hash table at s-1 and at s. | |||||
x := load6432(src, s-1) | |||||
o := e.cur + s - 1 | |||||
prevHashS := hash4x64(x, tableBits) | |||||
prevHashL := hash7(x, tableBits) | |||||
e.table[prevHashS] = tableEntry{offset: o, val: uint32(x)} | |||||
eLong := &e.bTable[prevHashL] | |||||
eLong.Cur, eLong.Prev = tableEntry{offset: o, val: uint32(x)}, eLong.Cur | |||||
cv = x >> 8 | |||||
} | |||||
emitRemainder: | |||||
if int(nextEmit) < len(src) { | |||||
// If nothing was added, don't encode literals. | |||||
if dst.n == 0 { | |||||
return | |||||
} | |||||
emitLiteral(dst, src[nextEmit:]) | |||||
} | |||||
} |
@ -0,0 +1,279 @@ | |||||
package flate | |||||
import "fmt" | |||||
type fastEncL6 struct { | |||||
fastGen | |||||
table [tableSize]tableEntry | |||||
bTable [tableSize]tableEntryPrev | |||||
} | |||||
func (e *fastEncL6) Encode(dst *tokens, src []byte) { | |||||
const ( | |||||
inputMargin = 12 - 1 | |||||
minNonLiteralBlockSize = 1 + 1 + inputMargin | |||||
) | |||||
// Protect against e.cur wraparound. | |||||
for e.cur >= bufferReset { | |||||
if len(e.hist) == 0 { | |||||
for i := range e.table[:] { | |||||
e.table[i] = tableEntry{} | |||||
} | |||||
for i := range e.bTable[:] { | |||||
e.bTable[i] = tableEntryPrev{} | |||||
} | |||||
e.cur = maxMatchOffset | |||||
break | |||||
} | |||||
// Shift down everything in the table that isn't already too far away. | |||||
minOff := e.cur + int32(len(e.hist)) - maxMatchOffset | |||||
for i := range e.table[:] { | |||||
v := e.table[i].offset | |||||
if v <= minOff { | |||||
v = 0 | |||||
} else { | |||||
v = v - e.cur + maxMatchOffset | |||||
} | |||||
e.table[i].offset = v | |||||
} | |||||
for i := range e.bTable[:] { | |||||
v := e.bTable[i] | |||||
if v.Cur.offset <= minOff { | |||||
v.Cur.offset = 0 | |||||
v.Prev.offset = 0 | |||||
} else { | |||||
v.Cur.offset = v.Cur.offset - e.cur + maxMatchOffset | |||||
if v.Prev.offset <= minOff { | |||||
v.Prev.offset = 0 | |||||
} else { | |||||
v.Prev.offset = v.Prev.offset - e.cur + maxMatchOffset | |||||
} | |||||
} | |||||
e.bTable[i] = v | |||||
} | |||||
e.cur = maxMatchOffset | |||||
} | |||||
s := e.addBlock(src) | |||||
// This check isn't in the Snappy implementation, but there, the caller | |||||
// instead of the callee handles this case. | |||||
if len(src) < minNonLiteralBlockSize { | |||||
// We do not fill the token table. | |||||
// This will be picked up by caller. | |||||
dst.n = uint16(len(src)) | |||||
return | |||||
} | |||||
// Override src | |||||
src = e.hist | |||||
nextEmit := s | |||||
// sLimit is when to stop looking for offset/length copies. The inputMargin | |||||
// lets us use a fast path for emitLiteral in the main loop, while we are | |||||
// looking for copies. | |||||
sLimit := int32(len(src) - inputMargin) | |||||
// nextEmit is where in src the next emitLiteral should start from. | |||||
cv := load6432(src, s) | |||||
// Repeat MUST be > 1 and within range | |||||
repeat := int32(1) | |||||
for { | |||||
const skipLog = 7 | |||||
const doEvery = 1 | |||||
nextS := s | |||||
var l int32 | |||||
var t int32 | |||||
for { | |||||
nextHashS := hash4x64(cv, tableBits) | |||||
nextHashL := hash7(cv, tableBits) | |||||
s = nextS | |||||
nextS = s + doEvery + (s-nextEmit)>>skipLog | |||||
if nextS > sLimit { | |||||
goto emitRemainder | |||||
} | |||||
// Fetch a short+long candidate | |||||
sCandidate := e.table[nextHashS] | |||||
lCandidate := e.bTable[nextHashL] | |||||
next := load6432(src, nextS) | |||||
entry := tableEntry{offset: s + e.cur, val: uint32(cv)} | |||||
e.table[nextHashS] = entry | |||||
eLong := &e.bTable[nextHashL] | |||||
eLong.Cur, eLong.Prev = entry, eLong.Cur | |||||
// Calculate hashes of 'next' | |||||
nextHashS = hash4x64(next, tableBits) | |||||
nextHashL = hash7(next, tableBits) | |||||
t = lCandidate.Cur.offset - e.cur | |||||
if s-t < maxMatchOffset { | |||||
if uint32(cv) == lCandidate.Cur.val { | |||||
// Long candidate matches at least 4 bytes. | |||||
// Store the next match | |||||
e.table[nextHashS] = tableEntry{offset: nextS + e.cur, val: uint32(next)} | |||||
eLong := &e.bTable[nextHashL] | |||||
eLong.Cur, eLong.Prev = tableEntry{offset: nextS + e.cur, val: uint32(next)}, eLong.Cur | |||||
// Check the previous long candidate as well. | |||||
t2 := lCandidate.Prev.offset - e.cur | |||||
if s-t2 < maxMatchOffset && uint32(cv) == lCandidate.Prev.val { | |||||
l = e.matchlen(s+4, t+4, src) + 4 | |||||
ml1 := e.matchlen(s+4, t2+4, src) + 4 | |||||
if ml1 > l { | |||||
t = t2 | |||||
l = ml1 | |||||
break | |||||
} | |||||
} | |||||
break | |||||
} | |||||
// Current value did not match, but check if previous long value does. | |||||
t = lCandidate.Prev.offset - e.cur | |||||
if s-t < maxMatchOffset && uint32(cv) == lCandidate.Prev.val { | |||||
// Store the next match | |||||
e.table[nextHashS] = tableEntry{offset: nextS + e.cur, val: uint32(next)} | |||||
eLong := &e.bTable[nextHashL] | |||||
eLong.Cur, eLong.Prev = tableEntry{offset: nextS + e.cur, val: uint32(next)}, eLong.Cur | |||||
break | |||||
} | |||||
} | |||||
t = sCandidate.offset - e.cur | |||||
if s-t < maxMatchOffset && uint32(cv) == sCandidate.val { | |||||
// Found a 4 match... | |||||
l = e.matchlen(s+4, t+4, src) + 4 | |||||
// Look up next long candidate (at nextS) | |||||
lCandidate = e.bTable[nextHashL] | |||||
// Store the next match | |||||
e.table[nextHashS] = tableEntry{offset: nextS + e.cur, val: uint32(next)} | |||||
eLong := &e.bTable[nextHashL] | |||||
eLong.Cur, eLong.Prev = tableEntry{offset: nextS + e.cur, val: uint32(next)}, eLong.Cur | |||||
// Check repeat at s + repOff | |||||
const repOff = 1 | |||||
t2 := s - repeat + repOff | |||||
if load3232(src, t2) == uint32(cv>>(8*repOff)) { | |||||
ml := e.matchlen(s+4+repOff, t2+4, src) + 4 | |||||
if ml > l { | |||||
t = t2 | |||||
l = ml | |||||
s += repOff | |||||
// Not worth checking more. | |||||
break | |||||
} | |||||
} | |||||
// If the next long is a candidate, use that... | |||||
t2 = lCandidate.Cur.offset - e.cur | |||||
if nextS-t2 < maxMatchOffset { | |||||
if lCandidate.Cur.val == uint32(next) { | |||||
ml := e.matchlen(nextS+4, t2+4, src) + 4 | |||||
if ml > l { | |||||
t = t2 | |||||
s = nextS | |||||
l = ml | |||||
// This is ok, but check previous as well. | |||||
} | |||||
} | |||||
// If the previous long is a candidate, use that... | |||||
t2 = lCandidate.Prev.offset - e.cur | |||||
if nextS-t2 < maxMatchOffset && lCandidate.Prev.val == uint32(next) { | |||||
ml := e.matchlen(nextS+4, t2+4, src) + 4 | |||||
if ml > l { | |||||
t = t2 | |||||
s = nextS | |||||
l = ml | |||||
break | |||||
} | |||||
} | |||||
} | |||||
break | |||||
} | |||||
cv = next | |||||
} | |||||
// A 4-byte match has been found. We'll later see if more than 4 bytes | |||||
// match. But, prior to the match, src[nextEmit:s] are unmatched. Emit | |||||
// them as literal bytes. | |||||
// Extend the 4-byte match as long as possible. | |||||
if l == 0 { | |||||
l = e.matchlenLong(s+4, t+4, src) + 4 | |||||
} else if l == maxMatchLength { | |||||
l += e.matchlenLong(s+l, t+l, src) | |||||
} | |||||
// Extend backwards | |||||
for t > 0 && s > nextEmit && src[t-1] == src[s-1] { | |||||
s-- | |||||
t-- | |||||
l++ | |||||
} | |||||
if nextEmit < s { | |||||
emitLiteral(dst, src[nextEmit:s]) | |||||
} | |||||
if false { | |||||
if t >= s { | |||||
panic(fmt.Sprintln("s-t", s, t)) | |||||
} | |||||
if (s - t) > maxMatchOffset { | |||||
panic(fmt.Sprintln("mmo", s-t)) | |||||
} | |||||
if l < baseMatchLength { | |||||
panic("bml") | |||||
} | |||||
} | |||||
dst.AddMatchLong(l, uint32(s-t-baseMatchOffset)) | |||||
repeat = s - t | |||||
s += l | |||||
nextEmit = s | |||||
if nextS >= s { | |||||
s = nextS + 1 | |||||
} | |||||
if s >= sLimit { | |||||
// Index after match end. | |||||
for i := nextS + 1; i < int32(len(src))-8; i += 2 { | |||||
cv := load6432(src, i) | |||||
e.table[hash4x64(cv, tableBits)] = tableEntry{offset: i + e.cur, val: uint32(cv)} | |||||
eLong := &e.bTable[hash7(cv, tableBits)] | |||||
eLong.Cur, eLong.Prev = tableEntry{offset: i + e.cur, val: uint32(cv)}, eLong.Cur | |||||
} | |||||
goto emitRemainder | |||||
} | |||||
// Store every long hash in-between and every second short. | |||||
if true { | |||||
for i := nextS + 1; i < s-1; i += 2 { | |||||
cv := load6432(src, i) | |||||
t := tableEntry{offset: i + e.cur, val: uint32(cv)} | |||||
t2 := tableEntry{offset: t.offset + 1, val: uint32(cv >> 8)} | |||||
eLong := &e.bTable[hash7(cv, tableBits)] | |||||
eLong2 := &e.bTable[hash7(cv>>8, tableBits)] | |||||
e.table[hash4x64(cv, tableBits)] = t | |||||
eLong.Cur, eLong.Prev = t, eLong.Cur | |||||
eLong2.Cur, eLong2.Prev = t2, eLong2.Cur | |||||
} | |||||
} | |||||
// We could immediately start working at s now, but to improve | |||||
// compression we first update the hash table at s-1 and at s. | |||||
cv = load6432(src, s) | |||||
} | |||||
emitRemainder: | |||||
if int(nextEmit) < len(src) { | |||||
// If nothing was added, don't encode literals. | |||||
if dst.n == 0 { | |||||
return | |||||
} | |||||
emitLiteral(dst, src[nextEmit:]) | |||||
} | |||||
} |
@ -1,48 +0,0 @@ | |||||
// Copyright 2009 The Go Authors. All rights reserved. | |||||
// Use of this source code is governed by a BSD-style | |||||
// license that can be found in the LICENSE file. | |||||
package flate | |||||
var reverseByte = [256]byte{ | |||||
0x00, 0x80, 0x40, 0xc0, 0x20, 0xa0, 0x60, 0xe0, | |||||
0x10, 0x90, 0x50, 0xd0, 0x30, 0xb0, 0x70, 0xf0, | |||||
0x08, 0x88, 0x48, 0xc8, 0x28, 0xa8, 0x68, 0xe8, | |||||
0x18, 0x98, 0x58, 0xd8, 0x38, 0xb8, 0x78, 0xf8, | |||||
0x04, 0x84, 0x44, 0xc4, 0x24, 0xa4, 0x64, 0xe4, | |||||
0x14, 0x94, 0x54, 0xd4, 0x34, 0xb4, 0x74, 0xf4, | |||||
0x0c, 0x8c, 0x4c, 0xcc, 0x2c, 0xac, 0x6c, 0xec, | |||||
0x1c, 0x9c, 0x5c, 0xdc, 0x3c, 0xbc, 0x7c, 0xfc, | |||||
0x02, 0x82, 0x42, 0xc2, 0x22, 0xa2, 0x62, 0xe2, | |||||
0x12, 0x92, 0x52, 0xd2, 0x32, 0xb2, 0x72, 0xf2, | |||||
0x0a, 0x8a, 0x4a, 0xca, 0x2a, 0xaa, 0x6a, 0xea, | |||||
0x1a, 0x9a, 0x5a, 0xda, 0x3a, 0xba, 0x7a, 0xfa, | |||||
0x06, 0x86, 0x46, 0xc6, 0x26, 0xa6, 0x66, 0xe6, | |||||
0x16, 0x96, 0x56, 0xd6, 0x36, 0xb6, 0x76, 0xf6, | |||||
0x0e, 0x8e, 0x4e, 0xce, 0x2e, 0xae, 0x6e, 0xee, | |||||
0x1e, 0x9e, 0x5e, 0xde, 0x3e, 0xbe, 0x7e, 0xfe, | |||||
0x01, 0x81, 0x41, 0xc1, 0x21, 0xa1, 0x61, 0xe1, | |||||
0x11, 0x91, 0x51, 0xd1, 0x31, 0xb1, 0x71, 0xf1, | |||||
0x09, 0x89, 0x49, 0xc9, 0x29, 0xa9, 0x69, 0xe9, | |||||
0x19, 0x99, 0x59, 0xd9, 0x39, 0xb9, 0x79, 0xf9, | |||||
0x05, 0x85, 0x45, 0xc5, 0x25, 0xa5, 0x65, 0xe5, | |||||
0x15, 0x95, 0x55, 0xd5, 0x35, 0xb5, 0x75, 0xf5, | |||||
0x0d, 0x8d, 0x4d, 0xcd, 0x2d, 0xad, 0x6d, 0xed, | |||||
0x1d, 0x9d, 0x5d, 0xdd, 0x3d, 0xbd, 0x7d, 0xfd, | |||||
0x03, 0x83, 0x43, 0xc3, 0x23, 0xa3, 0x63, 0xe3, | |||||
0x13, 0x93, 0x53, 0xd3, 0x33, 0xb3, 0x73, 0xf3, | |||||
0x0b, 0x8b, 0x4b, 0xcb, 0x2b, 0xab, 0x6b, 0xeb, | |||||
0x1b, 0x9b, 0x5b, 0xdb, 0x3b, 0xbb, 0x7b, 0xfb, | |||||
0x07, 0x87, 0x47, 0xc7, 0x27, 0xa7, 0x67, 0xe7, | |||||
0x17, 0x97, 0x57, 0xd7, 0x37, 0xb7, 0x77, 0xf7, | |||||
0x0f, 0x8f, 0x4f, 0xcf, 0x2f, 0xaf, 0x6f, 0xef, | |||||
0x1f, 0x9f, 0x5f, 0xdf, 0x3f, 0xbf, 0x7f, 0xff, | |||||
} | |||||
func reverseUint16(v uint16) uint16 { | |||||
return uint16(reverseByte[v>>8]) | uint16(reverseByte[v&0xFF])<<8 | |||||
} | |||||
func reverseBits(number uint16, bitLength byte) uint16 { | |||||
return reverseUint16(number << uint8(16-bitLength)) | |||||
} |
@ -1,856 +0,0 @@ | |||||
// Copyright 2011 The Snappy-Go Authors. All rights reserved. | |||||
// Modified for deflate by Klaus Post (c) 2015. | |||||
// Use of this source code is governed by a BSD-style | |||||
// license that can be found in the LICENSE file. | |||||
package flate | |||||
// emitLiteral writes a literal chunk and returns the number of bytes written. | |||||
func emitLiteral(dst *tokens, lit []byte) { | |||||
ol := int(dst.n) | |||||
for i, v := range lit { | |||||
dst.tokens[(i+ol)&maxStoreBlockSize] = token(v) | |||||
} | |||||
dst.n += uint16(len(lit)) | |||||
} | |||||
// emitCopy writes a copy chunk and returns the number of bytes written. | |||||
func emitCopy(dst *tokens, offset, length int) { | |||||
dst.tokens[dst.n] = matchToken(uint32(length-3), uint32(offset-minOffsetSize)) | |||||
dst.n++ | |||||
} | |||||
type snappyEnc interface { | |||||
Encode(dst *tokens, src []byte) | |||||
Reset() | |||||
} | |||||
func newSnappy(level int) snappyEnc { | |||||
switch level { | |||||
case 1: | |||||
return &snappyL1{} | |||||
case 2: | |||||
return &snappyL2{snappyGen: snappyGen{cur: maxStoreBlockSize, prev: make([]byte, 0, maxStoreBlockSize)}} | |||||
case 3: | |||||
return &snappyL3{snappyGen: snappyGen{cur: maxStoreBlockSize, prev: make([]byte, 0, maxStoreBlockSize)}} | |||||
case 4: | |||||
return &snappyL4{snappyL3{snappyGen: snappyGen{cur: maxStoreBlockSize, prev: make([]byte, 0, maxStoreBlockSize)}}} | |||||
default: | |||||
panic("invalid level specified") | |||||
} | |||||
} | |||||
const ( | |||||
tableBits = 14 // Bits used in the table | |||||
tableSize = 1 << tableBits // Size of the table | |||||
tableMask = tableSize - 1 // Mask for table indices. Redundant, but can eliminate bounds checks. | |||||
tableShift = 32 - tableBits // Right-shift to get the tableBits most significant bits of a uint32. | |||||
baseMatchOffset = 1 // The smallest match offset | |||||
baseMatchLength = 3 // The smallest match length per the RFC section 3.2.5 | |||||
maxMatchOffset = 1 << 15 // The largest match offset | |||||
) | |||||
func load32(b []byte, i int) uint32 { | |||||
b = b[i : i+4 : len(b)] // Help the compiler eliminate bounds checks on the next line. | |||||
return uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24 | |||||
} | |||||
func load64(b []byte, i int) uint64 { | |||||
b = b[i : i+8 : len(b)] // Help the compiler eliminate bounds checks on the next line. | |||||
return uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 | | |||||
uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56 | |||||
} | |||||
func hash(u uint32) uint32 { | |||||
return (u * 0x1e35a7bd) >> tableShift | |||||
} | |||||
// snappyL1 encapsulates level 1 compression | |||||
type snappyL1 struct{} | |||||
func (e *snappyL1) Reset() {} | |||||
func (e *snappyL1) Encode(dst *tokens, src []byte) { | |||||
const ( | |||||
inputMargin = 16 - 1 | |||||
minNonLiteralBlockSize = 1 + 1 + inputMargin | |||||
) | |||||
// This check isn't in the Snappy implementation, but there, the caller | |||||
// instead of the callee handles this case. | |||||
if len(src) < minNonLiteralBlockSize { | |||||
// We do not fill the token table. | |||||
// This will be picked up by caller. | |||||
dst.n = uint16(len(src)) | |||||
return | |||||
} | |||||
// Initialize the hash table. | |||||
// | |||||
// The table element type is uint16, as s < sLimit and sLimit < len(src) | |||||
// and len(src) <= maxStoreBlockSize and maxStoreBlockSize == 65535. | |||||
var table [tableSize]uint16 | |||||
// sLimit is when to stop looking for offset/length copies. The inputMargin | |||||
// lets us use a fast path for emitLiteral in the main loop, while we are | |||||
// looking for copies. | |||||
sLimit := len(src) - inputMargin | |||||
// nextEmit is where in src the next emitLiteral should start from. | |||||
nextEmit := 0 | |||||
// The encoded form must start with a literal, as there are no previous | |||||
// bytes to copy, so we start looking for hash matches at s == 1. | |||||
s := 1 | |||||
nextHash := hash(load32(src, s)) | |||||
for { | |||||
// Copied from the C++ snappy implementation: | |||||
// | |||||
// Heuristic match skipping: If 32 bytes are scanned with no matches | |||||
// found, start looking only at every other byte. If 32 more bytes are | |||||
// scanned (or skipped), look at every third byte, etc.. When a match | |||||
// is found, immediately go back to looking at every byte. This is a | |||||
// small loss (~5% performance, ~0.1% density) for compressible data | |||||
// due to more bookkeeping, but for non-compressible data (such as | |||||
// JPEG) it's a huge win since the compressor quickly "realizes" the | |||||
// data is incompressible and doesn't bother looking for matches | |||||
// everywhere. | |||||
// | |||||
// The "skip" variable keeps track of how many bytes there are since | |||||
// the last match; dividing it by 32 (ie. right-shifting by five) gives | |||||
// the number of bytes to move ahead for each iteration. | |||||
skip := 32 | |||||
nextS := s | |||||
candidate := 0 | |||||
for { | |||||
s = nextS | |||||
bytesBetweenHashLookups := skip >> 5 | |||||
nextS = s + bytesBetweenHashLookups | |||||
skip += bytesBetweenHashLookups | |||||
if nextS > sLimit { | |||||
goto emitRemainder | |||||
} | |||||
candidate = int(table[nextHash&tableMask]) | |||||
table[nextHash&tableMask] = uint16(s) | |||||
nextHash = hash(load32(src, nextS)) | |||||
// TODO: < should be <=, and add a test for that. | |||||
if s-candidate < maxMatchOffset && load32(src, s) == load32(src, candidate) { | |||||
break | |||||
} | |||||
} | |||||
// A 4-byte match has been found. We'll later see if more than 4 bytes | |||||
// match. But, prior to the match, src[nextEmit:s] are unmatched. Emit | |||||
// them as literal bytes. | |||||
emitLiteral(dst, src[nextEmit:s]) | |||||
// Call emitCopy, and then see if another emitCopy could be our next | |||||
// move. Repeat until we find no match for the input immediately after | |||||
// what was consumed by the last emitCopy call. | |||||
// | |||||
// If we exit this loop normally then we need to call emitLiteral next, | |||||
// though we don't yet know how big the literal will be. We handle that | |||||
// by proceeding to the next iteration of the main loop. We also can | |||||
// exit this loop via goto if we get close to exhausting the input. | |||||
for { | |||||
// Invariant: we have a 4-byte match at s, and no need to emit any | |||||
// literal bytes prior to s. | |||||
base := s | |||||
// Extend the 4-byte match as long as possible. | |||||
// | |||||
// This is an inlined version of Snappy's: | |||||
// s = extendMatch(src, candidate+4, s+4) | |||||
s += 4 | |||||
s1 := base + maxMatchLength | |||||
if s1 > len(src) { | |||||
s1 = len(src) | |||||
} | |||||
a := src[s:s1] | |||||
b := src[candidate+4:] | |||||
b = b[:len(a)] | |||||
l := len(a) | |||||
for i := range a { | |||||
if a[i] != b[i] { | |||||
l = i | |||||
break | |||||
} | |||||
} | |||||
s += l | |||||
// matchToken is flate's equivalent of Snappy's emitCopy. | |||||
dst.tokens[dst.n] = matchToken(uint32(s-base-baseMatchLength), uint32(base-candidate-baseMatchOffset)) | |||||
dst.n++ | |||||
nextEmit = s | |||||
if s >= sLimit { | |||||
goto emitRemainder | |||||
} | |||||
// We could immediately start working at s now, but to improve | |||||
// compression we first update the hash table at s-1 and at s. If | |||||
// another emitCopy is not our next move, also calculate nextHash | |||||
// at s+1. At least on GOARCH=amd64, these three hash calculations | |||||
// are faster as one load64 call (with some shifts) instead of | |||||
// three load32 calls. | |||||
x := load64(src, s-1) | |||||
prevHash := hash(uint32(x >> 0)) | |||||
table[prevHash&tableMask] = uint16(s - 1) | |||||
currHash := hash(uint32(x >> 8)) | |||||
candidate = int(table[currHash&tableMask]) | |||||
table[currHash&tableMask] = uint16(s) | |||||
// TODO: >= should be >, and add a test for that. | |||||
if s-candidate >= maxMatchOffset || uint32(x>>8) != load32(src, candidate) { | |||||
nextHash = hash(uint32(x >> 16)) | |||||
s++ | |||||
break | |||||
} | |||||
} | |||||
} | |||||
emitRemainder: | |||||
if nextEmit < len(src) { | |||||
emitLiteral(dst, src[nextEmit:]) | |||||
} | |||||
} | |||||
type tableEntry struct { | |||||
val uint32 | |||||
offset int32 | |||||
} | |||||
func load3232(b []byte, i int32) uint32 { | |||||
b = b[i : i+4 : len(b)] // Help the compiler eliminate bounds checks on the next line. | |||||
return uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24 | |||||
} | |||||
func load6432(b []byte, i int32) uint64 { | |||||
b = b[i : i+8 : len(b)] // Help the compiler eliminate bounds checks on the next line. | |||||
return uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 | | |||||
uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56 | |||||
} | |||||
// snappyGen maintains the table for matches, | |||||
// and the previous byte block for level 2. | |||||
// This is the generic implementation. | |||||
type snappyGen struct { | |||||
prev []byte | |||||
cur int32 | |||||
} | |||||
// snappyGen maintains the table for matches, | |||||
// and the previous byte block for level 2. | |||||
// This is the generic implementation. | |||||
type snappyL2 struct { | |||||
snappyGen | |||||
table [tableSize]tableEntry | |||||
} | |||||
// EncodeL2 uses a similar algorithm to level 1, but is capable | |||||
// of matching across blocks giving better compression at a small slowdown. | |||||
func (e *snappyL2) Encode(dst *tokens, src []byte) { | |||||
const ( | |||||
inputMargin = 16 - 1 | |||||
minNonLiteralBlockSize = 1 + 1 + inputMargin | |||||
) | |||||
// Ensure that e.cur doesn't wrap, mainly an issue on 32 bits. | |||||
if e.cur > 1<<30 { | |||||
for i := range e.table { | |||||
e.table[i] = tableEntry{} | |||||
} | |||||
e.cur = maxStoreBlockSize | |||||
} | |||||
// This check isn't in the Snappy implementation, but there, the caller | |||||
// instead of the callee handles this case. | |||||
if len(src) < minNonLiteralBlockSize { | |||||
// We do not fill the token table. | |||||
// This will be picked up by caller. | |||||
dst.n = uint16(len(src)) | |||||
e.cur += maxStoreBlockSize | |||||
e.prev = e.prev[:0] | |||||
return | |||||
} | |||||
// sLimit is when to stop looking for offset/length copies. The inputMargin | |||||
// lets us use a fast path for emitLiteral in the main loop, while we are | |||||
// looking for copies. | |||||
sLimit := int32(len(src) - inputMargin) | |||||
// nextEmit is where in src the next emitLiteral should start from. | |||||
nextEmit := int32(0) | |||||
s := int32(0) | |||||
cv := load3232(src, s) | |||||
nextHash := hash(cv) | |||||
for { | |||||
// Copied from the C++ snappy implementation: | |||||
// | |||||
// Heuristic match skipping: If 32 bytes are scanned with no matches | |||||
// found, start looking only at every other byte. If 32 more bytes are | |||||
// scanned (or skipped), look at every third byte, etc.. When a match | |||||
// is found, immediately go back to looking at every byte. This is a | |||||
// small loss (~5% performance, ~0.1% density) for compressible data | |||||
// due to more bookkeeping, but for non-compressible data (such as | |||||
// JPEG) it's a huge win since the compressor quickly "realizes" the | |||||
// data is incompressible and doesn't bother looking for matches | |||||
// everywhere. | |||||
// | |||||
// The "skip" variable keeps track of how many bytes there are since | |||||
// the last match; dividing it by 32 (ie. right-shifting by five) gives | |||||
// the number of bytes to move ahead for each iteration. | |||||
skip := int32(32) | |||||
nextS := s | |||||
var candidate tableEntry | |||||
for { | |||||
s = nextS | |||||
bytesBetweenHashLookups := skip >> 5 | |||||
nextS = s + bytesBetweenHashLookups | |||||
skip += bytesBetweenHashLookups | |||||
if nextS > sLimit { | |||||
goto emitRemainder | |||||
} | |||||
candidate = e.table[nextHash&tableMask] | |||||
now := load3232(src, nextS) | |||||
e.table[nextHash&tableMask] = tableEntry{offset: s + e.cur, val: cv} | |||||
nextHash = hash(now) | |||||
offset := s - (candidate.offset - e.cur) | |||||
if offset >= maxMatchOffset || cv != candidate.val { | |||||
// Out of range or not matched. | |||||
cv = now | |||||
continue | |||||
} | |||||
break | |||||
} | |||||
// A 4-byte match has been found. We'll later see if more than 4 bytes | |||||
// match. But, prior to the match, src[nextEmit:s] are unmatched. Emit | |||||
// them as literal bytes. | |||||
emitLiteral(dst, src[nextEmit:s]) | |||||
// Call emitCopy, and then see if another emitCopy could be our next | |||||
// move. Repeat until we find no match for the input immediately after | |||||
// what was consumed by the last emitCopy call. | |||||
// | |||||
// If we exit this loop normally then we need to call emitLiteral next, | |||||
// though we don't yet know how big the literal will be. We handle that | |||||
// by proceeding to the next iteration of the main loop. We also can | |||||
// exit this loop via goto if we get close to exhausting the input. | |||||
for { | |||||
// Invariant: we have a 4-byte match at s, and no need to emit any | |||||
// literal bytes prior to s. | |||||
// Extend the 4-byte match as long as possible. | |||||
// | |||||
s += 4 | |||||
t := candidate.offset - e.cur + 4 | |||||
l := e.matchlen(s, t, src) | |||||
// matchToken is flate's equivalent of Snappy's emitCopy. (length,offset) | |||||
dst.tokens[dst.n] = matchToken(uint32(l+4-baseMatchLength), uint32(s-t-baseMatchOffset)) | |||||
dst.n++ | |||||
s += l | |||||
nextEmit = s | |||||
if s >= sLimit { | |||||
goto emitRemainder | |||||
} | |||||
// We could immediately start working at s now, but to improve | |||||
// compression we first update the hash table at s-1 and at s. If | |||||
// another emitCopy is not our next move, also calculate nextHash | |||||
// at s+1. At least on GOARCH=amd64, these three hash calculations | |||||
// are faster as one load64 call (with some shifts) instead of | |||||
// three load32 calls. | |||||
x := load6432(src, s-1) | |||||
prevHash := hash(uint32(x)) | |||||
e.table[prevHash&tableMask] = tableEntry{offset: e.cur + s - 1, val: uint32(x)} | |||||
x >>= 8 | |||||
currHash := hash(uint32(x)) | |||||
candidate = e.table[currHash&tableMask] | |||||
e.table[currHash&tableMask] = tableEntry{offset: e.cur + s, val: uint32(x)} | |||||
offset := s - (candidate.offset - e.cur) | |||||
if offset >= maxMatchOffset || uint32(x) != candidate.val { | |||||
cv = uint32(x >> 8) | |||||
nextHash = hash(cv) | |||||
s++ | |||||
break | |||||
} | |||||
} | |||||
} | |||||
emitRemainder: | |||||
if int(nextEmit) < len(src) { | |||||
emitLiteral(dst, src[nextEmit:]) | |||||
} | |||||
e.cur += int32(len(src)) | |||||
e.prev = e.prev[:len(src)] | |||||
copy(e.prev, src) | |||||
} | |||||
type tableEntryPrev struct { | |||||
Cur tableEntry | |||||
Prev tableEntry | |||||
} | |||||
// snappyL3 | |||||
type snappyL3 struct { | |||||
snappyGen | |||||
table [tableSize]tableEntryPrev | |||||
} | |||||
// Encode uses a similar algorithm to level 2, will check up to two candidates. | |||||
func (e *snappyL3) Encode(dst *tokens, src []byte) { | |||||
const ( | |||||
inputMargin = 16 - 1 | |||||
minNonLiteralBlockSize = 1 + 1 + inputMargin | |||||
) | |||||
// Ensure that e.cur doesn't wrap, mainly an issue on 32 bits. | |||||
if e.cur > 1<<30 { | |||||
for i := range e.table { | |||||
e.table[i] = tableEntryPrev{} | |||||
} | |||||
e.cur = maxStoreBlockSize | |||||
} | |||||
// This check isn't in the Snappy implementation, but there, the caller | |||||
// instead of the callee handles this case. | |||||
if len(src) < minNonLiteralBlockSize { | |||||
// We do not fill the token table. | |||||
// This will be picked up by caller. | |||||
dst.n = uint16(len(src)) | |||||
e.cur += maxStoreBlockSize | |||||
e.prev = e.prev[:0] | |||||
return | |||||
} | |||||
// sLimit is when to stop looking for offset/length copies. The inputMargin | |||||
// lets us use a fast path for emitLiteral in the main loop, while we are | |||||
// looking for copies. | |||||
sLimit := int32(len(src) - inputMargin) | |||||
// nextEmit is where in src the next emitLiteral should start from. | |||||
nextEmit := int32(0) | |||||
s := int32(0) | |||||
cv := load3232(src, s) | |||||
nextHash := hash(cv) | |||||
for { | |||||
// Copied from the C++ snappy implementation: | |||||
// | |||||
// Heuristic match skipping: If 32 bytes are scanned with no matches | |||||
// found, start looking only at every other byte. If 32 more bytes are | |||||
// scanned (or skipped), look at every third byte, etc.. When a match | |||||
// is found, immediately go back to looking at every byte. This is a | |||||
// small loss (~5% performance, ~0.1% density) for compressible data | |||||
// due to more bookkeeping, but for non-compressible data (such as | |||||
// JPEG) it's a huge win since the compressor quickly "realizes" the | |||||
// data is incompressible and doesn't bother looking for matches | |||||
// everywhere. | |||||
// | |||||
// The "skip" variable keeps track of how many bytes there are since | |||||
// the last match; dividing it by 32 (ie. right-shifting by five) gives | |||||
// the number of bytes to move ahead for each iteration. | |||||
skip := int32(32) | |||||
nextS := s | |||||
var candidate tableEntry | |||||
for { | |||||
s = nextS | |||||
bytesBetweenHashLookups := skip >> 5 | |||||
nextS = s + bytesBetweenHashLookups | |||||
skip += bytesBetweenHashLookups | |||||
if nextS > sLimit { | |||||
goto emitRemainder | |||||
} | |||||
candidates := e.table[nextHash&tableMask] | |||||
now := load3232(src, nextS) | |||||
e.table[nextHash&tableMask] = tableEntryPrev{Prev: candidates.Cur, Cur: tableEntry{offset: s + e.cur, val: cv}} | |||||
nextHash = hash(now) | |||||
// Check both candidates | |||||
candidate = candidates.Cur | |||||
if cv == candidate.val { | |||||
offset := s - (candidate.offset - e.cur) | |||||
if offset < maxMatchOffset { | |||||
break | |||||
} | |||||
} else { | |||||
// We only check if value mismatches. | |||||
// Offset will always be invalid in other cases. | |||||
candidate = candidates.Prev | |||||
if cv == candidate.val { | |||||
offset := s - (candidate.offset - e.cur) | |||||
if offset < maxMatchOffset { | |||||
break | |||||
} | |||||
} | |||||
} | |||||
cv = now | |||||
} | |||||
// A 4-byte match has been found. We'll later see if more than 4 bytes | |||||
// match. But, prior to the match, src[nextEmit:s] are unmatched. Emit | |||||
// them as literal bytes. | |||||
emitLiteral(dst, src[nextEmit:s]) | |||||
// Call emitCopy, and then see if another emitCopy could be our next | |||||
// move. Repeat until we find no match for the input immediately after | |||||
// what was consumed by the last emitCopy call. | |||||
// | |||||
// If we exit this loop normally then we need to call emitLiteral next, | |||||
// though we don't yet know how big the literal will be. We handle that | |||||
// by proceeding to the next iteration of the main loop. We also can | |||||
// exit this loop via goto if we get close to exhausting the input. | |||||
for { | |||||
// Invariant: we have a 4-byte match at s, and no need to emit any | |||||
// literal bytes prior to s. | |||||
// Extend the 4-byte match as long as possible. | |||||
// | |||||
s += 4 | |||||
t := candidate.offset - e.cur + 4 | |||||
l := e.matchlen(s, t, src) | |||||
// matchToken is flate's equivalent of Snappy's emitCopy. (length,offset) | |||||
dst.tokens[dst.n] = matchToken(uint32(l+4-baseMatchLength), uint32(s-t-baseMatchOffset)) | |||||
dst.n++ | |||||
s += l | |||||
nextEmit = s | |||||
if s >= sLimit { | |||||
goto emitRemainder | |||||
} | |||||
// We could immediately start working at s now, but to improve | |||||
// compression we first update the hash table at s-2, s-1 and at s. If | |||||
// another emitCopy is not our next move, also calculate nextHash | |||||
// at s+1. At least on GOARCH=amd64, these three hash calculations | |||||
// are faster as one load64 call (with some shifts) instead of | |||||
// three load32 calls. | |||||
x := load6432(src, s-2) | |||||
prevHash := hash(uint32(x)) | |||||
e.table[prevHash&tableMask] = tableEntryPrev{ | |||||
Prev: e.table[prevHash&tableMask].Cur, | |||||
Cur: tableEntry{offset: e.cur + s - 2, val: uint32(x)}, | |||||
} | |||||
x >>= 8 | |||||
prevHash = hash(uint32(x)) | |||||
e.table[prevHash&tableMask] = tableEntryPrev{ | |||||
Prev: e.table[prevHash&tableMask].Cur, | |||||
Cur: tableEntry{offset: e.cur + s - 1, val: uint32(x)}, | |||||
} | |||||
x >>= 8 | |||||
currHash := hash(uint32(x)) | |||||
candidates := e.table[currHash&tableMask] | |||||
cv = uint32(x) | |||||
e.table[currHash&tableMask] = tableEntryPrev{ | |||||
Prev: candidates.Cur, | |||||
Cur: tableEntry{offset: s + e.cur, val: cv}, | |||||
} | |||||
// Check both candidates | |||||
candidate = candidates.Cur | |||||
if cv == candidate.val { | |||||
offset := s - (candidate.offset - e.cur) | |||||
if offset < maxMatchOffset { | |||||
continue | |||||
} | |||||
} else { | |||||
// We only check if value mismatches. | |||||
// Offset will always be invalid in other cases. | |||||
candidate = candidates.Prev | |||||
if cv == candidate.val { | |||||
offset := s - (candidate.offset - e.cur) | |||||
if offset < maxMatchOffset { | |||||
continue | |||||
} | |||||
} | |||||
} | |||||
cv = uint32(x >> 8) | |||||
nextHash = hash(cv) | |||||
s++ | |||||
break | |||||
} | |||||
} | |||||
emitRemainder: | |||||
if int(nextEmit) < len(src) { | |||||
emitLiteral(dst, src[nextEmit:]) | |||||
} | |||||
e.cur += int32(len(src)) | |||||
e.prev = e.prev[:len(src)] | |||||
copy(e.prev, src) | |||||
} | |||||
// snappyL4 | |||||
type snappyL4 struct { | |||||
snappyL3 | |||||
} | |||||
// Encode uses a similar algorithm to level 3, | |||||
// but will check up to two candidates if first isn't long enough. | |||||
func (e *snappyL4) Encode(dst *tokens, src []byte) { | |||||
const ( | |||||
inputMargin = 16 - 1 | |||||
minNonLiteralBlockSize = 1 + 1 + inputMargin | |||||
matchLenGood = 12 | |||||
) | |||||
// Ensure that e.cur doesn't wrap, mainly an issue on 32 bits. | |||||
if e.cur > 1<<30 { | |||||
for i := range e.table { | |||||
e.table[i] = tableEntryPrev{} | |||||
} | |||||
e.cur = maxStoreBlockSize | |||||
} | |||||
// This check isn't in the Snappy implementation, but there, the caller | |||||
// instead of the callee handles this case. | |||||
if len(src) < minNonLiteralBlockSize { | |||||
// We do not fill the token table. | |||||
// This will be picked up by caller. | |||||
dst.n = uint16(len(src)) | |||||
e.cur += maxStoreBlockSize | |||||
e.prev = e.prev[:0] | |||||
return | |||||
} | |||||
// sLimit is when to stop looking for offset/length copies. The inputMargin | |||||
// lets us use a fast path for emitLiteral in the main loop, while we are | |||||
// looking for copies. | |||||
sLimit := int32(len(src) - inputMargin) | |||||
// nextEmit is where in src the next emitLiteral should start from. | |||||
nextEmit := int32(0) | |||||
s := int32(0) | |||||
cv := load3232(src, s) | |||||
nextHash := hash(cv) | |||||
for { | |||||
// Copied from the C++ snappy implementation: | |||||
// | |||||
// Heuristic match skipping: If 32 bytes are scanned with no matches | |||||
// found, start looking only at every other byte. If 32 more bytes are | |||||
// scanned (or skipped), look at every third byte, etc.. When a match | |||||
// is found, immediately go back to looking at every byte. This is a | |||||
// small loss (~5% performance, ~0.1% density) for compressible data | |||||
// due to more bookkeeping, but for non-compressible data (such as | |||||
// JPEG) it's a huge win since the compressor quickly "realizes" the | |||||
// data is incompressible and doesn't bother looking for matches | |||||
// everywhere. | |||||
// | |||||
// The "skip" variable keeps track of how many bytes there are since | |||||
// the last match; dividing it by 32 (ie. right-shifting by five) gives | |||||
// the number of bytes to move ahead for each iteration. | |||||
skip := int32(32) | |||||
nextS := s | |||||
var candidate tableEntry | |||||
var candidateAlt tableEntry | |||||
for { | |||||
s = nextS | |||||
bytesBetweenHashLookups := skip >> 5 | |||||
nextS = s + bytesBetweenHashLookups | |||||
skip += bytesBetweenHashLookups | |||||
if nextS > sLimit { | |||||
goto emitRemainder | |||||
} | |||||
candidates := e.table[nextHash&tableMask] | |||||
now := load3232(src, nextS) | |||||
e.table[nextHash&tableMask] = tableEntryPrev{Prev: candidates.Cur, Cur: tableEntry{offset: s + e.cur, val: cv}} | |||||
nextHash = hash(now) | |||||
// Check both candidates | |||||
candidate = candidates.Cur | |||||
if cv == candidate.val { | |||||
offset := s - (candidate.offset - e.cur) | |||||
if offset < maxMatchOffset { | |||||
offset = s - (candidates.Prev.offset - e.cur) | |||||
if cv == candidates.Prev.val && offset < maxMatchOffset { | |||||
candidateAlt = candidates.Prev | |||||
} | |||||
break | |||||
} | |||||
} else { | |||||
// We only check if value mismatches. | |||||
// Offset will always be invalid in other cases. | |||||
candidate = candidates.Prev | |||||
if cv == candidate.val { | |||||
offset := s - (candidate.offset - e.cur) | |||||
if offset < maxMatchOffset { | |||||
break | |||||
} | |||||
} | |||||
} | |||||
cv = now | |||||
} | |||||
// A 4-byte match has been found. We'll later see if more than 4 bytes | |||||
// match. But, prior to the match, src[nextEmit:s] are unmatched. Emit | |||||
// them as literal bytes. | |||||
emitLiteral(dst, src[nextEmit:s]) | |||||
// Call emitCopy, and then see if another emitCopy could be our next | |||||
// move. Repeat until we find no match for the input immediately after | |||||
// what was consumed by the last emitCopy call. | |||||
// | |||||
// If we exit this loop normally then we need to call emitLiteral next, | |||||
// though we don't yet know how big the literal will be. We handle that | |||||
// by proceeding to the next iteration of the main loop. We also can | |||||
// exit this loop via goto if we get close to exhausting the input. | |||||
for { | |||||
// Invariant: we have a 4-byte match at s, and no need to emit any | |||||
// literal bytes prior to s. | |||||
// Extend the 4-byte match as long as possible. | |||||
// | |||||
s += 4 | |||||
t := candidate.offset - e.cur + 4 | |||||
l := e.matchlen(s, t, src) | |||||
// Try alternative candidate if match length < matchLenGood. | |||||
if l < matchLenGood-4 && candidateAlt.offset != 0 { | |||||
t2 := candidateAlt.offset - e.cur + 4 | |||||
l2 := e.matchlen(s, t2, src) | |||||
if l2 > l { | |||||
l = l2 | |||||
t = t2 | |||||
} | |||||
} | |||||
// matchToken is flate's equivalent of Snappy's emitCopy. (length,offset) | |||||
dst.tokens[dst.n] = matchToken(uint32(l+4-baseMatchLength), uint32(s-t-baseMatchOffset)) | |||||
dst.n++ | |||||
s += l | |||||
nextEmit = s | |||||
if s >= sLimit { | |||||
goto emitRemainder | |||||
} | |||||
// We could immediately start working at s now, but to improve | |||||
// compression we first update the hash table at s-2, s-1 and at s. If | |||||
// another emitCopy is not our next move, also calculate nextHash | |||||
// at s+1. At least on GOARCH=amd64, these three hash calculations | |||||
// are faster as one load64 call (with some shifts) instead of | |||||
// three load32 calls. | |||||
x := load6432(src, s-2) | |||||
prevHash := hash(uint32(x)) | |||||
e.table[prevHash&tableMask] = tableEntryPrev{ | |||||
Prev: e.table[prevHash&tableMask].Cur, | |||||
Cur: tableEntry{offset: e.cur + s - 2, val: uint32(x)}, | |||||
} | |||||
x >>= 8 | |||||
prevHash = hash(uint32(x)) | |||||
e.table[prevHash&tableMask] = tableEntryPrev{ | |||||
Prev: e.table[prevHash&tableMask].Cur, | |||||
Cur: tableEntry{offset: e.cur + s - 1, val: uint32(x)}, | |||||
} | |||||
x >>= 8 | |||||
currHash := hash(uint32(x)) | |||||
candidates := e.table[currHash&tableMask] | |||||
cv = uint32(x) | |||||
e.table[currHash&tableMask] = tableEntryPrev{ | |||||
Prev: candidates.Cur, | |||||
Cur: tableEntry{offset: s + e.cur, val: cv}, | |||||
} | |||||
// Check both candidates | |||||
candidate = candidates.Cur | |||||
candidateAlt = tableEntry{} | |||||
if cv == candidate.val { | |||||
offset := s - (candidate.offset - e.cur) | |||||
if offset < maxMatchOffset { | |||||
offset = s - (candidates.Prev.offset - e.cur) | |||||
if cv == candidates.Prev.val && offset < maxMatchOffset { | |||||
candidateAlt = candidates.Prev | |||||
} | |||||
continue | |||||
} | |||||
} else { | |||||
// We only check if value mismatches. | |||||
// Offset will always be invalid in other cases. | |||||
candidate = candidates.Prev | |||||
if cv == candidate.val { | |||||
offset := s - (candidate.offset - e.cur) | |||||
if offset < maxMatchOffset { | |||||
continue | |||||
} | |||||
} | |||||
} | |||||
cv = uint32(x >> 8) | |||||
nextHash = hash(cv) | |||||
s++ | |||||
break | |||||
} | |||||
} | |||||
emitRemainder: | |||||
if int(nextEmit) < len(src) { | |||||
emitLiteral(dst, src[nextEmit:]) | |||||
} | |||||
e.cur += int32(len(src)) | |||||
e.prev = e.prev[:len(src)] | |||||
copy(e.prev, src) | |||||
} | |||||
func (e *snappyGen) matchlen(s, t int32, src []byte) int32 { | |||||
s1 := int(s) + maxMatchLength - 4 | |||||
if s1 > len(src) { | |||||
s1 = len(src) | |||||
} | |||||
// If we are inside the current block | |||||
if t >= 0 { | |||||
b := src[t:] | |||||
a := src[s:s1] | |||||
b = b[:len(a)] | |||||
// Extend the match to be as long as possible. | |||||
for i := range a { | |||||
if a[i] != b[i] { | |||||
return int32(i) | |||||
} | |||||
} | |||||
return int32(len(a)) | |||||
} | |||||
// We found a match in the previous block. | |||||
tp := int32(len(e.prev)) + t | |||||
if tp < 0 { | |||||
return 0 | |||||
} | |||||
// Extend the match to be as long as possible. | |||||
a := src[s:s1] | |||||
b := e.prev[tp:] | |||||
if len(b) > len(a) { | |||||
b = b[:len(a)] | |||||
} | |||||
a = a[:len(b)] | |||||
for i := range b { | |||||
if a[i] != b[i] { | |||||
return int32(i) | |||||
} | |||||
} | |||||
n := int32(len(b)) | |||||
a = src[s+n : s1] | |||||
b = src[:len(a)] | |||||
for i := range a { | |||||
if a[i] != b[i] { | |||||
return int32(i) + n | |||||
} | |||||
} | |||||
return int32(len(a)) + n | |||||
} | |||||
// Reset the encoding table. | |||||
func (e *snappyGen) Reset() { | |||||
e.prev = e.prev[:0] | |||||
e.cur += maxMatchOffset + 1 | |||||
} |
@ -0,0 +1,252 @@ | |||||
package flate | |||||
import ( | |||||
"io" | |||||
"math" | |||||
) | |||||
const ( | |||||
maxStatelessBlock = math.MaxInt16 | |||||
slTableBits = 13 | |||||
slTableSize = 1 << slTableBits | |||||
slTableShift = 32 - slTableBits | |||||
) | |||||
type statelessWriter struct { | |||||
dst io.Writer | |||||
closed bool | |||||
} | |||||
func (s *statelessWriter) Close() error { | |||||
if s.closed { | |||||
return nil | |||||
} | |||||
s.closed = true | |||||
// Emit EOF block | |||||
return StatelessDeflate(s.dst, nil, true) | |||||
} | |||||
func (s *statelessWriter) Write(p []byte) (n int, err error) { | |||||
err = StatelessDeflate(s.dst, p, false) | |||||
if err != nil { | |||||
return 0, err | |||||
} | |||||
return len(p), nil | |||||
} | |||||
func (s *statelessWriter) Reset(w io.Writer) { | |||||
s.dst = w | |||||
s.closed = false | |||||
} | |||||
// NewStatelessWriter will do compression but without maintaining any state | |||||
// between Write calls. | |||||
// There will be no memory kept between Write calls, | |||||
// but compression and speed will be suboptimal. | |||||
// Because of this, the size of actual Write calls will affect output size. | |||||
func NewStatelessWriter(dst io.Writer) io.WriteCloser { | |||||
return &statelessWriter{dst: dst} | |||||
} | |||||
// StatelessDeflate allows to compress directly to a Writer without retaining state. | |||||
// When returning everything will be flushed. | |||||
func StatelessDeflate(out io.Writer, in []byte, eof bool) error { | |||||
var dst tokens | |||||
bw := newHuffmanBitWriter(out) | |||||
if eof && len(in) == 0 { | |||||
// Just write an EOF block. | |||||
// Could be faster... | |||||
bw.writeStoredHeader(0, true) | |||||
bw.flush() | |||||
return bw.err | |||||
} | |||||
for len(in) > 0 { | |||||
todo := in | |||||
if len(todo) > maxStatelessBlock { | |||||
todo = todo[:maxStatelessBlock] | |||||
} | |||||
in = in[len(todo):] | |||||
// Compress | |||||
statelessEnc(&dst, todo) | |||||
isEof := eof && len(in) == 0 | |||||
if dst.n == 0 { | |||||
bw.writeStoredHeader(len(todo), isEof) | |||||
if bw.err != nil { | |||||
return bw.err | |||||
} | |||||
bw.writeBytes(todo) | |||||
} else if int(dst.n) > len(todo)-len(todo)>>4 { | |||||
// If we removed less than 1/16th, huffman compress the block. | |||||
bw.writeBlockHuff(isEof, todo, false) | |||||
} else { | |||||
bw.writeBlockDynamic(&dst, isEof, todo, false) | |||||
} | |||||
if bw.err != nil { | |||||
return bw.err | |||||
} | |||||
dst.Reset() | |||||
} | |||||
if !eof { | |||||
// Align. | |||||
bw.writeStoredHeader(0, false) | |||||
} | |||||
bw.flush() | |||||
return bw.err | |||||
} | |||||
func hashSL(u uint32) uint32 { | |||||
return (u * 0x1e35a7bd) >> slTableShift | |||||
} | |||||
func load3216(b []byte, i int16) uint32 { | |||||
// Help the compiler eliminate bounds checks on the read so it can be done in a single read. | |||||
b = b[i:] | |||||
b = b[:4] | |||||
return uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24 | |||||
} | |||||
func load6416(b []byte, i int16) uint64 { | |||||
// Help the compiler eliminate bounds checks on the read so it can be done in a single read. | |||||
b = b[i:] | |||||
b = b[:8] | |||||
return uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 | | |||||
uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56 | |||||
} | |||||
func statelessEnc(dst *tokens, src []byte) { | |||||
const ( | |||||
inputMargin = 12 - 1 | |||||
minNonLiteralBlockSize = 1 + 1 + inputMargin | |||||
) | |||||
type tableEntry struct { | |||||
offset int16 | |||||
} | |||||
var table [slTableSize]tableEntry | |||||
// This check isn't in the Snappy implementation, but there, the caller | |||||
// instead of the callee handles this case. | |||||
if len(src) < minNonLiteralBlockSize { | |||||
// We do not fill the token table. | |||||
// This will be picked up by caller. | |||||
dst.n = uint16(len(src)) | |||||
return | |||||
} | |||||
s := int16(1) | |||||
nextEmit := int16(0) | |||||
// sLimit is when to stop looking for offset/length copies. The inputMargin | |||||
// lets us use a fast path for emitLiteral in the main loop, while we are | |||||
// looking for copies. | |||||
sLimit := int16(len(src) - inputMargin) | |||||
// nextEmit is where in src the next emitLiteral should start from. | |||||
cv := load3216(src, s) | |||||
for { | |||||
const skipLog = 5 | |||||
const doEvery = 2 | |||||
nextS := s | |||||
var candidate tableEntry | |||||
for { | |||||
nextHash := hashSL(cv) | |||||
candidate = table[nextHash] | |||||
nextS = s + doEvery + (s-nextEmit)>>skipLog | |||||
if nextS > sLimit || nextS <= 0 { | |||||
goto emitRemainder | |||||
} | |||||
now := load6416(src, nextS) | |||||
table[nextHash] = tableEntry{offset: s} | |||||
nextHash = hashSL(uint32(now)) | |||||
if cv == load3216(src, candidate.offset) { | |||||
table[nextHash] = tableEntry{offset: nextS} | |||||
break | |||||
} | |||||
// Do one right away... | |||||
cv = uint32(now) | |||||
s = nextS | |||||
nextS++ | |||||
candidate = table[nextHash] | |||||
now >>= 8 | |||||
table[nextHash] = tableEntry{offset: s} | |||||
if cv == load3216(src, candidate.offset) { | |||||
table[nextHash] = tableEntry{offset: nextS} | |||||
break | |||||
} | |||||
cv = uint32(now) | |||||
s = nextS | |||||
} | |||||
// A 4-byte match has been found. We'll later see if more than 4 bytes | |||||
// match. But, prior to the match, src[nextEmit:s] are unmatched. Emit | |||||
// them as literal bytes. | |||||
for { | |||||
// Invariant: we have a 4-byte match at s, and no need to emit any | |||||
// literal bytes prior to s. | |||||
// Extend the 4-byte match as long as possible. | |||||
t := candidate.offset | |||||
l := int16(matchLen(src[s+4:], src[t+4:]) + 4) | |||||
// Extend backwards | |||||
for t > 0 && s > nextEmit && src[t-1] == src[s-1] { | |||||
s-- | |||||
t-- | |||||
l++ | |||||
} | |||||
if nextEmit < s { | |||||
emitLiteral(dst, src[nextEmit:s]) | |||||
} | |||||
// Save the match found | |||||
dst.AddMatchLong(int32(l), uint32(s-t-baseMatchOffset)) | |||||
s += l | |||||
nextEmit = s | |||||
if nextS >= s { | |||||
s = nextS + 1 | |||||
} | |||||
if s >= sLimit { | |||||
goto emitRemainder | |||||
} | |||||
// We could immediately start working at s now, but to improve | |||||
// compression we first update the hash table at s-2 and at s. If | |||||
// another emitCopy is not our next move, also calculate nextHash | |||||
// at s+1. At least on GOARCH=amd64, these three hash calculations | |||||
// are faster as one load64 call (with some shifts) instead of | |||||
// three load32 calls. | |||||
x := load6416(src, s-2) | |||||
o := s - 2 | |||||
prevHash := hashSL(uint32(x)) | |||||
table[prevHash] = tableEntry{offset: o} | |||||
x >>= 16 | |||||
currHash := hashSL(uint32(x)) | |||||
candidate = table[currHash] | |||||
table[currHash] = tableEntry{offset: o + 2} | |||||
if uint32(x) != load3216(src, candidate.offset) { | |||||
cv = uint32(x >> 8) | |||||
s++ | |||||
break | |||||
} | |||||
} | |||||
} | |||||
emitRemainder: | |||||
if int(nextEmit) < len(src) { | |||||
// If nothing was added, don't encode literals. | |||||
if dst.n == 0 { | |||||
return | |||||
} | |||||
emitLiteral(dst, src[nextEmit:]) | |||||
} | |||||
} |
@ -1,24 +0,0 @@ | |||||
# Compiled Object files, Static and Dynamic libs (Shared Objects) | |||||
*.o | |||||
*.a | |||||
*.so | |||||
# Folders | |||||
_obj | |||||
_test | |||||
# Architecture specific extensions/prefixes | |||||
*.[568vq] | |||||
[568vq].out | |||||
*.cgo1.go | |||||
*.cgo2.c | |||||
_cgo_defun.c | |||||
_cgo_gotypes.go | |||||
_cgo_export.* | |||||
_testmain.go | |||||
*.exe | |||||
*.test | |||||
*.prof |
@ -1,8 +0,0 @@ | |||||
language: go | |||||
go: | |||||
- 1.3 | |||||
- 1.4 | |||||
- 1.5 | |||||
- 1.6 | |||||
- tip |
@ -1,22 +0,0 @@ | |||||
The MIT License (MIT) | |||||
Copyright (c) 2015 Klaus Post | |||||
Permission is hereby granted, free of charge, to any person obtaining a copy | |||||
of this software and associated documentation files (the "Software"), to deal | |||||
in the Software without restriction, including without limitation the rights | |||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell | |||||
copies of the Software, and to permit persons to whom the Software is | |||||
furnished to do so, subject to the following conditions: | |||||
The above copyright notice and this permission notice shall be included in all | |||||
copies or substantial portions of the Software. | |||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE | |||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | |||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, | |||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |||||
SOFTWARE. | |||||
@ -1,145 +0,0 @@ | |||||
# cpuid | |||||
Package cpuid provides information about the CPU running the current program. | |||||
CPU features are detected on startup, and kept for fast access through the life of the application. | |||||
Currently x86 / x64 (AMD64) is supported, and no external C (cgo) code is used, which should make the library very easy to use. | |||||
You can access the CPU information by accessing the shared CPU variable of the cpuid library. | |||||
Package home: https://github.com/klauspost/cpuid | |||||
[![GoDoc][1]][2] [![Build Status][3]][4] | |||||
[1]: https://godoc.org/github.com/klauspost/cpuid?status.svg | |||||
[2]: https://godoc.org/github.com/klauspost/cpuid | |||||
[3]: https://travis-ci.org/klauspost/cpuid.svg | |||||
[4]: https://travis-ci.org/klauspost/cpuid | |||||
# features | |||||
## CPU Instructions | |||||
* **CMOV** (i686 CMOV) | |||||
* **NX** (NX (No-Execute) bit) | |||||
* **AMD3DNOW** (AMD 3DNOW) | |||||
* **AMD3DNOWEXT** (AMD 3DNowExt) | |||||
* **MMX** (standard MMX) | |||||
* **MMXEXT** (SSE integer functions or AMD MMX ext) | |||||
* **SSE** (SSE functions) | |||||
* **SSE2** (P4 SSE functions) | |||||
* **SSE3** (Prescott SSE3 functions) | |||||
* **SSSE3** (Conroe SSSE3 functions) | |||||
* **SSE4** (Penryn SSE4.1 functions) | |||||
* **SSE4A** (AMD Barcelona microarchitecture SSE4a instructions) | |||||
* **SSE42** (Nehalem SSE4.2 functions) | |||||
* **AVX** (AVX functions) | |||||
* **AVX2** (AVX2 functions) | |||||
* **FMA3** (Intel FMA 3) | |||||
* **FMA4** (Bulldozer FMA4 functions) | |||||
* **XOP** (Bulldozer XOP functions) | |||||
* **F16C** (Half-precision floating-point conversion) | |||||
* **BMI1** (Bit Manipulation Instruction Set 1) | |||||
* **BMI2** (Bit Manipulation Instruction Set 2) | |||||
* **TBM** (AMD Trailing Bit Manipulation) | |||||
* **LZCNT** (LZCNT instruction) | |||||
* **POPCNT** (POPCNT instruction) | |||||
* **AESNI** (Advanced Encryption Standard New Instructions) | |||||
* **CLMUL** (Carry-less Multiplication) | |||||
* **HTT** (Hyperthreading (enabled)) | |||||
* **HLE** (Hardware Lock Elision) | |||||
* **RTM** (Restricted Transactional Memory) | |||||
* **RDRAND** (RDRAND instruction is available) | |||||
* **RDSEED** (RDSEED instruction is available) | |||||
* **ADX** (Intel ADX (Multi-Precision Add-Carry Instruction Extensions)) | |||||
* **SHA** (Intel SHA Extensions) | |||||
* **AVX512F** (AVX-512 Foundation) | |||||
* **AVX512DQ** (AVX-512 Doubleword and Quadword Instructions) | |||||
* **AVX512IFMA** (AVX-512 Integer Fused Multiply-Add Instructions) | |||||
* **AVX512PF** (AVX-512 Prefetch Instructions) | |||||
* **AVX512ER** (AVX-512 Exponential and Reciprocal Instructions) | |||||
* **AVX512CD** (AVX-512 Conflict Detection Instructions) | |||||
* **AVX512BW** (AVX-512 Byte and Word Instructions) | |||||
* **AVX512VL** (AVX-512 Vector Length Extensions) | |||||
* **AVX512VBMI** (AVX-512 Vector Bit Manipulation Instructions) | |||||
* **MPX** (Intel MPX (Memory Protection Extensions)) | |||||
* **ERMS** (Enhanced REP MOVSB/STOSB) | |||||
* **RDTSCP** (RDTSCP Instruction) | |||||
* **CX16** (CMPXCHG16B Instruction) | |||||
* **SGX** (Software Guard Extensions, with activation details) | |||||
## Performance | |||||
* **RDTSCP()** Returns current cycle count. Can be used for benchmarking. | |||||
* **SSE2SLOW** (SSE2 is supported, but usually not faster) | |||||
* **SSE3SLOW** (SSE3 is supported, but usually not faster) | |||||
* **ATOM** (Atom processor, some SSSE3 instructions are slower) | |||||
* **Cache line** (Probable size of a cache line). | |||||
* **L1, L2, L3 Cache size** on newer Intel/AMD CPUs. | |||||
## Cpu Vendor/VM | |||||
* **Intel** | |||||
* **AMD** | |||||
* **VIA** | |||||
* **Transmeta** | |||||
* **NSC** | |||||
* **KVM** (Kernel-based Virtual Machine) | |||||
* **MSVM** (Microsoft Hyper-V or Windows Virtual PC) | |||||
* **VMware** | |||||
* **XenHVM** | |||||
# installing | |||||
```go get github.com/klauspost/cpuid``` | |||||
# example | |||||
```Go | |||||
package main | |||||
import ( | |||||
"fmt" | |||||
"github.com/klauspost/cpuid" | |||||
) | |||||
func main() { | |||||
// Print basic CPU information: | |||||
fmt.Println("Name:", cpuid.CPU.BrandName) | |||||
fmt.Println("PhysicalCores:", cpuid.CPU.PhysicalCores) | |||||
fmt.Println("ThreadsPerCore:", cpuid.CPU.ThreadsPerCore) | |||||
fmt.Println("LogicalCores:", cpuid.CPU.LogicalCores) | |||||
fmt.Println("Family", cpuid.CPU.Family, "Model:", cpuid.CPU.Model) | |||||
fmt.Println("Features:", cpuid.CPU.Features) | |||||
fmt.Println("Cacheline bytes:", cpuid.CPU.CacheLine) | |||||
fmt.Println("L1 Data Cache:", cpuid.CPU.Cache.L1D, "bytes") | |||||
fmt.Println("L1 Instruction Cache:", cpuid.CPU.Cache.L1D, "bytes") | |||||
fmt.Println("L2 Cache:", cpuid.CPU.Cache.L2, "bytes") | |||||
fmt.Println("L3 Cache:", cpuid.CPU.Cache.L3, "bytes") | |||||
// Test if we have a specific feature: | |||||
if cpuid.CPU.SSE() { | |||||
fmt.Println("We have Streaming SIMD Extensions") | |||||
} | |||||
} | |||||
``` | |||||
Sample output: | |||||
``` | |||||
>go run main.go | |||||
Name: Intel(R) Core(TM) i5-2540M CPU @ 2.60GHz | |||||
PhysicalCores: 2 | |||||
ThreadsPerCore: 2 | |||||
LogicalCores: 4 | |||||
Family 6 Model: 42 | |||||
Features: CMOV,MMX,MMXEXT,SSE,SSE2,SSE3,SSSE3,SSE4.1,SSE4.2,AVX,AESNI,CLMUL | |||||
Cacheline bytes: 64 | |||||
We have Streaming SIMD Extensions | |||||
``` | |||||
# private package | |||||
In the "private" folder you can find an autogenerated version of the library you can include in your own packages. | |||||
For this purpose all exports are removed, and functions and constants are lowercased. | |||||
This is not a recommended way of using the library, but provided for convenience, if it is difficult for you to use external packages. | |||||
# license | |||||
This code is published under an MIT license. See LICENSE file for more information. |
@ -1,42 +0,0 @@ | |||||
// Copyright (c) 2015 Klaus Post, released under MIT License. See LICENSE file. | |||||
// +build 386,!gccgo | |||||
// func asmCpuid(op uint32) (eax, ebx, ecx, edx uint32) | |||||
TEXT ·asmCpuid(SB), 7, $0 | |||||
XORL CX, CX | |||||
MOVL op+0(FP), AX | |||||
CPUID | |||||
MOVL AX, eax+4(FP) | |||||
MOVL BX, ebx+8(FP) | |||||
MOVL CX, ecx+12(FP) | |||||
MOVL DX, edx+16(FP) | |||||
RET | |||||
// func asmCpuidex(op, op2 uint32) (eax, ebx, ecx, edx uint32) | |||||
TEXT ·asmCpuidex(SB), 7, $0 | |||||
MOVL op+0(FP), AX | |||||
MOVL op2+4(FP), CX | |||||
CPUID | |||||
MOVL AX, eax+8(FP) | |||||
MOVL BX, ebx+12(FP) | |||||
MOVL CX, ecx+16(FP) | |||||
MOVL DX, edx+20(FP) | |||||
RET | |||||
// func xgetbv(index uint32) (eax, edx uint32) | |||||
TEXT ·asmXgetbv(SB), 7, $0 | |||||
MOVL index+0(FP), CX | |||||
BYTE $0x0f; BYTE $0x01; BYTE $0xd0 // XGETBV | |||||
MOVL AX, eax+4(FP) | |||||
MOVL DX, edx+8(FP) | |||||
RET | |||||
// func asmRdtscpAsm() (eax, ebx, ecx, edx uint32) | |||||
TEXT ·asmRdtscpAsm(SB), 7, $0 | |||||
BYTE $0x0F; BYTE $0x01; BYTE $0xF9 // RDTSCP | |||||
MOVL AX, eax+0(FP) | |||||
MOVL BX, ebx+4(FP) | |||||
MOVL CX, ecx+8(FP) | |||||
MOVL DX, edx+12(FP) | |||||
RET |
@ -1,42 +0,0 @@ | |||||
// Copyright (c) 2015 Klaus Post, released under MIT License. See LICENSE file. | |||||
//+build amd64,!gccgo | |||||
// func asmCpuid(op uint32) (eax, ebx, ecx, edx uint32) | |||||
TEXT ·asmCpuid(SB), 7, $0 | |||||
XORQ CX, CX | |||||
MOVL op+0(FP), AX | |||||
CPUID | |||||
MOVL AX, eax+8(FP) | |||||
MOVL BX, ebx+12(FP) | |||||
MOVL CX, ecx+16(FP) | |||||
MOVL DX, edx+20(FP) | |||||
RET | |||||
// func asmCpuidex(op, op2 uint32) (eax, ebx, ecx, edx uint32) | |||||
TEXT ·asmCpuidex(SB), 7, $0 | |||||
MOVL op+0(FP), AX | |||||
MOVL op2+4(FP), CX | |||||
CPUID | |||||
MOVL AX, eax+8(FP) | |||||
MOVL BX, ebx+12(FP) | |||||
MOVL CX, ecx+16(FP) | |||||
MOVL DX, edx+20(FP) | |||||
RET | |||||
// func asmXgetbv(index uint32) (eax, edx uint32) | |||||
TEXT ·asmXgetbv(SB), 7, $0 | |||||
MOVL index+0(FP), CX | |||||
BYTE $0x0f; BYTE $0x01; BYTE $0xd0 // XGETBV | |||||
MOVL AX, eax+8(FP) | |||||
MOVL DX, edx+12(FP) | |||||
RET | |||||
// func asmRdtscpAsm() (eax, ebx, ecx, edx uint32) | |||||
TEXT ·asmRdtscpAsm(SB), 7, $0 | |||||
BYTE $0x0F; BYTE $0x01; BYTE $0xF9 // RDTSCP | |||||
MOVL AX, eax+0(FP) | |||||
MOVL BX, ebx+4(FP) | |||||
MOVL CX, ecx+8(FP) | |||||
MOVL DX, edx+12(FP) | |||||
RET |
@ -1,17 +0,0 @@ | |||||
// Copyright (c) 2015 Klaus Post, released under MIT License. See LICENSE file. | |||||
// +build 386,!gccgo amd64,!gccgo | |||||
package cpuid | |||||
func asmCpuid(op uint32) (eax, ebx, ecx, edx uint32) | |||||
func asmCpuidex(op, op2 uint32) (eax, ebx, ecx, edx uint32) | |||||
func asmXgetbv(index uint32) (eax, edx uint32) | |||||
func asmRdtscpAsm() (eax, ebx, ecx, edx uint32) | |||||
func initCPU() { | |||||
cpuid = asmCpuid | |||||
cpuidex = asmCpuidex | |||||
xgetbv = asmXgetbv | |||||
rdtscpAsm = asmRdtscpAsm | |||||
} |
@ -1,23 +0,0 @@ | |||||
// Copyright (c) 2015 Klaus Post, released under MIT License. See LICENSE file. | |||||
// +build !amd64,!386 gccgo | |||||
package cpuid | |||||
func initCPU() { | |||||
cpuid = func(op uint32) (eax, ebx, ecx, edx uint32) { | |||||
return 0, 0, 0, 0 | |||||
} | |||||
cpuidex = func(op, op2 uint32) (eax, ebx, ecx, edx uint32) { | |||||
return 0, 0, 0, 0 | |||||
} | |||||
xgetbv = func(index uint32) (eax, edx uint32) { | |||||
return 0, 0 | |||||
} | |||||
rdtscpAsm = func() (eax, ebx, ecx, edx uint32) { | |||||
return 0, 0, 0, 0 | |||||
} | |||||
} |
@ -1,3 +0,0 @@ | |||||
package cpuid | |||||
//go:generate go run private-gen.go |
@ -1,24 +0,0 @@ | |||||
# Compiled Object files, Static and Dynamic libs (Shared Objects) | |||||
*.o | |||||
*.a | |||||
*.so | |||||
# Folders | |||||
_obj | |||||
_test | |||||
# Architecture specific extensions/prefixes | |||||
*.[568vq] | |||||
[568vq].out | |||||
*.cgo1.go | |||||
*.cgo2.c | |||||
_cgo_defun.c | |||||
_cgo_gotypes.go | |||||
_cgo_export.* | |||||
_testmain.go | |||||
*.exe | |||||
*.test | |||||
*.prof |
@ -1,13 +0,0 @@ | |||||
language: go | |||||
go: | |||||
- 1.3 | |||||
- 1.4 | |||||
- 1.5 | |||||
- 1.6 | |||||
- 1.7 | |||||
- tip | |||||
script: | |||||
- go test -v . | |||||
- go test -v -race . |
@ -1,28 +0,0 @@ | |||||
Copyright (c) 2012 The Go Authors. All rights reserved. | |||||
Copyright (c) 2015 Klaus Post | |||||
Redistribution and use in source and binary forms, with or without | |||||
modification, are permitted provided that the following conditions are | |||||
met: | |||||
* Redistributions of source code must retain the above copyright | |||||
notice, this list of conditions and the following disclaimer. | |||||
* Redistributions in binary form must reproduce the above | |||||
copyright notice, this list of conditions and the following disclaimer | |||||
in the documentation and/or other materials provided with the | |||||
distribution. | |||||
* Neither the name of Google Inc. nor the names of its | |||||
contributors may be used to endorse or promote products derived from | |||||
this software without specific prior written permission. | |||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | |||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | |||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | |||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | |||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | |||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | |||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | |||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | |||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | |||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | |||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
@ -1,87 +0,0 @@ | |||||
# crc32 | |||||
CRC32 hash with x64 optimizations | |||||
This package is a drop-in replacement for the standard library `hash/crc32` package, that features SSE 4.2 optimizations on x64 platforms, for a 10x speedup. | |||||
[![Build Status](https://travis-ci.org/klauspost/crc32.svg?branch=master)](https://travis-ci.org/klauspost/crc32) | |||||
# usage | |||||
Install using `go get github.com/klauspost/crc32`. This library is based on Go 1.5 code and requires Go 1.3 or newer. | |||||
Replace `import "hash/crc32"` with `import "github.com/klauspost/crc32"` and you are good to go. | |||||
# changes | |||||
* Oct 20, 2016: Changes have been merged to upstream Go. Package updated to match. | |||||
* Dec 4, 2015: Uses the "slice-by-8" trick more extensively, which gives a 1.5 to 2.5x speedup if assembler is unavailable. | |||||
# performance | |||||
For *Go 1.7* performance is equivalent to the standard library. So if you use this package for Go 1.7 you can switch back. | |||||
For IEEE tables (the most common), there is approximately a factor 10 speedup with "CLMUL" (Carryless multiplication) instruction: | |||||
``` | |||||
benchmark old ns/op new ns/op delta | |||||
BenchmarkCrc32KB 99955 10258 -89.74% | |||||
benchmark old MB/s new MB/s speedup | |||||
BenchmarkCrc32KB 327.83 3194.20 9.74x | |||||
``` | |||||
For other tables and "CLMUL" capable machines the performance is the same as the standard library. | |||||
Here are some detailed benchmarks, comparing to go 1.5 standard library with and without assembler enabled. | |||||
``` | |||||
Std: Standard Go 1.5 library | |||||
Crc: Indicates IEEE type CRC. | |||||
40B: Size of each slice encoded. | |||||
NoAsm: Assembler was disabled (ie. not an AMD64 or SSE 4.2+ capable machine). | |||||
Castagnoli: Castagnoli CRC type. | |||||
BenchmarkStdCrc40B-4 10000000 158 ns/op 252.88 MB/s | |||||
BenchmarkCrc40BNoAsm-4 20000000 105 ns/op 377.38 MB/s (slice8) | |||||
BenchmarkCrc40B-4 20000000 105 ns/op 378.77 MB/s (slice8) | |||||
BenchmarkStdCrc1KB-4 500000 3604 ns/op 284.10 MB/s | |||||
BenchmarkCrc1KBNoAsm-4 1000000 1463 ns/op 699.79 MB/s (slice8) | |||||
BenchmarkCrc1KB-4 3000000 396 ns/op 2583.69 MB/s (asm) | |||||
BenchmarkStdCrc8KB-4 200000 11417 ns/op 717.48 MB/s (slice8) | |||||
BenchmarkCrc8KBNoAsm-4 200000 11317 ns/op 723.85 MB/s (slice8) | |||||
BenchmarkCrc8KB-4 500000 2919 ns/op 2805.73 MB/s (asm) | |||||
BenchmarkStdCrc32KB-4 30000 45749 ns/op 716.24 MB/s (slice8) | |||||
BenchmarkCrc32KBNoAsm-4 30000 45109 ns/op 726.42 MB/s (slice8) | |||||
BenchmarkCrc32KB-4 100000 11497 ns/op 2850.09 MB/s (asm) | |||||
BenchmarkStdNoAsmCastagnol40B-4 10000000 161 ns/op 246.94 MB/s | |||||
BenchmarkStdCastagnoli40B-4 50000000 28.4 ns/op 1410.69 MB/s (asm) | |||||
BenchmarkCastagnoli40BNoAsm-4 20000000 100 ns/op 398.01 MB/s (slice8) | |||||
BenchmarkCastagnoli40B-4 50000000 28.2 ns/op 1419.54 MB/s (asm) | |||||
BenchmarkStdNoAsmCastagnoli1KB-4 500000 3622 ns/op 282.67 MB/s | |||||
BenchmarkStdCastagnoli1KB-4 10000000 144 ns/op 7099.78 MB/s (asm) | |||||
BenchmarkCastagnoli1KBNoAsm-4 1000000 1475 ns/op 694.14 MB/s (slice8) | |||||
BenchmarkCastagnoli1KB-4 10000000 146 ns/op 6993.35 MB/s (asm) | |||||
BenchmarkStdNoAsmCastagnoli8KB-4 50000 28781 ns/op 284.63 MB/s | |||||
BenchmarkStdCastagnoli8KB-4 1000000 1029 ns/op 7957.89 MB/s (asm) | |||||
BenchmarkCastagnoli8KBNoAsm-4 200000 11410 ns/op 717.94 MB/s (slice8) | |||||
BenchmarkCastagnoli8KB-4 1000000 1000 ns/op 8188.71 MB/s (asm) | |||||
BenchmarkStdNoAsmCastagnoli32KB-4 10000 115426 ns/op 283.89 MB/s | |||||
BenchmarkStdCastagnoli32KB-4 300000 4065 ns/op 8059.13 MB/s (asm) | |||||
BenchmarkCastagnoli32KBNoAsm-4 30000 45171 ns/op 725.41 MB/s (slice8) | |||||
BenchmarkCastagnoli32KB-4 500000 4077 ns/op 8035.89 MB/s (asm) | |||||
``` | |||||
The IEEE assembler optimizations has been submitted and will be part of the Go 1.6 standard library. | |||||
However, the improved use of slice-by-8 has not, but will probably be submitted for Go 1.7. | |||||
# license | |||||
Standard Go license. Changes are Copyright (c) 2015 Klaus Post under same conditions. |
@ -1,207 +0,0 @@ | |||||
// Copyright 2009 The Go Authors. All rights reserved. | |||||
// Use of this source code is governed by a BSD-style | |||||
// license that can be found in the LICENSE file. | |||||
// Package crc32 implements the 32-bit cyclic redundancy check, or CRC-32, | |||||
// checksum. See http://en.wikipedia.org/wiki/Cyclic_redundancy_check for | |||||
// information. | |||||
// | |||||
// Polynomials are represented in LSB-first form also known as reversed representation. | |||||
// | |||||
// See http://en.wikipedia.org/wiki/Mathematics_of_cyclic_redundancy_checks#Reversed_representations_and_reciprocal_polynomials | |||||
// for information. | |||||
package crc32 | |||||
import ( | |||||
"hash" | |||||
"sync" | |||||
) | |||||
// The size of a CRC-32 checksum in bytes. | |||||
const Size = 4 | |||||
// Predefined polynomials. | |||||
const ( | |||||
// IEEE is by far and away the most common CRC-32 polynomial. | |||||
// Used by ethernet (IEEE 802.3), v.42, fddi, gzip, zip, png, ... | |||||
IEEE = 0xedb88320 | |||||
// Castagnoli's polynomial, used in iSCSI. | |||||
// Has better error detection characteristics than IEEE. | |||||
// http://dx.doi.org/10.1109/26.231911 | |||||
Castagnoli = 0x82f63b78 | |||||
// Koopman's polynomial. | |||||
// Also has better error detection characteristics than IEEE. | |||||
// http://dx.doi.org/10.1109/DSN.2002.1028931 | |||||
Koopman = 0xeb31d82e | |||||
) | |||||
// Table is a 256-word table representing the polynomial for efficient processing. | |||||
type Table [256]uint32 | |||||
// This file makes use of functions implemented in architecture-specific files. | |||||
// The interface that they implement is as follows: | |||||
// | |||||
// // archAvailableIEEE reports whether an architecture-specific CRC32-IEEE | |||||
// // algorithm is available. | |||||
// archAvailableIEEE() bool | |||||
// | |||||
// // archInitIEEE initializes the architecture-specific CRC3-IEEE algorithm. | |||||
// // It can only be called if archAvailableIEEE() returns true. | |||||
// archInitIEEE() | |||||
// | |||||
// // archUpdateIEEE updates the given CRC32-IEEE. It can only be called if | |||||
// // archInitIEEE() was previously called. | |||||
// archUpdateIEEE(crc uint32, p []byte) uint32 | |||||
// | |||||
// // archAvailableCastagnoli reports whether an architecture-specific | |||||
// // CRC32-C algorithm is available. | |||||
// archAvailableCastagnoli() bool | |||||
// | |||||
// // archInitCastagnoli initializes the architecture-specific CRC32-C | |||||
// // algorithm. It can only be called if archAvailableCastagnoli() returns | |||||
// // true. | |||||
// archInitCastagnoli() | |||||
// | |||||
// // archUpdateCastagnoli updates the given CRC32-C. It can only be called | |||||
// // if archInitCastagnoli() was previously called. | |||||
// archUpdateCastagnoli(crc uint32, p []byte) uint32 | |||||
// castagnoliTable points to a lazily initialized Table for the Castagnoli | |||||
// polynomial. MakeTable will always return this value when asked to make a | |||||
// Castagnoli table so we can compare against it to find when the caller is | |||||
// using this polynomial. | |||||
var castagnoliTable *Table | |||||
var castagnoliTable8 *slicing8Table | |||||
var castagnoliArchImpl bool | |||||
var updateCastagnoli func(crc uint32, p []byte) uint32 | |||||
var castagnoliOnce sync.Once | |||||
func castagnoliInit() { | |||||
castagnoliTable = simpleMakeTable(Castagnoli) | |||||
castagnoliArchImpl = archAvailableCastagnoli() | |||||
if castagnoliArchImpl { | |||||
archInitCastagnoli() | |||||
updateCastagnoli = archUpdateCastagnoli | |||||
} else { | |||||
// Initialize the slicing-by-8 table. | |||||
castagnoliTable8 = slicingMakeTable(Castagnoli) | |||||
updateCastagnoli = func(crc uint32, p []byte) uint32 { | |||||
return slicingUpdate(crc, castagnoliTable8, p) | |||||
} | |||||
} | |||||
} | |||||
// IEEETable is the table for the IEEE polynomial. | |||||
var IEEETable = simpleMakeTable(IEEE) | |||||
// ieeeTable8 is the slicing8Table for IEEE | |||||
var ieeeTable8 *slicing8Table | |||||
var ieeeArchImpl bool | |||||
var updateIEEE func(crc uint32, p []byte) uint32 | |||||
var ieeeOnce sync.Once | |||||
func ieeeInit() { | |||||
ieeeArchImpl = archAvailableIEEE() | |||||
if ieeeArchImpl { | |||||
archInitIEEE() | |||||
updateIEEE = archUpdateIEEE | |||||
} else { | |||||
// Initialize the slicing-by-8 table. | |||||
ieeeTable8 = slicingMakeTable(IEEE) | |||||
updateIEEE = func(crc uint32, p []byte) uint32 { | |||||
return slicingUpdate(crc, ieeeTable8, p) | |||||
} | |||||
} | |||||
} | |||||
// MakeTable returns a Table constructed from the specified polynomial. | |||||
// The contents of this Table must not be modified. | |||||
func MakeTable(poly uint32) *Table { | |||||
switch poly { | |||||
case IEEE: | |||||
ieeeOnce.Do(ieeeInit) | |||||
return IEEETable | |||||
case Castagnoli: | |||||
castagnoliOnce.Do(castagnoliInit) | |||||
return castagnoliTable | |||||
} | |||||
return simpleMakeTable(poly) | |||||
} | |||||
// digest represents the partial evaluation of a checksum. | |||||
type digest struct { | |||||
crc uint32 | |||||
tab *Table | |||||
} | |||||
// New creates a new hash.Hash32 computing the CRC-32 checksum | |||||
// using the polynomial represented by the Table. | |||||
// Its Sum method will lay the value out in big-endian byte order. | |||||
func New(tab *Table) hash.Hash32 { | |||||
if tab == IEEETable { | |||||
ieeeOnce.Do(ieeeInit) | |||||
} | |||||
return &digest{0, tab} | |||||
} | |||||
// NewIEEE creates a new hash.Hash32 computing the CRC-32 checksum | |||||
// using the IEEE polynomial. | |||||
// Its Sum method will lay the value out in big-endian byte order. | |||||
func NewIEEE() hash.Hash32 { return New(IEEETable) } | |||||
func (d *digest) Size() int { return Size } | |||||
func (d *digest) BlockSize() int { return 1 } | |||||
func (d *digest) Reset() { d.crc = 0 } | |||||
// Update returns the result of adding the bytes in p to the crc. | |||||
func Update(crc uint32, tab *Table, p []byte) uint32 { | |||||
switch tab { | |||||
case castagnoliTable: | |||||
return updateCastagnoli(crc, p) | |||||
case IEEETable: | |||||
// Unfortunately, because IEEETable is exported, IEEE may be used without a | |||||
// call to MakeTable. We have to make sure it gets initialized in that case. | |||||
ieeeOnce.Do(ieeeInit) | |||||
return updateIEEE(crc, p) | |||||
default: | |||||
return simpleUpdate(crc, tab, p) | |||||
} | |||||
} | |||||
func (d *digest) Write(p []byte) (n int, err error) { | |||||
switch d.tab { | |||||
case castagnoliTable: | |||||
d.crc = updateCastagnoli(d.crc, p) | |||||
case IEEETable: | |||||
// We only create digest objects through New() which takes care of | |||||
// initialization in this case. | |||||
d.crc = updateIEEE(d.crc, p) | |||||
default: | |||||
d.crc = simpleUpdate(d.crc, d.tab, p) | |||||
} | |||||
return len(p), nil | |||||
} | |||||
func (d *digest) Sum32() uint32 { return d.crc } | |||||
func (d *digest) Sum(in []byte) []byte { | |||||
s := d.Sum32() | |||||
return append(in, byte(s>>24), byte(s>>16), byte(s>>8), byte(s)) | |||||
} | |||||
// Checksum returns the CRC-32 checksum of data | |||||
// using the polynomial represented by the Table. | |||||
func Checksum(data []byte, tab *Table) uint32 { return Update(0, tab, data) } | |||||
// ChecksumIEEE returns the CRC-32 checksum of data | |||||
// using the IEEE polynomial. | |||||
func ChecksumIEEE(data []byte) uint32 { | |||||
ieeeOnce.Do(ieeeInit) | |||||
return updateIEEE(0, data) | |||||
} |
@ -1,230 +0,0 @@ | |||||
// Copyright 2011 The Go Authors. All rights reserved. | |||||
// Use of this source code is governed by a BSD-style | |||||
// license that can be found in the LICENSE file. | |||||
// +build !appengine,!gccgo | |||||
// AMD64-specific hardware-assisted CRC32 algorithms. See crc32.go for a | |||||
// description of the interface that each architecture-specific file | |||||
// implements. | |||||
package crc32 | |||||
import "unsafe" | |||||
// This file contains the code to call the SSE 4.2 version of the Castagnoli | |||||
// and IEEE CRC. | |||||
// haveSSE41/haveSSE42/haveCLMUL are defined in crc_amd64.s and use | |||||
// CPUID to test for SSE 4.1, 4.2 and CLMUL support. | |||||
func haveSSE41() bool | |||||
func haveSSE42() bool | |||||
func haveCLMUL() bool | |||||
// castagnoliSSE42 is defined in crc32_amd64.s and uses the SSE4.2 CRC32 | |||||
// instruction. | |||||
//go:noescape | |||||
func castagnoliSSE42(crc uint32, p []byte) uint32 | |||||
// castagnoliSSE42Triple is defined in crc32_amd64.s and uses the SSE4.2 CRC32 | |||||
// instruction. | |||||
//go:noescape | |||||
func castagnoliSSE42Triple( | |||||
crcA, crcB, crcC uint32, | |||||
a, b, c []byte, | |||||
rounds uint32, | |||||
) (retA uint32, retB uint32, retC uint32) | |||||
// ieeeCLMUL is defined in crc_amd64.s and uses the PCLMULQDQ | |||||
// instruction as well as SSE 4.1. | |||||
//go:noescape | |||||
func ieeeCLMUL(crc uint32, p []byte) uint32 | |||||
var sse42 = haveSSE42() | |||||
var useFastIEEE = haveCLMUL() && haveSSE41() | |||||
const castagnoliK1 = 168 | |||||
const castagnoliK2 = 1344 | |||||
type sse42Table [4]Table | |||||
var castagnoliSSE42TableK1 *sse42Table | |||||
var castagnoliSSE42TableK2 *sse42Table | |||||
func archAvailableCastagnoli() bool { | |||||
return sse42 | |||||
} | |||||
func archInitCastagnoli() { | |||||
if !sse42 { | |||||
panic("arch-specific Castagnoli not available") | |||||
} | |||||
castagnoliSSE42TableK1 = new(sse42Table) | |||||
castagnoliSSE42TableK2 = new(sse42Table) | |||||
// See description in updateCastagnoli. | |||||
// t[0][i] = CRC(i000, O) | |||||
// t[1][i] = CRC(0i00, O) | |||||
// t[2][i] = CRC(00i0, O) | |||||
// t[3][i] = CRC(000i, O) | |||||
// where O is a sequence of K zeros. | |||||
var tmp [castagnoliK2]byte | |||||
for b := 0; b < 4; b++ { | |||||
for i := 0; i < 256; i++ { | |||||
val := uint32(i) << uint32(b*8) | |||||
castagnoliSSE42TableK1[b][i] = castagnoliSSE42(val, tmp[:castagnoliK1]) | |||||
castagnoliSSE42TableK2[b][i] = castagnoliSSE42(val, tmp[:]) | |||||
} | |||||
} | |||||
} | |||||
// castagnoliShift computes the CRC32-C of K1 or K2 zeroes (depending on the | |||||
// table given) with the given initial crc value. This corresponds to | |||||
// CRC(crc, O) in the description in updateCastagnoli. | |||||
func castagnoliShift(table *sse42Table, crc uint32) uint32 { | |||||
return table[3][crc>>24] ^ | |||||
table[2][(crc>>16)&0xFF] ^ | |||||
table[1][(crc>>8)&0xFF] ^ | |||||
table[0][crc&0xFF] | |||||
} | |||||
func archUpdateCastagnoli(crc uint32, p []byte) uint32 { | |||||
if !sse42 { | |||||
panic("not available") | |||||
} | |||||
// This method is inspired from the algorithm in Intel's white paper: | |||||
// "Fast CRC Computation for iSCSI Polynomial Using CRC32 Instruction" | |||||
// The same strategy of splitting the buffer in three is used but the | |||||
// combining calculation is different; the complete derivation is explained | |||||
// below. | |||||
// | |||||
// -- The basic idea -- | |||||
// | |||||
// The CRC32 instruction (available in SSE4.2) can process 8 bytes at a | |||||
// time. In recent Intel architectures the instruction takes 3 cycles; | |||||
// however the processor can pipeline up to three instructions if they | |||||
// don't depend on each other. | |||||
// | |||||
// Roughly this means that we can process three buffers in about the same | |||||
// time we can process one buffer. | |||||
// | |||||
// The idea is then to split the buffer in three, CRC the three pieces | |||||
// separately and then combine the results. | |||||
// | |||||
// Combining the results requires precomputed tables, so we must choose a | |||||
// fixed buffer length to optimize. The longer the length, the faster; but | |||||
// only buffers longer than this length will use the optimization. We choose | |||||
// two cutoffs and compute tables for both: | |||||
// - one around 512: 168*3=504 | |||||
// - one around 4KB: 1344*3=4032 | |||||
// | |||||
// -- The nitty gritty -- | |||||
// | |||||
// Let CRC(I, X) be the non-inverted CRC32-C of the sequence X (with | |||||
// initial non-inverted CRC I). This function has the following properties: | |||||
// (a) CRC(I, AB) = CRC(CRC(I, A), B) | |||||
// (b) CRC(I, A xor B) = CRC(I, A) xor CRC(0, B) | |||||
// | |||||
// Say we want to compute CRC(I, ABC) where A, B, C are three sequences of | |||||
// K bytes each, where K is a fixed constant. Let O be the sequence of K zero | |||||
// bytes. | |||||
// | |||||
// CRC(I, ABC) = CRC(I, ABO xor C) | |||||
// = CRC(I, ABO) xor CRC(0, C) | |||||
// = CRC(CRC(I, AB), O) xor CRC(0, C) | |||||
// = CRC(CRC(I, AO xor B), O) xor CRC(0, C) | |||||
// = CRC(CRC(I, AO) xor CRC(0, B), O) xor CRC(0, C) | |||||
// = CRC(CRC(CRC(I, A), O) xor CRC(0, B), O) xor CRC(0, C) | |||||
// | |||||
// The castagnoliSSE42Triple function can compute CRC(I, A), CRC(0, B), | |||||
// and CRC(0, C) efficiently. We just need to find a way to quickly compute | |||||
// CRC(uvwx, O) given a 4-byte initial value uvwx. We can precompute these | |||||
// values; since we can't have a 32-bit table, we break it up into four | |||||
// 8-bit tables: | |||||
// | |||||
// CRC(uvwx, O) = CRC(u000, O) xor | |||||
// CRC(0v00, O) xor | |||||
// CRC(00w0, O) xor | |||||
// CRC(000x, O) | |||||
// | |||||
// We can compute tables corresponding to the four terms for all 8-bit | |||||
// values. | |||||
crc = ^crc | |||||
// If a buffer is long enough to use the optimization, process the first few | |||||
// bytes to align the buffer to an 8 byte boundary (if necessary). | |||||
if len(p) >= castagnoliK1*3 { | |||||
delta := int(uintptr(unsafe.Pointer(&p[0])) & 7) | |||||
if delta != 0 { | |||||
delta = 8 - delta | |||||
crc = castagnoliSSE42(crc, p[:delta]) | |||||
p = p[delta:] | |||||
} | |||||
} | |||||
// Process 3*K2 at a time. | |||||
for len(p) >= castagnoliK2*3 { | |||||
// Compute CRC(I, A), CRC(0, B), and CRC(0, C). | |||||
crcA, crcB, crcC := castagnoliSSE42Triple( | |||||
crc, 0, 0, | |||||
p, p[castagnoliK2:], p[castagnoliK2*2:], | |||||
castagnoliK2/24) | |||||
// CRC(I, AB) = CRC(CRC(I, A), O) xor CRC(0, B) | |||||
crcAB := castagnoliShift(castagnoliSSE42TableK2, crcA) ^ crcB | |||||
// CRC(I, ABC) = CRC(CRC(I, AB), O) xor CRC(0, C) | |||||
crc = castagnoliShift(castagnoliSSE42TableK2, crcAB) ^ crcC | |||||
p = p[castagnoliK2*3:] | |||||
} | |||||
// Process 3*K1 at a time. | |||||
for len(p) >= castagnoliK1*3 { | |||||
// Compute CRC(I, A), CRC(0, B), and CRC(0, C). | |||||
crcA, crcB, crcC := castagnoliSSE42Triple( | |||||
crc, 0, 0, | |||||
p, p[castagnoliK1:], p[castagnoliK1*2:], | |||||
castagnoliK1/24) | |||||
// CRC(I, AB) = CRC(CRC(I, A), O) xor CRC(0, B) | |||||
crcAB := castagnoliShift(castagnoliSSE42TableK1, crcA) ^ crcB | |||||
// CRC(I, ABC) = CRC(CRC(I, AB), O) xor CRC(0, C) | |||||
crc = castagnoliShift(castagnoliSSE42TableK1, crcAB) ^ crcC | |||||
p = p[castagnoliK1*3:] | |||||
} | |||||
// Use the simple implementation for what's left. | |||||
crc = castagnoliSSE42(crc, p) | |||||
return ^crc | |||||
} | |||||
func archAvailableIEEE() bool { | |||||
return useFastIEEE | |||||
} | |||||
var archIeeeTable8 *slicing8Table | |||||
func archInitIEEE() { | |||||
if !useFastIEEE { | |||||
panic("not available") | |||||
} | |||||
// We still use slicing-by-8 for small buffers. | |||||
archIeeeTable8 = slicingMakeTable(IEEE) | |||||
} | |||||
func archUpdateIEEE(crc uint32, p []byte) uint32 { | |||||
if !useFastIEEE { | |||||
panic("not available") | |||||
} | |||||
if len(p) >= 64 { | |||||
left := len(p) & 15 | |||||
do := len(p) - left | |||||
crc = ^ieeeCLMUL(^crc, p[:do]) | |||||
p = p[do:] | |||||
} | |||||
if len(p) == 0 { | |||||
return crc | |||||
} | |||||
return slicingUpdate(crc, archIeeeTable8, p) | |||||
} |
@ -1,319 +0,0 @@ | |||||
// Copyright 2011 The Go Authors. All rights reserved. | |||||
// Use of this source code is governed by a BSD-style | |||||
// license that can be found in the LICENSE file. | |||||
// +build gc | |||||
#define NOSPLIT 4 | |||||
#define RODATA 8 | |||||
// castagnoliSSE42 updates the (non-inverted) crc with the given buffer. | |||||
// | |||||
// func castagnoliSSE42(crc uint32, p []byte) uint32 | |||||
TEXT ·castagnoliSSE42(SB), NOSPLIT, $0 | |||||
MOVL crc+0(FP), AX // CRC value | |||||
MOVQ p+8(FP), SI // data pointer | |||||
MOVQ p_len+16(FP), CX // len(p) | |||||
// If there are fewer than 8 bytes to process, skip alignment. | |||||
CMPQ CX, $8 | |||||
JL less_than_8 | |||||
MOVQ SI, BX | |||||
ANDQ $7, BX | |||||
JZ aligned | |||||
// Process the first few bytes to 8-byte align the input. | |||||
// BX = 8 - BX. We need to process this many bytes to align. | |||||
SUBQ $1, BX | |||||
XORQ $7, BX | |||||
BTQ $0, BX | |||||
JNC align_2 | |||||
CRC32B (SI), AX | |||||
DECQ CX | |||||
INCQ SI | |||||
align_2: | |||||
BTQ $1, BX | |||||
JNC align_4 | |||||
// CRC32W (SI), AX | |||||
BYTE $0x66; BYTE $0xf2; BYTE $0x0f; BYTE $0x38; BYTE $0xf1; BYTE $0x06 | |||||
SUBQ $2, CX | |||||
ADDQ $2, SI | |||||
align_4: | |||||
BTQ $2, BX | |||||
JNC aligned | |||||
// CRC32L (SI), AX | |||||
BYTE $0xf2; BYTE $0x0f; BYTE $0x38; BYTE $0xf1; BYTE $0x06 | |||||
SUBQ $4, CX | |||||
ADDQ $4, SI | |||||
aligned: | |||||
// The input is now 8-byte aligned and we can process 8-byte chunks. | |||||
CMPQ CX, $8 | |||||
JL less_than_8 | |||||
CRC32Q (SI), AX | |||||
ADDQ $8, SI | |||||
SUBQ $8, CX | |||||
JMP aligned | |||||
less_than_8: | |||||
// We may have some bytes left over; process 4 bytes, then 2, then 1. | |||||
BTQ $2, CX | |||||
JNC less_than_4 | |||||
// CRC32L (SI), AX | |||||
BYTE $0xf2; BYTE $0x0f; BYTE $0x38; BYTE $0xf1; BYTE $0x06 | |||||
ADDQ $4, SI | |||||
less_than_4: | |||||
BTQ $1, CX | |||||
JNC less_than_2 | |||||
// CRC32W (SI), AX | |||||
BYTE $0x66; BYTE $0xf2; BYTE $0x0f; BYTE $0x38; BYTE $0xf1; BYTE $0x06 | |||||
ADDQ $2, SI | |||||
less_than_2: | |||||
BTQ $0, CX | |||||
JNC done | |||||
CRC32B (SI), AX | |||||
done: | |||||
MOVL AX, ret+32(FP) | |||||
RET | |||||
// castagnoliSSE42Triple updates three (non-inverted) crcs with (24*rounds) | |||||
// bytes from each buffer. | |||||
// | |||||
// func castagnoliSSE42Triple( | |||||
// crc1, crc2, crc3 uint32, | |||||
// a, b, c []byte, | |||||
// rounds uint32, | |||||
// ) (retA uint32, retB uint32, retC uint32) | |||||
TEXT ·castagnoliSSE42Triple(SB), NOSPLIT, $0 | |||||
MOVL crcA+0(FP), AX | |||||
MOVL crcB+4(FP), CX | |||||
MOVL crcC+8(FP), DX | |||||
MOVQ a+16(FP), R8 // data pointer | |||||
MOVQ b+40(FP), R9 // data pointer | |||||
MOVQ c+64(FP), R10 // data pointer | |||||
MOVL rounds+88(FP), R11 | |||||
loop: | |||||
CRC32Q (R8), AX | |||||
CRC32Q (R9), CX | |||||
CRC32Q (R10), DX | |||||
CRC32Q 8(R8), AX | |||||
CRC32Q 8(R9), CX | |||||
CRC32Q 8(R10), DX | |||||
CRC32Q 16(R8), AX | |||||
CRC32Q 16(R9), CX | |||||
CRC32Q 16(R10), DX | |||||
ADDQ $24, R8 | |||||
ADDQ $24, R9 | |||||
ADDQ $24, R10 | |||||
DECQ R11 | |||||
JNZ loop | |||||
MOVL AX, retA+96(FP) | |||||
MOVL CX, retB+100(FP) | |||||
MOVL DX, retC+104(FP) | |||||
RET | |||||
// func haveSSE42() bool | |||||
TEXT ·haveSSE42(SB), NOSPLIT, $0 | |||||
XORQ AX, AX | |||||
INCL AX | |||||
CPUID | |||||
SHRQ $20, CX | |||||
ANDQ $1, CX | |||||
MOVB CX, ret+0(FP) | |||||
RET | |||||
// func haveCLMUL() bool | |||||
TEXT ·haveCLMUL(SB), NOSPLIT, $0 | |||||
XORQ AX, AX | |||||
INCL AX | |||||
CPUID | |||||
SHRQ $1, CX | |||||
ANDQ $1, CX | |||||
MOVB CX, ret+0(FP) | |||||
RET | |||||
// func haveSSE41() bool | |||||
TEXT ·haveSSE41(SB), NOSPLIT, $0 | |||||
XORQ AX, AX | |||||
INCL AX | |||||
CPUID | |||||
SHRQ $19, CX | |||||
ANDQ $1, CX | |||||
MOVB CX, ret+0(FP) | |||||
RET | |||||
// CRC32 polynomial data | |||||
// | |||||
// These constants are lifted from the | |||||
// Linux kernel, since they avoid the costly | |||||
// PSHUFB 16 byte reversal proposed in the | |||||
// original Intel paper. | |||||
DATA r2r1kp<>+0(SB)/8, $0x154442bd4 | |||||
DATA r2r1kp<>+8(SB)/8, $0x1c6e41596 | |||||
DATA r4r3kp<>+0(SB)/8, $0x1751997d0 | |||||
DATA r4r3kp<>+8(SB)/8, $0x0ccaa009e | |||||
DATA rupolykp<>+0(SB)/8, $0x1db710641 | |||||
DATA rupolykp<>+8(SB)/8, $0x1f7011641 | |||||
DATA r5kp<>+0(SB)/8, $0x163cd6124 | |||||
GLOBL r2r1kp<>(SB), RODATA, $16 | |||||
GLOBL r4r3kp<>(SB), RODATA, $16 | |||||
GLOBL rupolykp<>(SB), RODATA, $16 | |||||
GLOBL r5kp<>(SB), RODATA, $8 | |||||
// Based on http://www.intel.com/content/dam/www/public/us/en/documents/white-papers/fast-crc-computation-generic-polynomials-pclmulqdq-paper.pdf | |||||
// len(p) must be at least 64, and must be a multiple of 16. | |||||
// func ieeeCLMUL(crc uint32, p []byte) uint32 | |||||
TEXT ·ieeeCLMUL(SB), NOSPLIT, $0 | |||||
MOVL crc+0(FP), X0 // Initial CRC value | |||||
MOVQ p+8(FP), SI // data pointer | |||||
MOVQ p_len+16(FP), CX // len(p) | |||||
MOVOU (SI), X1 | |||||
MOVOU 16(SI), X2 | |||||
MOVOU 32(SI), X3 | |||||
MOVOU 48(SI), X4 | |||||
PXOR X0, X1 | |||||
ADDQ $64, SI // buf+=64 | |||||
SUBQ $64, CX // len-=64 | |||||
CMPQ CX, $64 // Less than 64 bytes left | |||||
JB remain64 | |||||
MOVOA r2r1kp<>+0(SB), X0 | |||||
loopback64: | |||||
MOVOA X1, X5 | |||||
MOVOA X2, X6 | |||||
MOVOA X3, X7 | |||||
MOVOA X4, X8 | |||||
PCLMULQDQ $0, X0, X1 | |||||
PCLMULQDQ $0, X0, X2 | |||||
PCLMULQDQ $0, X0, X3 | |||||
PCLMULQDQ $0, X0, X4 | |||||
// Load next early | |||||
MOVOU (SI), X11 | |||||
MOVOU 16(SI), X12 | |||||
MOVOU 32(SI), X13 | |||||
MOVOU 48(SI), X14 | |||||
PCLMULQDQ $0x11, X0, X5 | |||||
PCLMULQDQ $0x11, X0, X6 | |||||
PCLMULQDQ $0x11, X0, X7 | |||||
PCLMULQDQ $0x11, X0, X8 | |||||
PXOR X5, X1 | |||||
PXOR X6, X2 | |||||
PXOR X7, X3 | |||||
PXOR X8, X4 | |||||
PXOR X11, X1 | |||||
PXOR X12, X2 | |||||
PXOR X13, X3 | |||||
PXOR X14, X4 | |||||
ADDQ $0x40, DI | |||||
ADDQ $64, SI // buf+=64 | |||||
SUBQ $64, CX // len-=64 | |||||
CMPQ CX, $64 // Less than 64 bytes left? | |||||
JGE loopback64 | |||||
// Fold result into a single register (X1) | |||||
remain64: | |||||
MOVOA r4r3kp<>+0(SB), X0 | |||||
MOVOA X1, X5 | |||||
PCLMULQDQ $0, X0, X1 | |||||
PCLMULQDQ $0x11, X0, X5 | |||||
PXOR X5, X1 | |||||
PXOR X2, X1 | |||||
MOVOA X1, X5 | |||||
PCLMULQDQ $0, X0, X1 | |||||
PCLMULQDQ $0x11, X0, X5 | |||||
PXOR X5, X1 | |||||
PXOR X3, X1 | |||||
MOVOA X1, X5 | |||||
PCLMULQDQ $0, X0, X1 | |||||
PCLMULQDQ $0x11, X0, X5 | |||||
PXOR X5, X1 | |||||
PXOR X4, X1 | |||||
// If there is less than 16 bytes left we are done | |||||
CMPQ CX, $16 | |||||
JB finish | |||||
// Encode 16 bytes | |||||
remain16: | |||||
MOVOU (SI), X10 | |||||
MOVOA X1, X5 | |||||
PCLMULQDQ $0, X0, X1 | |||||
PCLMULQDQ $0x11, X0, X5 | |||||
PXOR X5, X1 | |||||
PXOR X10, X1 | |||||
SUBQ $16, CX | |||||
ADDQ $16, SI | |||||
CMPQ CX, $16 | |||||
JGE remain16 | |||||
finish: | |||||
// Fold final result into 32 bits and return it | |||||
PCMPEQB X3, X3 | |||||
PCLMULQDQ $1, X1, X0 | |||||
PSRLDQ $8, X1 | |||||
PXOR X0, X1 | |||||
MOVOA X1, X2 | |||||
MOVQ r5kp<>+0(SB), X0 | |||||
// Creates 32 bit mask. Note that we don't care about upper half. | |||||
PSRLQ $32, X3 | |||||
PSRLDQ $4, X2 | |||||
PAND X3, X1 | |||||
PCLMULQDQ $0, X0, X1 | |||||
PXOR X2, X1 | |||||
MOVOA rupolykp<>+0(SB), X0 | |||||
MOVOA X1, X2 | |||||
PAND X3, X1 | |||||
PCLMULQDQ $0x10, X0, X1 | |||||
PAND X3, X1 | |||||
PCLMULQDQ $0, X0, X1 | |||||
PXOR X2, X1 | |||||
// PEXTRD $1, X1, AX (SSE 4.1) | |||||
BYTE $0x66; BYTE $0x0f; BYTE $0x3a | |||||
BYTE $0x16; BYTE $0xc8; BYTE $0x01 | |||||
MOVL AX, ret+32(FP) | |||||
RET |
@ -1,43 +0,0 @@ | |||||
// Copyright 2011 The Go Authors. All rights reserved. | |||||
// Use of this source code is governed by a BSD-style | |||||
// license that can be found in the LICENSE file. | |||||
// +build !appengine,!gccgo | |||||
package crc32 | |||||
// This file contains the code to call the SSE 4.2 version of the Castagnoli | |||||
// CRC. | |||||
// haveSSE42 is defined in crc32_amd64p32.s and uses CPUID to test for SSE 4.2 | |||||
// support. | |||||
func haveSSE42() bool | |||||
// castagnoliSSE42 is defined in crc32_amd64p32.s and uses the SSE4.2 CRC32 | |||||
// instruction. | |||||
//go:noescape | |||||
func castagnoliSSE42(crc uint32, p []byte) uint32 | |||||
var sse42 = haveSSE42() | |||||
func archAvailableCastagnoli() bool { | |||||
return sse42 | |||||
} | |||||
func archInitCastagnoli() { | |||||
if !sse42 { | |||||
panic("not available") | |||||
} | |||||
// No initialization necessary. | |||||
} | |||||
func archUpdateCastagnoli(crc uint32, p []byte) uint32 { | |||||
if !sse42 { | |||||
panic("not available") | |||||
} | |||||
return castagnoliSSE42(crc, p) | |||||
} | |||||
func archAvailableIEEE() bool { return false } | |||||
func archInitIEEE() { panic("not available") } | |||||
func archUpdateIEEE(crc uint32, p []byte) uint32 { panic("not available") } |
@ -1,67 +0,0 @@ | |||||
// Copyright 2011 The Go Authors. All rights reserved. | |||||
// Use of this source code is governed by a BSD-style | |||||
// license that can be found in the LICENSE file. | |||||
// +build gc | |||||
#define NOSPLIT 4 | |||||
#define RODATA 8 | |||||
// func castagnoliSSE42(crc uint32, p []byte) uint32 | |||||
TEXT ·castagnoliSSE42(SB), NOSPLIT, $0 | |||||
MOVL crc+0(FP), AX // CRC value | |||||
MOVL p+4(FP), SI // data pointer | |||||
MOVL p_len+8(FP), CX // len(p) | |||||
NOTL AX | |||||
// If there's less than 8 bytes to process, we do it byte-by-byte. | |||||
CMPQ CX, $8 | |||||
JL cleanup | |||||
// Process individual bytes until the input is 8-byte aligned. | |||||
startup: | |||||
MOVQ SI, BX | |||||
ANDQ $7, BX | |||||
JZ aligned | |||||
CRC32B (SI), AX | |||||
DECQ CX | |||||
INCQ SI | |||||
JMP startup | |||||
aligned: | |||||
// The input is now 8-byte aligned and we can process 8-byte chunks. | |||||
CMPQ CX, $8 | |||||
JL cleanup | |||||
CRC32Q (SI), AX | |||||
ADDQ $8, SI | |||||
SUBQ $8, CX | |||||
JMP aligned | |||||
cleanup: | |||||
// We may have some bytes left over that we process one at a time. | |||||
CMPQ CX, $0 | |||||
JE done | |||||
CRC32B (SI), AX | |||||
INCQ SI | |||||
DECQ CX | |||||
JMP cleanup | |||||
done: | |||||
NOTL AX | |||||
MOVL AX, ret+16(FP) | |||||
RET | |||||
// func haveSSE42() bool | |||||
TEXT ·haveSSE42(SB), NOSPLIT, $0 | |||||
XORQ AX, AX | |||||
INCL AX | |||||
CPUID | |||||
SHRQ $20, CX | |||||
ANDQ $1, CX | |||||
MOVB CX, ret+0(FP) | |||||
RET | |||||
@ -1,89 +0,0 @@ | |||||
// Copyright 2011 The Go Authors. All rights reserved. | |||||
// Use of this source code is governed by a BSD-style | |||||
// license that can be found in the LICENSE file. | |||||
// This file contains CRC32 algorithms that are not specific to any architecture | |||||
// and don't use hardware acceleration. | |||||
// | |||||
// The simple (and slow) CRC32 implementation only uses a 256*4 bytes table. | |||||
// | |||||
// The slicing-by-8 algorithm is a faster implementation that uses a bigger | |||||
// table (8*256*4 bytes). | |||||
package crc32 | |||||
// simpleMakeTable allocates and constructs a Table for the specified | |||||
// polynomial. The table is suitable for use with the simple algorithm | |||||
// (simpleUpdate). | |||||
func simpleMakeTable(poly uint32) *Table { | |||||
t := new(Table) | |||||
simplePopulateTable(poly, t) | |||||
return t | |||||
} | |||||
// simplePopulateTable constructs a Table for the specified polynomial, suitable | |||||
// for use with simpleUpdate. | |||||
func simplePopulateTable(poly uint32, t *Table) { | |||||
for i := 0; i < 256; i++ { | |||||
crc := uint32(i) | |||||
for j := 0; j < 8; j++ { | |||||
if crc&1 == 1 { | |||||
crc = (crc >> 1) ^ poly | |||||
} else { | |||||
crc >>= 1 | |||||
} | |||||
} | |||||
t[i] = crc | |||||
} | |||||
} | |||||
// simpleUpdate uses the simple algorithm to update the CRC, given a table that | |||||
// was previously computed using simpleMakeTable. | |||||
func simpleUpdate(crc uint32, tab *Table, p []byte) uint32 { | |||||
crc = ^crc | |||||
for _, v := range p { | |||||
crc = tab[byte(crc)^v] ^ (crc >> 8) | |||||
} | |||||
return ^crc | |||||
} | |||||
// Use slicing-by-8 when payload >= this value. | |||||
const slicing8Cutoff = 16 | |||||
// slicing8Table is array of 8 Tables, used by the slicing-by-8 algorithm. | |||||
type slicing8Table [8]Table | |||||
// slicingMakeTable constructs a slicing8Table for the specified polynomial. The | |||||
// table is suitable for use with the slicing-by-8 algorithm (slicingUpdate). | |||||
func slicingMakeTable(poly uint32) *slicing8Table { | |||||
t := new(slicing8Table) | |||||
simplePopulateTable(poly, &t[0]) | |||||
for i := 0; i < 256; i++ { | |||||
crc := t[0][i] | |||||
for j := 1; j < 8; j++ { | |||||
crc = t[0][crc&0xFF] ^ (crc >> 8) | |||||
t[j][i] = crc | |||||
} | |||||
} | |||||
return t | |||||
} | |||||
// slicingUpdate uses the slicing-by-8 algorithm to update the CRC, given a | |||||
// table that was previously computed using slicingMakeTable. | |||||
func slicingUpdate(crc uint32, tab *slicing8Table, p []byte) uint32 { | |||||
if len(p) >= slicing8Cutoff { | |||||
crc = ^crc | |||||
for len(p) > 8 { | |||||
crc ^= uint32(p[0]) | uint32(p[1])<<8 | uint32(p[2])<<16 | uint32(p[3])<<24 | |||||
crc = tab[0][p[7]] ^ tab[1][p[6]] ^ tab[2][p[5]] ^ tab[3][p[4]] ^ | |||||
tab[4][crc>>24] ^ tab[5][(crc>>16)&0xFF] ^ | |||||
tab[6][(crc>>8)&0xFF] ^ tab[7][crc&0xFF] | |||||
p = p[8:] | |||||
} | |||||
crc = ^crc | |||||
} | |||||
if len(p) == 0 { | |||||
return crc | |||||
} | |||||
return simpleUpdate(crc, &tab[0], p) | |||||
} |
@ -1,15 +0,0 @@ | |||||
// Copyright 2011 The Go Authors. All rights reserved. | |||||
// Use of this source code is governed by a BSD-style | |||||
// license that can be found in the LICENSE file. | |||||
// +build !amd64,!amd64p32,!s390x | |||||
package crc32 | |||||
func archAvailableIEEE() bool { return false } | |||||
func archInitIEEE() { panic("not available") } | |||||
func archUpdateIEEE(crc uint32, p []byte) uint32 { panic("not available") } | |||||
func archAvailableCastagnoli() bool { return false } | |||||
func archInitCastagnoli() { panic("not available") } | |||||
func archUpdateCastagnoli(crc uint32, p []byte) uint32 { panic("not available") } |
@ -1,91 +0,0 @@ | |||||
// Copyright 2016 The Go Authors. All rights reserved. | |||||
// Use of this source code is governed by a BSD-style | |||||
// license that can be found in the LICENSE file. | |||||
// +build s390x | |||||
package crc32 | |||||
const ( | |||||
vxMinLen = 64 | |||||
vxAlignMask = 15 // align to 16 bytes | |||||
) | |||||
// hasVectorFacility reports whether the machine has the z/Architecture | |||||
// vector facility installed and enabled. | |||||
func hasVectorFacility() bool | |||||
var hasVX = hasVectorFacility() | |||||
// vectorizedCastagnoli implements CRC32 using vector instructions. | |||||
// It is defined in crc32_s390x.s. | |||||
//go:noescape | |||||
func vectorizedCastagnoli(crc uint32, p []byte) uint32 | |||||
// vectorizedIEEE implements CRC32 using vector instructions. | |||||
// It is defined in crc32_s390x.s. | |||||
//go:noescape | |||||
func vectorizedIEEE(crc uint32, p []byte) uint32 | |||||
func archAvailableCastagnoli() bool { | |||||
return hasVX | |||||
} | |||||
var archCastagnoliTable8 *slicing8Table | |||||
func archInitCastagnoli() { | |||||
if !hasVX { | |||||
panic("not available") | |||||
} | |||||
// We still use slicing-by-8 for small buffers. | |||||
archCastagnoliTable8 = slicingMakeTable(Castagnoli) | |||||
} | |||||
// archUpdateCastagnoli calculates the checksum of p using | |||||
// vectorizedCastagnoli. | |||||
func archUpdateCastagnoli(crc uint32, p []byte) uint32 { | |||||
if !hasVX { | |||||
panic("not available") | |||||
} | |||||
// Use vectorized function if data length is above threshold. | |||||
if len(p) >= vxMinLen { | |||||
aligned := len(p) & ^vxAlignMask | |||||
crc = vectorizedCastagnoli(crc, p[:aligned]) | |||||
p = p[aligned:] | |||||
} | |||||
if len(p) == 0 { | |||||
return crc | |||||
} | |||||
return slicingUpdate(crc, archCastagnoliTable8, p) | |||||
} | |||||
func archAvailableIEEE() bool { | |||||
return hasVX | |||||
} | |||||
var archIeeeTable8 *slicing8Table | |||||
func archInitIEEE() { | |||||
if !hasVX { | |||||
panic("not available") | |||||
} | |||||
// We still use slicing-by-8 for small buffers. | |||||
archIeeeTable8 = slicingMakeTable(IEEE) | |||||
} | |||||
// archUpdateIEEE calculates the checksum of p using vectorizedIEEE. | |||||
func archUpdateIEEE(crc uint32, p []byte) uint32 { | |||||
if !hasVX { | |||||
panic("not available") | |||||
} | |||||
// Use vectorized function if data length is above threshold. | |||||
if len(p) >= vxMinLen { | |||||
aligned := len(p) & ^vxAlignMask | |||||
crc = vectorizedIEEE(crc, p[:aligned]) | |||||
p = p[aligned:] | |||||
} | |||||
if len(p) == 0 { | |||||
return crc | |||||
} | |||||
return slicingUpdate(crc, archIeeeTable8, p) | |||||
} |
@ -1,249 +0,0 @@ | |||||
// Copyright 2016 The Go Authors. All rights reserved. | |||||
// Use of this source code is governed by a BSD-style | |||||
// license that can be found in the LICENSE file. | |||||
// +build s390x | |||||
#include "textflag.h" | |||||
// Vector register range containing CRC-32 constants | |||||
#define CONST_PERM_LE2BE V9 | |||||
#define CONST_R2R1 V10 | |||||
#define CONST_R4R3 V11 | |||||
#define CONST_R5 V12 | |||||
#define CONST_RU_POLY V13 | |||||
#define CONST_CRC_POLY V14 | |||||
// The CRC-32 constant block contains reduction constants to fold and | |||||
// process particular chunks of the input data stream in parallel. | |||||
// | |||||
// Note that the constant definitions below are extended in order to compute | |||||
// intermediate results with a single VECTOR GALOIS FIELD MULTIPLY instruction. | |||||
// The rightmost doubleword can be 0 to prevent contribution to the result or | |||||
// can be multiplied by 1 to perform an XOR without the need for a separate | |||||
// VECTOR EXCLUSIVE OR instruction. | |||||
// | |||||
// The polynomials used are bit-reflected: | |||||
// | |||||
// IEEE: P'(x) = 0x0edb88320 | |||||
// Castagnoli: P'(x) = 0x082f63b78 | |||||
// IEEE polynomial constants | |||||
DATA ·crcleconskp+0(SB)/8, $0x0F0E0D0C0B0A0908 // LE-to-BE mask | |||||
DATA ·crcleconskp+8(SB)/8, $0x0706050403020100 | |||||
DATA ·crcleconskp+16(SB)/8, $0x00000001c6e41596 // R2 | |||||
DATA ·crcleconskp+24(SB)/8, $0x0000000154442bd4 // R1 | |||||
DATA ·crcleconskp+32(SB)/8, $0x00000000ccaa009e // R4 | |||||
DATA ·crcleconskp+40(SB)/8, $0x00000001751997d0 // R3 | |||||
DATA ·crcleconskp+48(SB)/8, $0x0000000000000000 | |||||
DATA ·crcleconskp+56(SB)/8, $0x0000000163cd6124 // R5 | |||||
DATA ·crcleconskp+64(SB)/8, $0x0000000000000000 | |||||
DATA ·crcleconskp+72(SB)/8, $0x00000001F7011641 // u' | |||||
DATA ·crcleconskp+80(SB)/8, $0x0000000000000000 | |||||
DATA ·crcleconskp+88(SB)/8, $0x00000001DB710641 // P'(x) << 1 | |||||
GLOBL ·crcleconskp(SB), RODATA, $144 | |||||
// Castagonli Polynomial constants | |||||
DATA ·crccleconskp+0(SB)/8, $0x0F0E0D0C0B0A0908 // LE-to-BE mask | |||||
DATA ·crccleconskp+8(SB)/8, $0x0706050403020100 | |||||
DATA ·crccleconskp+16(SB)/8, $0x000000009e4addf8 // R2 | |||||
DATA ·crccleconskp+24(SB)/8, $0x00000000740eef02 // R1 | |||||
DATA ·crccleconskp+32(SB)/8, $0x000000014cd00bd6 // R4 | |||||
DATA ·crccleconskp+40(SB)/8, $0x00000000f20c0dfe // R3 | |||||
DATA ·crccleconskp+48(SB)/8, $0x0000000000000000 | |||||
DATA ·crccleconskp+56(SB)/8, $0x00000000dd45aab8 // R5 | |||||
DATA ·crccleconskp+64(SB)/8, $0x0000000000000000 | |||||
DATA ·crccleconskp+72(SB)/8, $0x00000000dea713f1 // u' | |||||
DATA ·crccleconskp+80(SB)/8, $0x0000000000000000 | |||||
DATA ·crccleconskp+88(SB)/8, $0x0000000105ec76f0 // P'(x) << 1 | |||||
GLOBL ·crccleconskp(SB), RODATA, $144 | |||||
// func hasVectorFacility() bool | |||||
TEXT ·hasVectorFacility(SB), NOSPLIT, $24-1 | |||||
MOVD $x-24(SP), R1 | |||||
XC $24, 0(R1), 0(R1) // clear the storage | |||||
MOVD $2, R0 // R0 is the number of double words stored -1 | |||||
WORD $0xB2B01000 // STFLE 0(R1) | |||||
XOR R0, R0 // reset the value of R0 | |||||
MOVBZ z-8(SP), R1 | |||||
AND $0x40, R1 | |||||
BEQ novector | |||||
vectorinstalled: | |||||
// check if the vector instruction has been enabled | |||||
VLEIB $0, $0xF, V16 | |||||
VLGVB $0, V16, R1 | |||||
CMPBNE R1, $0xF, novector | |||||
MOVB $1, ret+0(FP) // have vx | |||||
RET | |||||
novector: | |||||
MOVB $0, ret+0(FP) // no vx | |||||
RET | |||||
// The CRC-32 function(s) use these calling conventions: | |||||
// | |||||
// Parameters: | |||||
// | |||||
// R2: Initial CRC value, typically ~0; and final CRC (return) value. | |||||
// R3: Input buffer pointer, performance might be improved if the | |||||
// buffer is on a doubleword boundary. | |||||
// R4: Length of the buffer, must be 64 bytes or greater. | |||||
// | |||||
// Register usage: | |||||
// | |||||
// R5: CRC-32 constant pool base pointer. | |||||
// V0: Initial CRC value and intermediate constants and results. | |||||
// V1..V4: Data for CRC computation. | |||||
// V5..V8: Next data chunks that are fetched from the input buffer. | |||||
// | |||||
// V9..V14: CRC-32 constants. | |||||
// func vectorizedIEEE(crc uint32, p []byte) uint32 | |||||
TEXT ·vectorizedIEEE(SB), NOSPLIT, $0 | |||||
MOVWZ crc+0(FP), R2 // R2 stores the CRC value | |||||
MOVD p+8(FP), R3 // data pointer | |||||
MOVD p_len+16(FP), R4 // len(p) | |||||
MOVD $·crcleconskp(SB), R5 | |||||
BR vectorizedBody<>(SB) | |||||
// func vectorizedCastagnoli(crc uint32, p []byte) uint32 | |||||
TEXT ·vectorizedCastagnoli(SB), NOSPLIT, $0 | |||||
MOVWZ crc+0(FP), R2 // R2 stores the CRC value | |||||
MOVD p+8(FP), R3 // data pointer | |||||
MOVD p_len+16(FP), R4 // len(p) | |||||
// R5: crc-32 constant pool base pointer, constant is used to reduce crc | |||||
MOVD $·crccleconskp(SB), R5 | |||||
BR vectorizedBody<>(SB) | |||||
TEXT vectorizedBody<>(SB), NOSPLIT, $0 | |||||
XOR $0xffffffff, R2 // NOTW R2 | |||||
VLM 0(R5), CONST_PERM_LE2BE, CONST_CRC_POLY | |||||
// Load the initial CRC value into the rightmost word of V0 | |||||
VZERO V0 | |||||
VLVGF $3, R2, V0 | |||||
// Crash if the input size is less than 64-bytes. | |||||
CMP R4, $64 | |||||
BLT crash | |||||
// Load a 64-byte data chunk and XOR with CRC | |||||
VLM 0(R3), V1, V4 // 64-bytes into V1..V4 | |||||
// Reflect the data if the CRC operation is in the bit-reflected domain | |||||
VPERM V1, V1, CONST_PERM_LE2BE, V1 | |||||
VPERM V2, V2, CONST_PERM_LE2BE, V2 | |||||
VPERM V3, V3, CONST_PERM_LE2BE, V3 | |||||
VPERM V4, V4, CONST_PERM_LE2BE, V4 | |||||
VX V0, V1, V1 // V1 ^= CRC | |||||
ADD $64, R3 // BUF = BUF + 64 | |||||
ADD $(-64), R4 | |||||
// Check remaining buffer size and jump to proper folding method | |||||
CMP R4, $64 | |||||
BLT less_than_64bytes | |||||
fold_64bytes_loop: | |||||
// Load the next 64-byte data chunk into V5 to V8 | |||||
VLM 0(R3), V5, V8 | |||||
VPERM V5, V5, CONST_PERM_LE2BE, V5 | |||||
VPERM V6, V6, CONST_PERM_LE2BE, V6 | |||||
VPERM V7, V7, CONST_PERM_LE2BE, V7 | |||||
VPERM V8, V8, CONST_PERM_LE2BE, V8 | |||||
// Perform a GF(2) multiplication of the doublewords in V1 with | |||||
// the reduction constants in V0. The intermediate result is | |||||
// then folded (accumulated) with the next data chunk in V5 and | |||||
// stored in V1. Repeat this step for the register contents | |||||
// in V2, V3, and V4 respectively. | |||||
VGFMAG CONST_R2R1, V1, V5, V1 | |||||
VGFMAG CONST_R2R1, V2, V6, V2 | |||||
VGFMAG CONST_R2R1, V3, V7, V3 | |||||
VGFMAG CONST_R2R1, V4, V8, V4 | |||||
// Adjust buffer pointer and length for next loop | |||||
ADD $64, R3 // BUF = BUF + 64 | |||||
ADD $(-64), R4 // LEN = LEN - 64 | |||||
CMP R4, $64 | |||||
BGE fold_64bytes_loop | |||||
less_than_64bytes: | |||||
// Fold V1 to V4 into a single 128-bit value in V1 | |||||
VGFMAG CONST_R4R3, V1, V2, V1 | |||||
VGFMAG CONST_R4R3, V1, V3, V1 | |||||
VGFMAG CONST_R4R3, V1, V4, V1 | |||||
// Check whether to continue with 64-bit folding | |||||
CMP R4, $16 | |||||
BLT final_fold | |||||
fold_16bytes_loop: | |||||
VL 0(R3), V2 // Load next data chunk | |||||
VPERM V2, V2, CONST_PERM_LE2BE, V2 | |||||
VGFMAG CONST_R4R3, V1, V2, V1 // Fold next data chunk | |||||
// Adjust buffer pointer and size for folding next data chunk | |||||
ADD $16, R3 | |||||
ADD $-16, R4 | |||||
// Process remaining data chunks | |||||
CMP R4, $16 | |||||
BGE fold_16bytes_loop | |||||
final_fold: | |||||
VLEIB $7, $0x40, V9 | |||||
VSRLB V9, CONST_R4R3, V0 | |||||
VLEIG $0, $1, V0 | |||||
VGFMG V0, V1, V1 | |||||
VLEIB $7, $0x20, V9 // Shift by words | |||||
VSRLB V9, V1, V2 // Store remaining bits in V2 | |||||
VUPLLF V1, V1 // Split rightmost doubleword | |||||
VGFMAG CONST_R5, V1, V2, V1 // V1 = (V1 * R5) XOR V2 | |||||
// The input values to the Barret reduction are the degree-63 polynomial | |||||
// in V1 (R(x)), degree-32 generator polynomial, and the reduction | |||||
// constant u. The Barret reduction result is the CRC value of R(x) mod | |||||
// P(x). | |||||
// | |||||
// The Barret reduction algorithm is defined as: | |||||
// | |||||
// 1. T1(x) = floor( R(x) / x^32 ) GF2MUL u | |||||
// 2. T2(x) = floor( T1(x) / x^32 ) GF2MUL P(x) | |||||
// 3. C(x) = R(x) XOR T2(x) mod x^32 | |||||
// | |||||
// Note: To compensate the division by x^32, use the vector unpack | |||||
// instruction to move the leftmost word into the leftmost doubleword | |||||
// of the vector register. The rightmost doubleword is multiplied | |||||
// with zero to not contribute to the intermedate results. | |||||
// T1(x) = floor( R(x) / x^32 ) GF2MUL u | |||||
VUPLLF V1, V2 | |||||
VGFMG CONST_RU_POLY, V2, V2 | |||||
// Compute the GF(2) product of the CRC polynomial in VO with T1(x) in | |||||
// V2 and XOR the intermediate result, T2(x), with the value in V1. | |||||
// The final result is in the rightmost word of V2. | |||||
VUPLLF V2, V2 | |||||
VGFMAG CONST_CRC_POLY, V2, V1, V2 | |||||
done: | |||||
VLGVF $2, V2, R2 | |||||
XOR $0xffffffff, R2 // NOTW R2 | |||||
MOVWZ R2, ret + 32(FP) | |||||
RET | |||||
crash: | |||||
MOVD $0, (R0) // input size is less than 64-bytes |