vendor

vendor/github.com/ulikunitz/xz/internal/hash/cyclic_poly.go

@@ -0,0 +1,181 @@
+// Copyright 2014-2017 Ulrich Kunitz. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package hash
+
+// CyclicPoly provides a cyclic polynomial rolling hash.
+type CyclicPoly struct {
+	h uint64
+	p []uint64
+	i int
+}
+
+// ror rotates the unsigned 64-bit integer to right. The argument s must be
+// less than 64.
+func ror(x uint64, s uint) uint64 {
+	return (x >> s) | (x << (64 - s))
+}
+
+// NewCyclicPoly creates a new instance of the CyclicPoly structure. The
+// argument n gives the number of bytes for which a hash will be executed.
+// This number must be positive; the method panics if this isn't the case.
+func NewCyclicPoly(n int) *CyclicPoly {
+	if n < 1 {
+		panic("argument n must be positive")
+	}
+	return &CyclicPoly{p: make([]uint64, 0, n)}
+}
+
+// Len returns the length of the byte sequence for which a hash is generated.
+func (r *CyclicPoly) Len() int {
+	return cap(r.p)
+}
+
+// RollByte hashes the next byte and returns a hash value. The complete becomes
+// available after at least Len() bytes have been hashed.
+func (r *CyclicPoly) RollByte(x byte) uint64 {
+	y := hash[x]
+	if len(r.p) < cap(r.p) {
+		r.h = ror(r.h, 1) ^ y
+		r.p = append(r.p, y)
+	} else {
+		r.h ^= ror(r.p[r.i], uint(cap(r.p)-1))
+		r.h = ror(r.h, 1) ^ y
+		r.p[r.i] = y
+		r.i = (r.i + 1) % cap(r.p)
+	}
+	return r.h
+}
+
+// Stores the hash for the individual bytes.
+var hash = [256]uint64{
+	0x2e4fc3f904065142, 0xc790984cfbc99527,
+	0x879f95eb8c62f187, 0x3b61be86b5021ef2,
+	0x65a896a04196f0a5, 0xc5b307b80470b59e,
+	0xd3bff376a70df14b, 0xc332f04f0b3f1701,
+	0x753b5f0e9abf3e0d, 0xb41538fdfe66ef53,
+	0x1906a10c2c1c0208, 0xfb0c712a03421c0d,
+	0x38be311a65c9552b, 0xfee7ee4ca6445c7e,
+	0x71aadeded184f21e, 0xd73426fccda23b2d,
+	0x29773fb5fb9600b5, 0xce410261cd32981a,
+	0xfe2848b3c62dbc2d, 0x459eaaff6e43e11c,
+	0xc13e35fc9c73a887, 0xf30ed5c201e76dbc,
+	0xa5f10b3910482cea, 0x2945d59be02dfaad,
+	0x06ee334ff70571b5, 0xbabf9d8070f44380,
+	0xee3e2e9912ffd27c, 0x2a7118d1ea6b8ea7,
+	0x26183cb9f7b1664c, 0xea71dac7da068f21,
+	0xea92eca5bd1d0bb7, 0x415595862defcd75,
+	0x248a386023c60648, 0x9cf021ab284b3c8a,
+	0xfc9372df02870f6c, 0x2b92d693eeb3b3fc,
+	0x73e799d139dc6975, 0x7b15ae312486363c,
+	0xb70e5454a2239c80, 0x208e3fb31d3b2263,
+	0x01f563cabb930f44, 0x2ac4533d2a3240d8,
+	0x84231ed1064f6f7c, 0xa9f020977c2a6d19,
+	0x213c227271c20122, 0x09fe8a9a0a03d07a,
+	0x4236dc75bcaf910c, 0x460a8b2bead8f17e,
+	0xd9b27be1aa07055f, 0xd202d5dc4b11c33e,
+	0x70adb010543bea12, 0xcdae938f7ea6f579,
+	0x3f3d870208672f4d, 0x8e6ccbce9d349536,
+	0xe4c0871a389095ae, 0xf5f2a49152bca080,
+	0x9a43f9b97269934e, 0xc17b3753cb6f475c,
+	0xd56d941e8e206bd4, 0xac0a4f3e525eda00,
+	0xa06d5a011912a550, 0x5537ed19537ad1df,
+	0xa32fe713d611449d, 0x2a1d05b47c3b579f,
+	0x991d02dbd30a2a52, 0x39e91e7e28f93eb0,
+	0x40d06adb3e92c9ac, 0x9b9d3afde1c77c97,
+	0x9a3f3f41c02c616f, 0x22ecd4ba00f60c44,
+	0x0b63d5d801708420, 0x8f227ca8f37ffaec,
+	0x0256278670887c24, 0x107e14877dbf540b,
+	0x32c19f2786ac1c05, 0x1df5b12bb4bc9c61,
+	0xc0cac129d0d4c4e2, 0x9fdb52ee9800b001,
+	0x31f601d5d31c48c4, 0x72ff3c0928bcaec7,
+	0xd99264421147eb03, 0x535a2d6d38aefcfe,
+	0x6ba8b4454a916237, 0xfa39366eaae4719c,
+	0x10f00fd7bbb24b6f, 0x5bd23185c76c84d4,
+	0xb22c3d7e1b00d33f, 0x3efc20aa6bc830a8,
+	0xd61c2503fe639144, 0x30ce625441eb92d3,
+	0xe5d34cf359e93100, 0xa8e5aa13f2b9f7a5,
+	0x5c2b8d851ca254a6, 0x68fb6c5e8b0d5fdf,
+	0xc7ea4872c96b83ae, 0x6dd5d376f4392382,
+	0x1be88681aaa9792f, 0xfef465ee1b6c10d9,
+	0x1f98b65ed43fcb2e, 0x4d1ca11eb6e9a9c9,
+	0x7808e902b3857d0b, 0x171c9c4ea4607972,
+	0x58d66274850146df, 0x42b311c10d3981d1,
+	0x647fa8c621c41a4c, 0xf472771c66ddfedc,
+	0x338d27e3f847b46b, 0x6402ce3da97545ce,
+	0x5162db616fc38638, 0x9c83be97bc22a50e,
+	0x2d3d7478a78d5e72, 0xe621a9b938fd5397,
+	0x9454614eb0f81c45, 0x395fb6e742ed39b6,
+	0x77dd9179d06037bf, 0xc478d0fee4d2656d,
+	0x35d9d6cb772007af, 0x83a56e92c883f0f6,
+	0x27937453250c00a1, 0x27bd6ebc3a46a97d,
+	0x9f543bf784342d51, 0xd158f38c48b0ed52,
+	0x8dd8537c045f66b4, 0x846a57230226f6d5,
+	0x6b13939e0c4e7cdf, 0xfca25425d8176758,
+	0x92e5fc6cd52788e6, 0x9992e13d7a739170,
+	0x518246f7a199e8ea, 0xf104c2a71b9979c7,
+	0x86b3ffaabea4768f, 0x6388061cf3e351ad,
+	0x09d9b5295de5bbb5, 0x38bf1638c2599e92,
+	0x1d759846499e148d, 0x4c0ff015e5f96ef4,
+	0xa41a94cfa270f565, 0x42d76f9cb2326c0b,
+	0x0cf385dd3c9c23ba, 0x0508a6c7508d6e7a,
+	0x337523aabbe6cf8d, 0x646bb14001d42b12,
+	0xc178729d138adc74, 0xf900ef4491f24086,
+	0xee1a90d334bb5ac4, 0x9755c92247301a50,
+	0xb999bf7c4ff1b610, 0x6aeeb2f3b21e8fc9,
+	0x0fa8084cf91ac6ff, 0x10d226cf136e6189,
+	0xd302057a07d4fb21, 0x5f03800e20a0fcc3,
+	0x80118d4ae46bd210, 0x58ab61a522843733,
+	0x51edd575c5432a4b, 0x94ee6ff67f9197f7,
+	0x765669e0e5e8157b, 0xa5347830737132f0,
+	0x3ba485a69f01510c, 0x0b247d7b957a01c3,
+	0x1b3d63449fd807dc, 0x0fdc4721c30ad743,
+	0x8b535ed3829b2b14, 0xee41d0cad65d232c,
+	0xe6a99ed97a6a982f, 0x65ac6194c202003d,
+	0x692accf3a70573eb, 0xcc3c02c3e200d5af,
+	0x0d419e8b325914a3, 0x320f160f42c25e40,
+	0x00710d647a51fe7a, 0x3c947692330aed60,
+	0x9288aa280d355a7a, 0xa1806a9b791d1696,
+	0x5d60e38496763da1, 0x6c69e22e613fd0f4,
+	0x977fc2a5aadffb17, 0xfb7bd063fc5a94ba,
+	0x460c17992cbaece1, 0xf7822c5444d3297f,
+	0x344a9790c69b74aa, 0xb80a42e6cae09dce,
+	0x1b1361eaf2b1e757, 0xd84c1e758e236f01,
+	0x88e0b7be347627cc, 0x45246009b7a99490,
+	0x8011c6dd3fe50472, 0xc341d682bffb99d7,
+	0x2511be93808e2d15, 0xd5bc13d7fd739840,
+	0x2a3cd030679ae1ec, 0x8ad9898a4b9ee157,
+	0x3245fef0a8eaf521, 0x3d6d8dbbb427d2b0,
+	0x1ed146d8968b3981, 0x0c6a28bf7d45f3fc,
+	0x4a1fd3dbcee3c561, 0x4210ff6a476bf67e,
+	0xa559cce0d9199aac, 0xde39d47ef3723380,
+	0xe5b69d848ce42e35, 0xefa24296f8e79f52,
+	0x70190b59db9a5afc, 0x26f166cdb211e7bf,
+	0x4deaf2df3c6b8ef5, 0xf171dbdd670f1017,
+	0xb9059b05e9420d90, 0x2f0da855c9388754,
+	0x611d5e9ab77949cc, 0x2912038ac01163f4,
+	0x0231df50402b2fba, 0x45660fc4f3245f58,
+	0xb91cc97c7c8dac50, 0xb72d2aafe4953427,
+	0xfa6463f87e813d6b, 0x4515f7ee95d5c6a2,
+	0x1310e1c1a48d21c3, 0xad48a7810cdd8544,
+	0x4d5bdfefd5c9e631, 0xa43ed43f1fdcb7de,
+	0xe70cfc8fe1ee9626, 0xef4711b0d8dda442,
+	0xb80dd9bd4dab6c93, 0xa23be08d31ba4d93,
+	0x9b37db9d0335a39c, 0x494b6f870f5cfebc,
+	0x6d1b3c1149dda943, 0x372c943a518c1093,
+	0xad27af45e77c09c4, 0x3b6f92b646044604,
+	0xac2917909f5fcf4f, 0x2069a60e977e5557,
+	0x353a469e71014de5, 0x24be356281f55c15,
+	0x2b6d710ba8e9adea, 0x404ad1751c749c29,
+	0xed7311bf23d7f185, 0xba4f6976b4acc43e,
+	0x32d7198d2bc39000, 0xee667019014d6e01,
+	0x494ef3e128d14c83, 0x1f95a152baecd6be,
+	0x201648dff1f483a5, 0x68c28550c8384af6,
+	0x5fc834a6824a7f48, 0x7cd06cb7365eaf28,
+	0xd82bbd95e9b30909, 0x234f0d1694c53f6d,
+	0xd2fb7f4a96d83f4a, 0xff0d5da83acac05e,
+	0xf8f6b97f5585080a, 0x74236084be57b95b,
+	0xa25e40c03bbc36ad, 0x6b6e5c14ce88465b,
+	0x4378ffe93e1528c5, 0x94ca92a17118e2d2,
+}

vendor/github.com/ulikunitz/xz/internal/hash/cyclic_poly_test.go

@@ -0,0 +1,30 @@
+// Copyright 2014-2017 Ulrich Kunitz. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package hash
+
+import "testing"
+
+func TestCyclicPolySimple(t *testing.T) {
+	p := []byte("abcde")
+	r := NewCyclicPoly(4)
+	h2 := Hashes(r, p)
+	for i, h := range h2 {
+		w := Hashes(r, p[i:i+4])[0]
+		t.Logf("%d h=%#016x w=%#016x", i, h, w)
+		if h != w {
+			t.Errorf("rolling hash %d: %#016x; want %#016x",
+				i, h, w)
+		}
+	}
+}
+
+func BenchmarkCyclicPoly(b *testing.B) {
+	p := makeBenchmarkBytes(4096)
+	r := NewCyclicPoly(4)
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		Hashes(r, p)
+	}
+}

vendor/github.com/ulikunitz/xz/internal/hash/doc.go

@@ -0,0 +1,14 @@
+// Copyright 2014-2017 Ulrich Kunitz. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+/*
+Package hash provides rolling hashes.
+
+Rolling hashes have to be used for maintaining the positions of n-byte
+sequences in the dictionary buffer.
+
+The package provides currently the Rabin-Karp rolling hash and a Cyclic
+Polynomial hash. Both support the Hashes method to be used with an interface.
+*/
+package hash

vendor/github.com/ulikunitz/xz/internal/hash/rabin_karp.go

@@ -0,0 +1,66 @@
+// Copyright 2014-2017 Ulrich Kunitz. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package hash
+
+// A is the default constant for Robin-Karp rolling hash. This is a random
+// prime.
+const A = 0x97b548add41d5da1
+
+// RabinKarp supports the computation of a rolling hash.
+type RabinKarp struct {
+	A uint64
+	// a^n
+	aOldest uint64
+	h       uint64
+	p       []byte
+	i       int
+}
+
+// NewRabinKarp creates a new RabinKarp value. The argument n defines the
+// length of the byte sequence to be hashed. The default constant will will be
+// used.
+func NewRabinKarp(n int) *RabinKarp {
+	return NewRabinKarpConst(n, A)
+}
+
+// NewRabinKarpConst creates a new RabinKarp value. The argument n defines the
+// length of the byte sequence to be hashed. The argument a provides the
+// constant used to compute the hash.
+func NewRabinKarpConst(n int, a uint64) *RabinKarp {
+	if n <= 0 {
+		panic("number of bytes n must be positive")
+	}
+	aOldest := uint64(1)
+	// There are faster methods. For the small n required by the LZMA
+	// compressor O(n) is sufficient.
+	for i := 0; i < n; i++ {
+		aOldest *= a
+	}
+	return &RabinKarp{
+		A: a, aOldest: aOldest,
+		p: make([]byte, 0, n),
+	}
+}
+
+// Len returns the length of the byte sequence.
+func (r *RabinKarp) Len() int {
+	return cap(r.p)
+}
+
+// RollByte computes the hash after x has been added.
+func (r *RabinKarp) RollByte(x byte) uint64 {
+	if len(r.p) < cap(r.p) {
+		r.h += uint64(x)
+		r.h *= r.A
+		r.p = append(r.p, x)
+	} else {
+		r.h -= uint64(r.p[r.i]) * r.aOldest
+		r.h += uint64(x)
+		r.h *= r.A
+		r.p[r.i] = x
+		r.i = (r.i + 1) % cap(r.p)
+	}
+	return r.h
+}

vendor/github.com/ulikunitz/xz/internal/hash/rabin_karp_test.go

@@ -0,0 +1,42 @@
+// Copyright 2014-2017 Ulrich Kunitz. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package hash
+
+import (
+	"math/rand"
+	"testing"
+)
+
+func TestRabinKarpSimple(t *testing.T) {
+	p := []byte("abcde")
+	r := NewRabinKarp(4)
+	h2 := Hashes(r, p)
+	for i, h := range h2 {
+		w := Hashes(r, p[i:i+4])[0]
+		t.Logf("%d h=%#016x w=%#016x", i, h, w)
+		if h != w {
+			t.Errorf("rolling hash %d: %#016x; want %#016x",
+				i, h, w)
+		}
+	}
+}
+
+func makeBenchmarkBytes(n int) []byte {
+	rnd := rand.New(rand.NewSource(42))
+	p := make([]byte, n)
+	for i := range p {
+		p[i] = byte(rnd.Uint32())
+	}
+	return p
+}
+
+func BenchmarkRabinKarp(b *testing.B) {
+	p := makeBenchmarkBytes(4096)
+	r := NewRabinKarp(4)
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		Hashes(r, p)
+	}
+}

vendor/github.com/ulikunitz/xz/internal/hash/roller.go

@@ -0,0 +1,29 @@
+// Copyright 2014-2017 Ulrich Kunitz. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package hash
+
+// Roller provides an interface for rolling hashes. The hash value will become
+// valid after hash has been called Len times.
+type Roller interface {
+	Len() int
+	RollByte(x byte) uint64
+}
+
+// Hashes computes all hash values for the array p. Note that the state of the
+// roller is changed.
+func Hashes(r Roller, p []byte) []uint64 {
+	n := r.Len()
+	if len(p) < n {
+		return nil
+	}
+	h := make([]uint64, len(p)-n+1)
+	for i := 0; i < n-1; i++ {
+		r.RollByte(p[i])
+	}
+	for i := range h {
+		h[i] = r.RollByte(p[i+n-1])
+	}
+	return h
+}

vendor/github.com/ulikunitz/xz/internal/randtxt/groupreader.go

@@ -0,0 +1,82 @@
+// Copyright 2014-2017 Ulrich Kunitz. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package randtxt
+
+import (
+	"bufio"
+	"io"
+	"unicode"
+)
+
+// GroupReader groups the incoming text in groups of 5, whereby the
+// number of groups per line can be controlled.
+type GroupReader struct {
+	R             io.ByteReader
+	GroupsPerLine int
+	off           int64
+	eof           bool
+}
+
+// NewGroupReader creates a new group reader.
+func NewGroupReader(r io.Reader) *GroupReader {
+	return &GroupReader{R: bufio.NewReader(r)}
+}
+
+// Read formats the data provided by the internal reader in groups of 5
+// characters. If GroupsPerLine hasn't been initialized 8 groups per
+// line will be produced.
+func (r *GroupReader) Read(p []byte) (n int, err error) {
+	if r.eof {
+		return 0, io.EOF
+	}
+	groupsPerLine := r.GroupsPerLine
+	if groupsPerLine < 1 {
+		groupsPerLine = 8
+	}
+	lineLen := int64(groupsPerLine * 6)
+	var c byte
+	for i := range p {
+		switch {
+		case r.off%lineLen == lineLen-1:
+			if i+1 == len(p) && len(p) > 1 {
+				return i, nil
+			}
+			c = '\n'
+		case r.off%6 == 5:
+			if i+1 == len(p) && len(p) > 1 {
+				return i, nil
+			}
+			c = ' '
+		default:
+			c, err = r.R.ReadByte()
+			if err == io.EOF {
+				r.eof = true
+				if i > 0 {
+					switch p[i-1] {
+					case ' ':
+						p[i-1] = '\n'
+						fallthrough
+					case '\n':
+						return i, io.EOF
+					}
+				}
+				p[i] = '\n'
+				return i + 1, io.EOF
+			}
+			if err != nil {
+				return i, err
+			}
+			switch {
+			case c == ' ':
+				c = '_'
+			case !unicode.IsPrint(rune(c)):
+				c = '-'
+			}
+		}
+		p[i] = c
+		r.off++
+	}
+	return len(p), nil
+}

vendor/github.com/ulikunitz/xz/internal/randtxt/probs.go

@@ -0,0 +1,185 @@
+// Copyright 2014-2017 Ulrich Kunitz. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package randtxt supports the generation of random text using a
+// trigram model for the English language.
+package randtxt
+
+import (
+	"math"
+	"math/rand"
+	"sort"
+)
+
+// ngram stores an entry from the language model.
+type ngram struct {
+	s   string
+	lgP float64
+	lgQ float64
+}
+
+// ngrams represents a slice of ngram values and is used to represent a
+// language model.
+type ngrams []ngram
+
+func (s ngrams) Len() int           { return len(s) }
+func (s ngrams) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
+func (s ngrams) Less(i, j int) bool { return s[i].s < s[j].s }
+
+// Sorts the language model in the sequence of their ngrams.
+func (s ngrams) Sort() { sort.Sort(s) }
+
+// Search is looking for an ngram or the position where it would be
+// inserted.
+func (s ngrams) Search(g string) int {
+	return sort.Search(len(s), func(k int) bool { return s[k].s >= g })
+}
+
+// prob represents a string, usually an ngram, and a probability value.
+type prob struct {
+	s string
+	p float64
+}
+
+// probs is a slice of prob values that can be sorted and searched.
+type probs []prob
+
+func (s probs) Len() int           { return len(s) }
+func (s probs) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
+func (s probs) Less(i, j int) bool { return s[i].s < s[j].s }
+
+// SortByNgram sorts the probs slice by ngram, field s.
+func (s probs) SortByNgram() { sort.Sort(s) }
+
+// SortsByProb sorts the probs slice by probability, field p.
+func (s probs) SortByProb() { sort.Sort(byProb{s}) }
+
+// SearchNgram searches for an ngram or the position where it would be
+// inserted.
+func (s probs) SearchNgram(g string) int {
+	return sort.Search(len(s), func(k int) bool { return s[k].s >= g })
+}
+
+// SearchProb searches ngrams for a specific probability or where it
+// would be inserted.
+func (s probs) SearchProb(p float64) int {
+	return sort.Search(len(s), func(k int) bool { return s[k].p >= p })
+}
+
+// byProb is used to sort probs slice by probability, field p.
+type byProb struct {
+	probs
+}
+
+func (s byProb) Less(i, j int) bool {
+	return s.probs[i].p < s.probs[j].p
+}
+
+// cdf can be used to setup a cumulative distribution function
+// represented by a probs slice. We should have returned an actual
+// function.
+func cdf(n int, p func(i int) prob) probs {
+	prs := make(probs, n)
+	sum := 0.0
+	for i := range prs {
+		pr := p(i)
+		sum += pr.p
+		prs[i] = pr
+	}
+	q := 1.0 / sum
+	x := 0.0
+	for i, pr := range prs {
+		x += pr.p * q
+		if x > 1.0 {
+			x = 1.0
+		}
+		prs[i].p = x
+	}
+	if !sort.IsSorted(byProb{prs}) {
+		panic("cdf not sorted")
+	}
+	return prs
+}
+
+// pCDFOfLM converts a language model into a cumulative distribution
+// function represented by probs.
+func pCDFOfLM(lm ngrams) probs {
+	return cdf(len(lm), func(i int) prob {
+		return prob{lm[i].s, math.Exp2(lm[i].lgP)}
+	})
+}
+
+// cCDF converts a ngrams slice into a cumulative distribution function
+// using the conditional probability lgQ.
+func cCDF(s ngrams) probs {
+	return cdf(len(s), func(i int) prob {
+		return prob{s[i].s, math.Exp2(s[i].lgQ)}
+	})
+}
+
+// comap contains a map of conditional distribution function for the
+// last character.
+type comap map[string]probs
+
+// comapOfLM converts a language model in a map of conditional
+// distribution functions.
+func comapOfLM(lm ngrams) comap {
+	if !sort.IsSorted(lm) {
+		panic("lm is not sorted")
+	}
+	m := make(comap, 26*26)
+	for i := 0; i < len(lm); {
+		j := i
+		g := lm[i].s
+		g2 := g[:2]
+		z := g2 + "Z"
+		i = lm.Search(z)
+		if i >= len(lm) || lm[i].s != z {
+			panic("unexpected search result")
+		}
+		i++
+		m[g2] = cCDF(lm[j:i])
+	}
+	return m
+}
+
+// trigram returns the trigram with prefix g2 using a probability value
+// in the range [0.0,1.0).
+func (c comap) trigram(g2 string, p float64) string {
+	prs := c[g2]
+	i := prs.SearchProb(p)
+	return prs[i].s
+}
+
+var (
+	// CDF for normal probabilities
+	pcdf = pCDFOfLM(englm3)
+	// map of two letter conditionals
+	cmap = comapOfLM(englm3)
+)
+
+// Reader generates a stream of text of uppercase letters with trigrams
+// distributed according to a language model of the English language.
+type Reader struct {
+	rnd *rand.Rand
+	g3  string
+}
+
+// NewReader creates a new reader. The argument src must create a uniformly
+// distributed stream of random values.
+func NewReader(src rand.Source) *Reader {
+	rnd := rand.New(src)
+	i := pcdf.SearchProb(rnd.Float64())
+	return &Reader{rnd, pcdf[i].s}
+}
+
+// Read reads random text. The Read function will always return len(p)
+// bytes and will never return an error.
+func (r *Reader) Read(p []byte) (n int, err error) {
+	for i := range p {
+		r.g3 = cmap.trigram(r.g3[1:], r.rnd.Float64())
+		p[i] = r.g3[2]
+	}
+	return len(p), nil
+}

vendor/github.com/ulikunitz/xz/internal/randtxt/probs_test.go

@@ -0,0 +1,37 @@
+// Copyright 2014-2017 Ulrich Kunitz. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package randtxt
+
+import (
+	"bufio"
+	"io"
+	"math/rand"
+	"testing"
+)
+
+func TestReader(t *testing.T) {
+	lr := io.LimitReader(NewReader(rand.NewSource(13)), 195)
+	pretty := NewGroupReader(lr)
+	scanner := bufio.NewScanner(pretty)
+	for scanner.Scan() {
+		t.Log(scanner.Text())
+	}
+	if err := scanner.Err(); err != nil {
+		t.Fatalf("scanner error %s", err)
+	}
+}
+
+func TestComap(t *testing.T) {
+	prs := cmap["TH"]
+	for _, p := range prs[3:6] {
+		t.Logf("%v", p)
+	}
+	p := 0.2
+	x := cmap.trigram("TH", p)
+	if x != "THE" {
+		t.Fatalf("cmap.trigram(%q, %.1f) returned %q; want %q",
+			"TH", p, x, "THE")
+	}
+}

vendor/github.com/ulikunitz/xz/internal/xlog/xlog.go

@@ -0,0 +1,457 @@
+// Copyright 2014-2017 Ulrich Kunitz. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package xlog provides a simple logging package that allows to disable
+// certain message categories. It defines a type, Logger, with multiple
+// methods for formatting output. The package has also a predefined
+// 'standard' Logger accessible through helper function Print[f|ln],
+// Fatal[f|ln], Panic[f|ln], Warn[f|ln], Print[f|ln] and Debug[f|ln]
+// that are easier to use then creating a Logger manually. That logger
+// writes to standard error and prints the date and time of each logged
+// message, which can be configured using the function SetFlags.
+//
+// The Fatal functions call os.Exit(1) after the message is output
+// unless not suppressed by the flags. The Panic functions call panic
+// after the writing the log message unless suppressed.
+package xlog
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"runtime"
+	"sync"
+	"time"
+)
+
+// The flags define what information is prefixed to each log entry
+// generated by the Logger. The Lno* versions allow the suppression of
+// specific output. The bits are or'ed together to control what will be
+// printed. There is no control over the order of the items printed and
+// the format. The full format is:
+//
+//   2009-01-23 01:23:23.123123 /a/b/c/d.go:23: message
+//
+const (
+	Ldate         = 1 << iota // the date: 2009-01-23
+	Ltime                     // the time: 01:23:23
+	Lmicroseconds             // microsecond resolution: 01:23:23.123123
+	Llongfile                 // full file name and line number: /a/b/c/d.go:23
+	Lshortfile                // final file name element and line number: d.go:23
+	Lnopanic                  // suppresses output from Panic[f|ln] but not the panic call
+	Lnofatal                  // suppresses output from Fatal[f|ln] but not the exit
+	Lnowarn                   // suppresses output from Warn[f|ln]
+	Lnoprint                  // suppresses output from Print[f|ln]
+	Lnodebug                  // suppresses output from Debug[f|ln]
+	// initial values for the standard logger
+	Lstdflags = Ldate | Ltime | Lnodebug
+)
+
+// A Logger represents an active logging object that generates lines of
+// output to an io.Writer. Each logging operation if not suppressed
+// makes a single call to the Writer's Write method. A Logger can be
+// used simultaneously from multiple goroutines; it guarantees to
+// serialize access to the Writer.
+type Logger struct {
+	mu sync.Mutex // ensures atomic writes; and protects the following
+	// fields
+	prefix string    // prefix to write at beginning of each line
+	flag   int       // properties
+	out    io.Writer // destination for output
+	buf    []byte    // for accumulating text to write
+}
+
+// New creates a new Logger. The out argument sets the destination to
+// which the log output will be written. The prefix appears at the
+// beginning of each log line. The flag argument defines the logging
+// properties.
+func New(out io.Writer, prefix string, flag int) *Logger {
+	return &Logger{out: out, prefix: prefix, flag: flag}
+}
+
+// std is the standard logger used by the package scope functions.
+var std = New(os.Stderr, "", Lstdflags)
+
+// itoa converts the integer to ASCII. A negative widths will avoid
+// zero-padding. The function supports only non-negative integers.
+func itoa(buf *[]byte, i int, wid int) {
+	var u = uint(i)
+	if u == 0 && wid <= 1 {
+		*buf = append(*buf, '0')
+		return
+	}
+	var b [32]byte
+	bp := len(b)
+	for ; u > 0 || wid > 0; u /= 10 {
+		bp--
+		wid--
+		b[bp] = byte(u%10) + '0'
+	}
+	*buf = append(*buf, b[bp:]...)
+}
+
+// formatHeader puts the header into the buf field of the buffer.
+func (l *Logger) formatHeader(t time.Time, file string, line int) {
+	l.buf = append(l.buf, l.prefix...)
+	if l.flag&(Ldate|Ltime|Lmicroseconds) != 0 {
+		if l.flag&Ldate != 0 {
+			year, month, day := t.Date()
+			itoa(&l.buf, year, 4)
+			l.buf = append(l.buf, '-')
+			itoa(&l.buf, int(month), 2)
+			l.buf = append(l.buf, '-')
+			itoa(&l.buf, day, 2)
+			l.buf = append(l.buf, ' ')
+		}
+		if l.flag&(Ltime|Lmicroseconds) != 0 {
+			hour, min, sec := t.Clock()
+			itoa(&l.buf, hour, 2)
+			l.buf = append(l.buf, ':')
+			itoa(&l.buf, min, 2)
+			l.buf = append(l.buf, ':')
+			itoa(&l.buf, sec, 2)
+			if l.flag&Lmicroseconds != 0 {
+				l.buf = append(l.buf, '.')
+				itoa(&l.buf, t.Nanosecond()/1e3, 6)
+			}
+			l.buf = append(l.buf, ' ')
+		}
+	}
+	if l.flag&(Lshortfile|Llongfile) != 0 {
+		if l.flag&Lshortfile != 0 {
+			short := file
+			for i := len(file) - 1; i > 0; i-- {
+				if file[i] == '/' {
+					short = file[i+1:]
+					break
+				}
+			}
+			file = short
+		}
+		l.buf = append(l.buf, file...)
+		l.buf = append(l.buf, ':')
+		itoa(&l.buf, line, -1)
+		l.buf = append(l.buf, ": "...)
+	}
+}
+
+func (l *Logger) output(calldepth int, now time.Time, s string) error {
+	var file string
+	var line int
+	if l.flag&(Lshortfile|Llongfile) != 0 {
+		l.mu.Unlock()
+		var ok bool
+		_, file, line, ok = runtime.Caller(calldepth)
+		if !ok {
+			file = "???"
+			line = 0
+		}
+		l.mu.Lock()
+	}
+	l.buf = l.buf[:0]
+	l.formatHeader(now, file, line)
+	l.buf = append(l.buf, s...)
+	if len(s) == 0 || s[len(s)-1] != '\n' {
+		l.buf = append(l.buf, '\n')
+	}
+	_, err := l.out.Write(l.buf)
+	return err
+}
+
+// Output writes the string s with the header controlled by the flags to
+// the l.out writer. A newline will be appended if s doesn't end in a
+// newline. Calldepth is used to recover the PC, although all current
+// calls of Output use the call depth 2. Access to the function is serialized.
+func (l *Logger) Output(calldepth, noflag int, v ...interface{}) error {
+	now := time.Now()
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	if l.flag&noflag != 0 {
+		return nil
+	}
+	s := fmt.Sprint(v...)
+	return l.output(calldepth+1, now, s)
+}
+
+// Outputf works like output but formats the output like Printf.
+func (l *Logger) Outputf(calldepth int, noflag int, format string, v ...interface{}) error {
+	now := time.Now()
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	if l.flag&noflag != 0 {
+		return nil
+	}
+	s := fmt.Sprintf(format, v...)
+	return l.output(calldepth+1, now, s)
+}
+
+// Outputln works like output but formats the output like Println.
+func (l *Logger) Outputln(calldepth int, noflag int, v ...interface{}) error {
+	now := time.Now()
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	if l.flag&noflag != 0 {
+		return nil
+	}
+	s := fmt.Sprintln(v...)
+	return l.output(calldepth+1, now, s)
+}
+
+// Panic prints the message like Print and calls panic. The printing
+// might be suppressed by the flag Lnopanic.
+func (l *Logger) Panic(v ...interface{}) {
+	l.Output(2, Lnopanic, v...)
+	s := fmt.Sprint(v...)
+	panic(s)
+}
+
+// Panic prints the message like Print and calls panic. The printing
+// might be suppressed by the flag Lnopanic.
+func Panic(v ...interface{}) {
+	std.Output(2, Lnopanic, v...)
+	s := fmt.Sprint(v...)
+	panic(s)
+}
+
+// Panicf prints the message like Printf and calls panic. The printing
+// might be suppressed by the flag Lnopanic.
+func (l *Logger) Panicf(format string, v ...interface{}) {
+	l.Outputf(2, Lnopanic, format, v...)
+	s := fmt.Sprintf(format, v...)
+	panic(s)
+}
+
+// Panicf prints the message like Printf and calls panic. The printing
+// might be suppressed by the flag Lnopanic.
+func Panicf(format string, v ...interface{}) {
+	std.Outputf(2, Lnopanic, format, v...)
+	s := fmt.Sprintf(format, v...)
+	panic(s)
+}
+
+// Panicln prints the message like Println and calls panic. The printing
+// might be suppressed by the flag Lnopanic.
+func (l *Logger) Panicln(v ...interface{}) {
+	l.Outputln(2, Lnopanic, v...)
+	s := fmt.Sprintln(v...)
+	panic(s)
+}
+
+// Panicln prints the message like Println and calls panic. The printing
+// might be suppressed by the flag Lnopanic.
+func Panicln(v ...interface{}) {
+	std.Outputln(2, Lnopanic, v...)
+	s := fmt.Sprintln(v...)
+	panic(s)
+}
+
+// Fatal prints the message like Print and calls os.Exit(1). The
+// printing might be suppressed by the flag Lnofatal.
+func (l *Logger) Fatal(v ...interface{}) {
+	l.Output(2, Lnofatal, v...)
+	os.Exit(1)
+}
+
+// Fatal prints the message like Print and calls os.Exit(1). The
+// printing might be suppressed by the flag Lnofatal.
+func Fatal(v ...interface{}) {
+	std.Output(2, Lnofatal, v...)
+	os.Exit(1)
+}
+
+// Fatalf prints the message like Printf and calls os.Exit(1). The
+// printing might be suppressed by the flag Lnofatal.
+func (l *Logger) Fatalf(format string, v ...interface{}) {
+	l.Outputf(2, Lnofatal, format, v...)
+	os.Exit(1)
+}
+
+// Fatalf prints the message like Printf and calls os.Exit(1). The
+// printing might be suppressed by the flag Lnofatal.
+func Fatalf(format string, v ...interface{}) {
+	std.Outputf(2, Lnofatal, format, v...)
+	os.Exit(1)
+}
+
+// Fatalln prints the message like Println and calls os.Exit(1). The
+// printing might be suppressed by the flag Lnofatal.
+func (l *Logger) Fatalln(format string, v ...interface{}) {
+	l.Outputln(2, Lnofatal, v...)
+	os.Exit(1)
+}
+
+// Fatalln prints the message like Println and calls os.Exit(1). The
+// printing might be suppressed by the flag Lnofatal.
+func Fatalln(format string, v ...interface{}) {
+	std.Outputln(2, Lnofatal, v...)
+	os.Exit(1)
+}
+
+// Warn prints the message like Print. The printing might be suppressed
+// by the flag Lnowarn.
+func (l *Logger) Warn(v ...interface{}) {
+	l.Output(2, Lnowarn, v...)
+}
+
+// Warn prints the message like Print. The printing might be suppressed
+// by the flag Lnowarn.
+func Warn(v ...interface{}) {
+	std.Output(2, Lnowarn, v...)
+}
+
+// Warnf prints the message like Printf. The printing might be suppressed
+// by the flag Lnowarn.
+func (l *Logger) Warnf(format string, v ...interface{}) {
+	l.Outputf(2, Lnowarn, format, v...)
+}
+
+// Warnf prints the message like Printf. The printing might be suppressed
+// by the flag Lnowarn.
+func Warnf(format string, v ...interface{}) {
+	std.Outputf(2, Lnowarn, format, v...)
+}
+
+// Warnln prints the message like Println. The printing might be suppressed
+// by the flag Lnowarn.
+func (l *Logger) Warnln(v ...interface{}) {
+	l.Outputln(2, Lnowarn, v...)
+}
+
+// Warnln prints the message like Println. The printing might be suppressed
+// by the flag Lnowarn.
+func Warnln(v ...interface{}) {
+	std.Outputln(2, Lnowarn, v...)
+}
+
+// Print prints the message like fmt.Print. The printing might be suppressed
+// by the flag Lnoprint.
+func (l *Logger) Print(v ...interface{}) {
+	l.Output(2, Lnoprint, v...)
+}
+
+// Print prints the message like fmt.Print. The printing might be suppressed
+// by the flag Lnoprint.
+func Print(v ...interface{}) {
+	std.Output(2, Lnoprint, v...)
+}
+
+// Printf prints the message like fmt.Printf. The printing might be suppressed
+// by the flag Lnoprint.
+func (l *Logger) Printf(format string, v ...interface{}) {
+	l.Outputf(2, Lnoprint, format, v...)
+}
+
+// Printf prints the message like fmt.Printf. The printing might be suppressed
+// by the flag Lnoprint.
+func Printf(format string, v ...interface{}) {
+	std.Outputf(2, Lnoprint, format, v...)
+}
+
+// Println prints the message like fmt.Println. The printing might be
+// suppressed by the flag Lnoprint.
+func (l *Logger) Println(v ...interface{}) {
+	l.Outputln(2, Lnoprint, v...)
+}
+
+// Println prints the message like fmt.Println. The printing might be
+// suppressed by the flag Lnoprint.
+func Println(v ...interface{}) {
+	std.Outputln(2, Lnoprint, v...)
+}
+
+// Debug prints the message like Print. The printing might be suppressed
+// by the flag Lnodebug.
+func (l *Logger) Debug(v ...interface{}) {
+	l.Output(2, Lnodebug, v...)
+}
+
+// Debug prints the message like Print. The printing might be suppressed
+// by the flag Lnodebug.
+func Debug(v ...interface{}) {
+	std.Output(2, Lnodebug, v...)
+}
+
+// Debugf prints the message like Printf. The printing might be suppressed
+// by the flag Lnodebug.
+func (l *Logger) Debugf(format string, v ...interface{}) {
+	l.Outputf(2, Lnodebug, format, v...)
+}
+
+// Debugf prints the message like Printf. The printing might be suppressed
+// by the flag Lnodebug.
+func Debugf(format string, v ...interface{}) {
+	std.Outputf(2, Lnodebug, format, v...)
+}
+
+// Debugln prints the message like Println. The printing might be suppressed
+// by the flag Lnodebug.
+func (l *Logger) Debugln(v ...interface{}) {
+	l.Outputln(2, Lnodebug, v...)
+}
+
+// Debugln prints the message like Println. The printing might be suppressed
+// by the flag Lnodebug.
+func Debugln(v ...interface{}) {
+	std.Outputln(2, Lnodebug, v...)
+}
+
+// Flags returns the current flags used by the logger.
+func (l *Logger) Flags() int {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	return l.flag
+}
+
+// Flags returns the current flags used by the standard logger.
+func Flags() int {
+	return std.Flags()
+}
+
+// SetFlags sets the flags of the logger.
+func (l *Logger) SetFlags(flag int) {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	l.flag = flag
+}
+
+// SetFlags sets the flags for the standard logger.
+func SetFlags(flag int) {
+	std.SetFlags(flag)
+}
+
+// Prefix returns the prefix used by the logger.
+func (l *Logger) Prefix() string {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	return l.prefix
+}
+
+// Prefix returns the prefix used by the standard logger of the package.
+func Prefix() string {
+	return std.Prefix()
+}
+
+// SetPrefix sets the prefix for the logger.
+func (l *Logger) SetPrefix(prefix string) {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	l.prefix = prefix
+}
+
+// SetPrefix sets the prefix of the standard logger of the package.
+func SetPrefix(prefix string) {
+	std.SetPrefix(prefix)
+}
+
+// SetOutput sets the output of the logger.
+func (l *Logger) SetOutput(w io.Writer) {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	l.out = w
+}
+
+// SetOutput sets the output for the standard logger of the package.
+func SetOutput(w io.Writer) {
+	std.SetOutput(w)
+}