huffman_code.go 9.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344
  1. // Copyright 2009 The Go Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style
  3. // license that can be found in the LICENSE file.
  4. package flate
  5. import (
  6. "math"
  7. "sort"
  8. )
  9. // hcode is a huffman code with a bit code and bit length.
  10. type hcode struct {
  11. code, len uint16
  12. }
  13. type huffmanEncoder struct {
  14. codes []hcode
  15. freqcache []literalNode
  16. bitCount [17]int32
  17. lns byLiteral // stored to avoid repeated allocation in generate
  18. lfs byFreq // stored to avoid repeated allocation in generate
  19. }
  20. type literalNode struct {
  21. literal uint16
  22. freq int32
  23. }
  24. // A levelInfo describes the state of the constructed tree for a given depth.
  25. type levelInfo struct {
  26. // Our level. for better printing
  27. level int32
  28. // The frequency of the last node at this level
  29. lastFreq int32
  30. // The frequency of the next character to add to this level
  31. nextCharFreq int32
  32. // The frequency of the next pair (from level below) to add to this level.
  33. // Only valid if the "needed" value of the next lower level is 0.
  34. nextPairFreq int32
  35. // The number of chains remaining to generate for this level before moving
  36. // up to the next level
  37. needed int32
  38. }
  39. // set sets the code and length of an hcode.
  40. func (h *hcode) set(code uint16, length uint16) {
  41. h.len = length
  42. h.code = code
  43. }
  44. func maxNode() literalNode { return literalNode{math.MaxUint16, math.MaxInt32} }
  45. func newHuffmanEncoder(size int) *huffmanEncoder {
  46. return &huffmanEncoder{codes: make([]hcode, size)}
  47. }
  48. // Generates a HuffmanCode corresponding to the fixed literal table
  49. func generateFixedLiteralEncoding() *huffmanEncoder {
  50. h := newHuffmanEncoder(maxNumLit)
  51. codes := h.codes
  52. var ch uint16
  53. for ch = 0; ch < maxNumLit; ch++ {
  54. var bits uint16
  55. var size uint16
  56. switch {
  57. case ch < 144:
  58. // size 8, 000110000 .. 10111111
  59. bits = ch + 48
  60. size = 8
  61. break
  62. case ch < 256:
  63. // size 9, 110010000 .. 111111111
  64. bits = ch + 400 - 144
  65. size = 9
  66. break
  67. case ch < 280:
  68. // size 7, 0000000 .. 0010111
  69. bits = ch - 256
  70. size = 7
  71. break
  72. default:
  73. // size 8, 11000000 .. 11000111
  74. bits = ch + 192 - 280
  75. size = 8
  76. }
  77. codes[ch] = hcode{code: reverseBits(bits, byte(size)), len: size}
  78. }
  79. return h
  80. }
  81. func generateFixedOffsetEncoding() *huffmanEncoder {
  82. h := newHuffmanEncoder(30)
  83. codes := h.codes
  84. for ch := range codes {
  85. codes[ch] = hcode{code: reverseBits(uint16(ch), 5), len: 5}
  86. }
  87. return h
  88. }
  89. var fixedLiteralEncoding *huffmanEncoder = generateFixedLiteralEncoding()
  90. var fixedOffsetEncoding *huffmanEncoder = generateFixedOffsetEncoding()
  91. func (h *huffmanEncoder) bitLength(freq []int32) int {
  92. var total int
  93. for i, f := range freq {
  94. if f != 0 {
  95. total += int(f) * int(h.codes[i].len)
  96. }
  97. }
  98. return total
  99. }
  100. const maxBitsLimit = 16
  101. // Return the number of literals assigned to each bit size in the Huffman encoding
  102. //
  103. // This method is only called when list.length >= 3
  104. // The cases of 0, 1, and 2 literals are handled by special case code.
  105. //
  106. // list An array of the literals with non-zero frequencies
  107. // and their associated frequencies. The array is in order of increasing
  108. // frequency, and has as its last element a special element with frequency
  109. // MaxInt32
  110. // maxBits The maximum number of bits that should be used to encode any literal.
  111. // Must be less than 16.
  112. // return An integer array in which array[i] indicates the number of literals
  113. // that should be encoded in i bits.
  114. func (h *huffmanEncoder) bitCounts(list []literalNode, maxBits int32) []int32 {
  115. if maxBits >= maxBitsLimit {
  116. panic("flate: maxBits too large")
  117. }
  118. n := int32(len(list))
  119. list = list[0 : n+1]
  120. list[n] = maxNode()
  121. // The tree can't have greater depth than n - 1, no matter what. This
  122. // saves a little bit of work in some small cases
  123. if maxBits > n-1 {
  124. maxBits = n - 1
  125. }
  126. // Create information about each of the levels.
  127. // A bogus "Level 0" whose sole purpose is so that
  128. // level1.prev.needed==0. This makes level1.nextPairFreq
  129. // be a legitimate value that never gets chosen.
  130. var levels [maxBitsLimit]levelInfo
  131. // leafCounts[i] counts the number of literals at the left
  132. // of ancestors of the rightmost node at level i.
  133. // leafCounts[i][j] is the number of literals at the left
  134. // of the level j ancestor.
  135. var leafCounts [maxBitsLimit][maxBitsLimit]int32
  136. for level := int32(1); level <= maxBits; level++ {
  137. // For every level, the first two items are the first two characters.
  138. // We initialize the levels as if we had already figured this out.
  139. levels[level] = levelInfo{
  140. level: level,
  141. lastFreq: list[1].freq,
  142. nextCharFreq: list[2].freq,
  143. nextPairFreq: list[0].freq + list[1].freq,
  144. }
  145. leafCounts[level][level] = 2
  146. if level == 1 {
  147. levels[level].nextPairFreq = math.MaxInt32
  148. }
  149. }
  150. // We need a total of 2*n - 2 items at top level and have already generated 2.
  151. levels[maxBits].needed = 2*n - 4
  152. level := maxBits
  153. for {
  154. l := &levels[level]
  155. if l.nextPairFreq == math.MaxInt32 && l.nextCharFreq == math.MaxInt32 {
  156. // We've run out of both leafs and pairs.
  157. // End all calculations for this level.
  158. // To make sure we never come back to this level or any lower level,
  159. // set nextPairFreq impossibly large.
  160. l.needed = 0
  161. levels[level+1].nextPairFreq = math.MaxInt32
  162. level++
  163. continue
  164. }
  165. prevFreq := l.lastFreq
  166. if l.nextCharFreq < l.nextPairFreq {
  167. // The next item on this row is a leaf node.
  168. n := leafCounts[level][level] + 1
  169. l.lastFreq = l.nextCharFreq
  170. // Lower leafCounts are the same of the previous node.
  171. leafCounts[level][level] = n
  172. l.nextCharFreq = list[n].freq
  173. } else {
  174. // The next item on this row is a pair from the previous row.
  175. // nextPairFreq isn't valid until we generate two
  176. // more values in the level below
  177. l.lastFreq = l.nextPairFreq
  178. // Take leaf counts from the lower level, except counts[level] remains the same.
  179. copy(leafCounts[level][:level], leafCounts[level-1][:level])
  180. levels[l.level-1].needed = 2
  181. }
  182. if l.needed--; l.needed == 0 {
  183. // We've done everything we need to do for this level.
  184. // Continue calculating one level up. Fill in nextPairFreq
  185. // of that level with the sum of the two nodes we've just calculated on
  186. // this level.
  187. if l.level == maxBits {
  188. // All done!
  189. break
  190. }
  191. levels[l.level+1].nextPairFreq = prevFreq + l.lastFreq
  192. level++
  193. } else {
  194. // If we stole from below, move down temporarily to replenish it.
  195. for levels[level-1].needed > 0 {
  196. level--
  197. }
  198. }
  199. }
  200. // Somethings is wrong if at the end, the top level is null or hasn't used
  201. // all of the leaves.
  202. if leafCounts[maxBits][maxBits] != n {
  203. panic("leafCounts[maxBits][maxBits] != n")
  204. }
  205. bitCount := h.bitCount[:maxBits+1]
  206. bits := 1
  207. counts := &leafCounts[maxBits]
  208. for level := maxBits; level > 0; level-- {
  209. // chain.leafCount gives the number of literals requiring at least "bits"
  210. // bits to encode.
  211. bitCount[bits] = counts[level] - counts[level-1]
  212. bits++
  213. }
  214. return bitCount
  215. }
  216. // Look at the leaves and assign them a bit count and an encoding as specified
  217. // in RFC 1951 3.2.2
  218. func (h *huffmanEncoder) assignEncodingAndSize(bitCount []int32, list []literalNode) {
  219. code := uint16(0)
  220. for n, bits := range bitCount {
  221. code <<= 1
  222. if n == 0 || bits == 0 {
  223. continue
  224. }
  225. // The literals list[len(list)-bits] .. list[len(list)-bits]
  226. // are encoded using "bits" bits, and get the values
  227. // code, code + 1, .... The code values are
  228. // assigned in literal order (not frequency order).
  229. chunk := list[len(list)-int(bits):]
  230. h.lns.sort(chunk)
  231. for _, node := range chunk {
  232. h.codes[node.literal] = hcode{code: reverseBits(code, uint8(n)), len: uint16(n)}
  233. code++
  234. }
  235. list = list[0 : len(list)-int(bits)]
  236. }
  237. }
  238. // Update this Huffman Code object to be the minimum code for the specified frequency count.
  239. //
  240. // freq An array of frequencies, in which frequency[i] gives the frequency of literal i.
  241. // maxBits The maximum number of bits to use for any literal.
  242. func (h *huffmanEncoder) generate(freq []int32, maxBits int32) {
  243. if h.freqcache == nil {
  244. // Allocate a reusable buffer with the longest possible frequency table.
  245. // Possible lengths are codegenCodeCount, offsetCodeCount and maxNumLit.
  246. // The largest of these is maxNumLit, so we allocate for that case.
  247. h.freqcache = make([]literalNode, maxNumLit+1)
  248. }
  249. list := h.freqcache[:len(freq)+1]
  250. // Number of non-zero literals
  251. count := 0
  252. // Set list to be the set of all non-zero literals and their frequencies
  253. for i, f := range freq {
  254. if f != 0 {
  255. list[count] = literalNode{uint16(i), f}
  256. count++
  257. } else {
  258. list[count] = literalNode{}
  259. h.codes[i].len = 0
  260. }
  261. }
  262. list[len(freq)] = literalNode{}
  263. list = list[:count]
  264. if count <= 2 {
  265. // Handle the small cases here, because they are awkward for the general case code. With
  266. // two or fewer literals, everything has bit length 1.
  267. for i, node := range list {
  268. // "list" is in order of increasing literal value.
  269. h.codes[node.literal].set(uint16(i), 1)
  270. }
  271. return
  272. }
  273. h.lfs.sort(list)
  274. // Get the number of literals for each bit count
  275. bitCount := h.bitCounts(list, maxBits)
  276. // And do the assignment
  277. h.assignEncodingAndSize(bitCount, list)
  278. }
  279. type byLiteral []literalNode
  280. func (s *byLiteral) sort(a []literalNode) {
  281. *s = byLiteral(a)
  282. sort.Sort(s)
  283. }
  284. func (s byLiteral) Len() int { return len(s) }
  285. func (s byLiteral) Less(i, j int) bool {
  286. return s[i].literal < s[j].literal
  287. }
  288. func (s byLiteral) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
  289. type byFreq []literalNode
  290. func (s *byFreq) sort(a []literalNode) {
  291. *s = byFreq(a)
  292. sort.Sort(s)
  293. }
  294. func (s byFreq) Len() int { return len(s) }
  295. func (s byFreq) Less(i, j int) bool {
  296. if s[i].freq == s[j].freq {
  297. return s[i].literal < s[j].literal
  298. }
  299. return s[i].freq < s[j].freq
  300. }
  301. func (s byFreq) Swap(i, j int) { s[i], s[j] = s[j], s[i] }