RLE Compression for File Formats #95
|
@ -4,6 +4,7 @@ import (
|
|||
"bytes"
|
||||
"encoding/binary"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"image"
|
||||
"math"
|
||||
|
@ -19,6 +20,7 @@ import (
|
|||
// Types of chunks.
|
||||
const (
|
||||
MapType uint64 = iota
|
||||
RLEType
|
||||
GridType
|
||||
)
|
||||
|
||||
|
@ -53,6 +55,7 @@ type JSONChunk struct {
|
|||
// Accessor provides a high-level API to interact with absolute pixel coordinates
|
||||
// while abstracting away the details of how they're stored.
|
||||
type Accessor interface {
|
||||
SetChunkCoordinate(render.Point, uint8)
|
||||
Inflate(*Palette) error
|
||||
Iter() <-chan Pixel
|
||||
IterViewport(viewport render.Rect) <-chan Pixel
|
||||
|
@ -62,15 +65,13 @@ type Accessor interface {
|
|||
Len() int
|
||||
MarshalBinary() ([]byte, error)
|
||||
UnmarshalBinary([]byte) error
|
||||
MarshalJSON() ([]byte, error)
|
||||
UnmarshalJSON([]byte) error
|
||||
}
|
||||
|
||||
// NewChunk creates a new chunk.
|
||||
func NewChunk() *Chunk {
|
||||
return &Chunk{
|
||||
Type: MapType,
|
||||
Accessor: NewMapAccessor(),
|
||||
Type: RLEType,
|
||||
Accessor: NewRLEAccessor(),
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -330,23 +331,6 @@ func (c *Chunk) Usage(size int) float64 {
|
|||
return float64(c.Len()) / float64(size)
|
||||
}
|
||||
|
||||
// MarshalJSON writes the chunk to JSON.
|
||||
//
|
||||
// DEPRECATED: MarshalBinary will encode chunks to a tighter binary format.
|
||||
func (c *Chunk) MarshalJSON() ([]byte, error) {
|
||||
data, err := c.Accessor.MarshalJSON()
|
||||
if err != nil {
|
||||
return []byte{}, err
|
||||
}
|
||||
|
||||
generic := &JSONChunk{
|
||||
Type: c.Type,
|
||||
Data: data,
|
||||
}
|
||||
b, err := json.Marshal(generic)
|
||||
return b, err
|
||||
}
|
||||
|
||||
// UnmarshalJSON loads the chunk from JSON and uses the correct accessor to
|
||||
// parse the inner details.
|
||||
//
|
||||
|
@ -363,7 +347,10 @@ func (c *Chunk) UnmarshalJSON(b []byte) error {
|
|||
switch c.Type {
|
||||
case MapType:
|
||||
c.Accessor = NewMapAccessor()
|
||||
return c.Accessor.UnmarshalJSON(generic.Data)
|
||||
if unmarshaler, ok := c.Accessor.(json.Unmarshaler); ok {
|
||||
return unmarshaler.UnmarshalJSON(generic.Data)
|
||||
}
|
||||
return errors.New("Chunk.UnmarshalJSON: this chunk doesn't support JSON unmarshaling")
|
||||
default:
|
||||
return fmt.Errorf("Chunk.UnmarshalJSON: unsupported chunk type '%d'", c.Type)
|
||||
}
|
||||
|
@ -407,6 +394,11 @@ func (c *Chunk) UnmarshalBinary(b []byte) error {
|
|||
switch chunkType {
|
||||
case MapType:
|
||||
c.Accessor = NewMapAccessor()
|
||||
c.Accessor.SetChunkCoordinate(c.Point, c.Size)
|
||||
return c.Accessor.UnmarshalBinary(reader.Bytes())
|
||||
case RLEType:
|
||||
c.Accessor = NewRLEAccessor()
|
||||
c.Accessor.SetChunkCoordinate(c.Point, c.Size)
|
||||
return c.Accessor.UnmarshalBinary(reader.Bytes())
|
||||
default:
|
||||
return fmt.Errorf("Chunk.UnmarshalJSON: unsupported chunk type '%d'", c.Type)
|
||||
|
|
|
@ -16,6 +16,8 @@ import (
|
|||
// MapAccessor implements a chunk accessor by using a map of points to their
|
||||
// palette indexes. This is the simplest accessor and is best for sparse chunks.
|
||||
type MapAccessor struct {
|
||||
coord render.Point `json:"-"` // chunk coordinate, assigned by Chunker
|
||||
size uint8 `json:"-"` // chunk size, assigned by Chunker
|
||||
grid map[render.Point]*Swatch
|
||||
mu sync.RWMutex
|
||||
}
|
||||
|
@ -27,6 +29,12 @@ func NewMapAccessor() *MapAccessor {
|
|||
}
|
||||
}
|
||||
|
||||
// SetChunkCoordinate receives our chunk's coordinate from the Chunker.
|
||||
func (a *MapAccessor) SetChunkCoordinate(p render.Point, size uint8) {
|
||||
a.coord = p
|
||||
a.size = size
|
||||
}
|
||||
|
||||
// Inflate the sparse swatches from their palette indexes.
|
||||
func (a *MapAccessor) Inflate(pal *Palette) error {
|
||||
for point, swatch := range a.grid {
|
||||
|
@ -271,7 +279,7 @@ func (a *MapAccessor) UnmarshalBinary(compressed []byte) error {
|
|||
defer a.mu.Unlock()
|
||||
|
||||
// New format: decompress the byte stream.
|
||||
//log.Debug("MapAccessor.Unmarshal: Reading %d bytes of compressed chunk data", len(compressed))
|
||||
log.Debug("MapAccessor.Unmarshal: Reading %d bytes of compressed chunk data", len(compressed))
|
||||
|
||||
var reader = bytes.NewBuffer(compressed)
|
||||
|
||||
|
|
201
pkg/level/chunk_rle.go
Normal file
201
pkg/level/chunk_rle.go
Normal file
|
@ -0,0 +1,201 @@
|
|||
package level
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
|
||||
"git.kirsle.net/SketchyMaze/doodle/pkg/log"
|
||||
"git.kirsle.net/go/render"
|
||||
)
|
||||
|
||||
// RLEAccessor implements a chunk accessor which stores its on-disk format using
|
||||
// Run Length Encoding (RLE), but in memory behaves equivalently to the MapAccessor.
|
||||
type RLEAccessor struct {
|
||||
acc *MapAccessor
|
||||
}
|
||||
|
||||
// NewRLEAccessor initializes a RLEAccessor.
|
||||
func NewRLEAccessor() *RLEAccessor {
|
||||
return &RLEAccessor{
|
||||
acc: NewMapAccessor(),
|
||||
}
|
||||
}
|
||||
|
||||
// SetChunkCoordinate receives our chunk's coordinate from the Chunker.
|
||||
func (a *RLEAccessor) SetChunkCoordinate(p render.Point, size uint8) {
|
||||
a.acc.coord = p
|
||||
a.acc.size = size
|
||||
}
|
||||
|
||||
// Inflate the sparse swatches from their palette indexes.
|
||||
func (a *RLEAccessor) Inflate(pal *Palette) error {
|
||||
return a.acc.Inflate(pal)
|
||||
}
|
||||
|
||||
// Len returns the current size of the map, or number of pixels registered.
|
||||
func (a *RLEAccessor) Len() int {
|
||||
return a.acc.Len()
|
||||
}
|
||||
|
||||
// IterViewport returns a channel to loop over pixels in the viewport.
|
||||
func (a *RLEAccessor) IterViewport(viewport render.Rect) <-chan Pixel {
|
||||
return a.acc.IterViewport(viewport)
|
||||
}
|
||||
|
||||
// Iter returns a channel to loop over all points in this chunk.
|
||||
func (a *RLEAccessor) Iter() <-chan Pixel {
|
||||
return a.acc.Iter()
|
||||
}
|
||||
|
||||
// Get a pixel from the map.
|
||||
func (a *RLEAccessor) Get(p render.Point) (*Swatch, error) {
|
||||
return a.acc.Get(p)
|
||||
}
|
||||
|
||||
// Set a pixel on the map.
|
||||
func (a *RLEAccessor) Set(p render.Point, sw *Swatch) error {
|
||||
return a.acc.Set(p, sw)
|
||||
}
|
||||
|
||||
// Delete a pixel from the map.
|
||||
func (a *RLEAccessor) Delete(p render.Point) error {
|
||||
return a.acc.Delete(p)
|
||||
}
|
||||
|
||||
// Make2DChunkGrid creates a 2D map of uint64 pointers matching the square dimensions of the given size.
|
||||
//
|
||||
// It is used by the RLEAccessor to flatten a chunk into a grid for run-length encoding.
|
||||
func Make2DChunkGrid(size int) ([][]*uint64, error) {
|
||||
// Sanity check if the chunk was properly initialized.
|
||||
if size == 0 {
|
||||
return nil, errors.New("chunk not initialized correctly with its size and coordinate")
|
||||
}
|
||||
|
||||
var grid = make([][]*uint64, size)
|
||||
for i := 0; i < size; i++ {
|
||||
grid[i] = make([]*uint64, size)
|
||||
}
|
||||
|
||||
return grid, nil
|
||||
}
|
||||
|
||||
/*
|
||||
MarshalBinary converts the chunk data to a binary representation.
|
||||
|
||||
This accessor uses Run Length Encoding (RLE) in its binary format. Starting
|
||||
with the top-left pixel of this chunk, the binary format is a stream of bytes
|
||||
formatted as such:
|
||||
|
||||
- UVarint for the palette index number (0-255), with 0xFF meaning void
|
||||
- UVarint for the length of repetition of that palette index
|
||||
*/
|
||||
func (a *RLEAccessor) MarshalBinary() ([]byte, error) {
|
||||
// Flatten the chunk out into a full 2D array of all its points.
|
||||
var (
|
||||
size = int(a.acc.size)
|
||||
grid, err = Make2DChunkGrid(size)
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Populate the dense 2D array of its pixels.
|
||||
for px := range a.Iter() {
|
||||
var (
|
||||
point = render.NewPoint(px.X, px.Y)
|
||||
relative = RelativeCoordinate(point, a.acc.coord, a.acc.size)
|
||||
ptr = uint64(px.PaletteIndex)
|
||||
)
|
||||
grid[relative.Y][relative.X] = &ptr
|
||||
}
|
||||
|
||||
// log.Error("2D GRID:\n%+v", grid)
|
||||
|
||||
// Run-length encode the grid.
|
||||
var (
|
||||
compressed []byte
|
||||
firstColor = true
|
||||
lastColor uint64
|
||||
runLength uint64
|
||||
)
|
||||
for _, row := range grid {
|
||||
for _, color := range row {
|
||||
var index uint64
|
||||
if color == nil {
|
||||
index = 0xFF
|
||||
}
|
||||
|
||||
if firstColor {
|
||||
lastColor = index
|
||||
runLength = 1
|
||||
firstColor = false
|
||||
continue
|
||||
}
|
||||
|
||||
if index != lastColor {
|
||||
compressed = binary.AppendUvarint(compressed, index)
|
||||
compressed = binary.AppendUvarint(compressed, runLength)
|
||||
lastColor = index
|
||||
runLength = 1
|
||||
continue
|
||||
}
|
||||
|
||||
runLength++
|
||||
}
|
||||
}
|
||||
|
||||
log.Error("RLE compressed: %v", compressed)
|
||||
|
||||
return compressed, nil
|
||||
}
|
||||
|
||||
// UnmarshalBinary will decode a compressed RLEAccessor byte stream.
|
||||
func (a *RLEAccessor) UnmarshalBinary(compressed []byte) error {
|
||||
a.acc.mu.Lock()
|
||||
defer a.acc.mu.Unlock()
|
||||
|
||||
// New format: decompress the byte stream.
|
||||
log.Debug("RLEAccessor.Unmarshal: Reading %d bytes of compressed chunk data", len(compressed))
|
||||
|
||||
// Prepare the 2D grid to decompress the RLE stream into.
|
||||
var (
|
||||
size = int(a.acc.size)
|
||||
_, err = Make2DChunkGrid(size)
|
||||
x, y, cursor int
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var reader = bytes.NewBuffer(compressed)
|
||||
|
||||
for {
|
||||
var (
|
||||
paletteIndex, err1 = binary.ReadUvarint(reader)
|
||||
repeatCount, err2 = binary.ReadUvarint(reader)
|
||||
)
|
||||
|
||||
if err1 != nil || err2 != nil {
|
||||
log.Error("reading Uvarints from compressed data: {%s, %s}", err1, err2)
|
||||
break
|
||||
}
|
||||
|
||||
for i := uint64(0); i < repeatCount; i++ {
|
||||
cursor++
|
||||
if cursor%size == 0 {
|
||||
y++
|
||||
x = 0
|
||||
} else {
|
||||
x++
|
||||
}
|
||||
|
||||
point := render.NewPoint(int(x), int(y))
|
||||
if paletteIndex != 0xFF {
|
||||
a.acc.grid[point] = NewSparseSwatch(int(paletteIndex))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
|
@ -242,54 +242,88 @@ func TestChunkCoordinates(t *testing.T) {
|
|||
c := level.NewChunker(128)
|
||||
|
||||
type testCase struct {
|
||||
In render.Point
|
||||
Expect render.Point
|
||||
WorldCoordinate render.Point
|
||||
ChunkCoordinate render.Point
|
||||
RelativeCoordinate render.Point
|
||||
}
|
||||
tests := []testCase{
|
||||
testCase{
|
||||
In: render.NewPoint(0, 0),
|
||||
Expect: render.NewPoint(0, 0),
|
||||
WorldCoordinate: render.NewPoint(0, 0),
|
||||
ChunkCoordinate: render.NewPoint(0, 0),
|
||||
RelativeCoordinate: render.NewPoint(0, 0),
|
||||
},
|
||||
testCase{
|
||||
In: render.NewPoint(128, 128),
|
||||
Expect: render.NewPoint(0, 0),
|
||||
WorldCoordinate: render.NewPoint(4, 8),
|
||||
ChunkCoordinate: render.NewPoint(0, 0),
|
||||
RelativeCoordinate: render.NewPoint(4, 8),
|
||||
},
|
||||
testCase{
|
||||
In: render.NewPoint(1024, 128),
|
||||
Expect: render.NewPoint(1, 0),
|
||||
WorldCoordinate: render.NewPoint(128, 128),
|
||||
ChunkCoordinate: render.NewPoint(1, 1),
|
||||
RelativeCoordinate: render.NewPoint(0, 0),
|
||||
},
|
||||
testCase{
|
||||
In: render.NewPoint(3600, 1228),
|
||||
Expect: render.NewPoint(3, 1),
|
||||
WorldCoordinate: render.NewPoint(130, 156),
|
||||
ChunkCoordinate: render.NewPoint(1, 1),
|
||||
RelativeCoordinate: render.NewPoint(2, 28),
|
||||
},
|
||||
testCase{
|
||||
In: render.NewPoint(-100, -1),
|
||||
Expect: render.NewPoint(-1, -1),
|
||||
WorldCoordinate: render.NewPoint(1024, 128),
|
||||
ChunkCoordinate: render.NewPoint(8, 1),
|
||||
RelativeCoordinate: render.NewPoint(0, 0),
|
||||
},
|
||||
testCase{
|
||||
In: render.NewPoint(-950, 100),
|
||||
Expect: render.NewPoint(-1, 0),
|
||||
WorldCoordinate: render.NewPoint(3600, 1228),
|
||||
ChunkCoordinate: render.NewPoint(28, 9),
|
||||
RelativeCoordinate: render.NewPoint(16, 76),
|
||||
},
|
||||
testCase{
|
||||
In: render.NewPoint(-1001, -856),
|
||||
Expect: render.NewPoint(-2, -1),
|
||||
WorldCoordinate: render.NewPoint(-100, -1),
|
||||
ChunkCoordinate: render.NewPoint(-1, -1),
|
||||
RelativeCoordinate: render.NewPoint(28, 127),
|
||||
},
|
||||
testCase{
|
||||
In: render.NewPoint(-3600, -4800),
|
||||
Expect: render.NewPoint(-4, -5),
|
||||
WorldCoordinate: render.NewPoint(-950, 100),
|
||||
ChunkCoordinate: render.NewPoint(-8, 0),
|
||||
RelativeCoordinate: render.NewPoint(74, 100),
|
||||
},
|
||||
testCase{
|
||||
WorldCoordinate: render.NewPoint(-1001, -856),
|
||||
ChunkCoordinate: render.NewPoint(-8, -7),
|
||||
RelativeCoordinate: render.NewPoint(23, 40),
|
||||
},
|
||||
testCase{
|
||||
WorldCoordinate: render.NewPoint(-3600, -4800),
|
||||
ChunkCoordinate: render.NewPoint(-29, -38),
|
||||
RelativeCoordinate: render.NewPoint(112, 64),
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
actual := c.ChunkCoordinate(test.In)
|
||||
if actual != test.Expect {
|
||||
// Test conversion from world to chunk coordinate.
|
||||
actual := c.ChunkCoordinate(test.WorldCoordinate)
|
||||
if actual != test.ChunkCoordinate {
|
||||
t.Errorf(
|
||||
"Failed ChunkCoordinate conversion:\n"+
|
||||
" Input: %s\n"+
|
||||
"Expected: %s\n"+
|
||||
" Got: %s",
|
||||
test.In,
|
||||
test.Expect,
|
||||
test.WorldCoordinate,
|
||||
test.ChunkCoordinate,
|
||||
actual,
|
||||
)
|
||||
}
|
||||
|
||||
// Test the relative (inside-chunk) coordinate.
|
||||
actual = level.RelativeCoordinate(test.WorldCoordinate, actual, c.Size)
|
||||
if actual != test.RelativeCoordinate {
|
||||
t.Errorf(
|
||||
"Failed RelativeCoordinate conversion:\n"+
|
||||
" Input: %s\n"+
|
||||
"Expected: %s\n"+
|
||||
" Got: %s",
|
||||
test.WorldCoordinate,
|
||||
test.RelativeCoordinate,
|
||||
actual,
|
||||
)
|
||||
}
|
||||
|
|
|
@ -74,6 +74,7 @@ func (c *Chunker) Inflate(pal *Palette) error {
|
|||
for coord, chunk := range c.Chunks {
|
||||
chunk.Point = coord
|
||||
chunk.Size = c.Size
|
||||
chunk.SetChunkCoordinate(chunk.Point, chunk.Size)
|
||||
chunk.Inflate(pal)
|
||||
}
|
||||
return nil
|
||||
|
@ -445,6 +446,7 @@ func (c *Chunker) FreeCaches() int {
|
|||
// This function should be the singular writer to the chunk cache.
|
||||
func (c *Chunker) SetChunk(p render.Point, chunk *Chunk) {
|
||||
c.chunkMu.Lock()
|
||||
chunk.SetChunkCoordinate(p, chunk.Size)
|
||||
c.Chunks[p] = chunk
|
||||
c.chunkMu.Unlock()
|
||||
|
||||
|
@ -605,6 +607,30 @@ func (c *Chunker) ChunkCoordinate(abs render.Point) render.Point {
|
|||
)
|
||||
}
|
||||
|
||||
// RelativeCoordinate will translate from an absolute world coordinate, into one that
|
||||
// is relative to fit inside of the chunk with the given chunk coordinate and size.
|
||||
//
|
||||
// Example:
|
||||
//
|
||||
// - With 128x128 chunks and a world coordinate of (280,-600)
|
||||
// - The ChunkCoordinate would be (2,-4) which encompasses (256,-512) to (383,-639)
|
||||
// - And relative inside that chunk, the pixel is at (24,)
|
||||
func RelativeCoordinate(abs render.Point, chunkCoord render.Point, chunkSize uint8) render.Point {
|
||||
// Pixel coordinate offset.
|
||||
var (
|
||||
size = int(chunkSize)
|
||||
offset = render.Point{
|
||||
X: chunkCoord.X * size,
|
||||
Y: chunkCoord.Y * size,
|
||||
}
|
||||
)
|
||||
|
||||
return render.Point{
|
||||
X: abs.X - offset.X,
|
||||
Y: abs.Y - offset.Y,
|
||||
}
|
||||
}
|
||||
|
||||
// ChunkMap maps a chunk coordinate to its chunk data.
|
||||
type ChunkMap map[render.Point]*Chunk
|
||||
|
||||
|
|
|
@ -2,6 +2,7 @@ package level
|
|||
|
||||
import (
|
||||
"archive/zip"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"regexp"
|
||||
|
@ -190,11 +191,7 @@ func (c *Chunk) ToZipfile(zf *zip.Writer, layer int, coord render.Point) error {
|
|||
data = bytes
|
||||
}
|
||||
} else {
|
||||
if json, err := c.MarshalJSON(); err != nil {
|
||||
return err
|
||||
} else {
|
||||
data = json
|
||||
}
|
||||
return errors.New("Chunk.ToZipfile: JSON chunk format no longer supported for writing")
|
||||
}
|
||||
|
||||
// Write the file contents to zip whether binary or json.
|
||||
|
@ -226,6 +223,7 @@ func ChunkFromZipfile(zf *zip.Reader, layer int, coord render.Point) (*Chunk, er
|
|||
|
||||
err = chunk.UnmarshalBinary(bin)
|
||||
if err != nil {
|
||||
log.Error("ChunkFromZipfile(%s): %s", coord, err)
|
||||
return nil, err
|
||||
}
|
||||
} else if file, err := zf.Open(jsonfile); err == nil {
|
||||
|
|
Loading…
Reference in New Issue
Block a user