WIP Run Length Encoding for Levels

pkg/level/chunk.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"encoding/binary"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"image"
 	"math"
@@ -19,6 +20,7 @@ import (
 // Types of chunks.
 const (
 	MapType uint64 = iota
+	RLEType
 	GridType
 )
 
@@ -53,6 +55,7 @@ type JSONChunk struct {
 // Accessor provides a high-level API to interact with absolute pixel coordinates
 // while abstracting away the details of how they're stored.
 type Accessor interface {
+	SetChunkCoordinate(render.Point, uint8)
 	Inflate(*Palette) error
 	Iter() <-chan Pixel
 	IterViewport(viewport render.Rect) <-chan Pixel
@@ -62,15 +65,13 @@ type Accessor interface {
 	Len() int
 	MarshalBinary() ([]byte, error)
 	UnmarshalBinary([]byte) error
-	MarshalJSON() ([]byte, error)
-	UnmarshalJSON([]byte) error
 }
 
 // NewChunk creates a new chunk.
 func NewChunk() *Chunk {
 	return &Chunk{
-		Type:     MapType,
-		Accessor: NewMapAccessor(),
+		Type:     RLEType,
+		Accessor: NewRLEAccessor(),
 	}
 }
 
@@ -330,23 +331,6 @@ func (c *Chunk) Usage(size int) float64 {
 	return float64(c.Len()) / float64(size)
 }
 
-// MarshalJSON writes the chunk to JSON.
-//
-// DEPRECATED: MarshalBinary will encode chunks to a tighter binary format.
-func (c *Chunk) MarshalJSON() ([]byte, error) {
-	data, err := c.Accessor.MarshalJSON()
-	if err != nil {
-		return []byte{}, err
-	}
-
-	generic := &JSONChunk{
-		Type: c.Type,
-		Data: data,
-	}
-	b, err := json.Marshal(generic)
-	return b, err
-}
-
 // UnmarshalJSON loads the chunk from JSON and uses the correct accessor to
 // parse the inner details.
 //
@@ -363,7 +347,10 @@ func (c *Chunk) UnmarshalJSON(b []byte) error {
 	switch c.Type {
 	case MapType:
 		c.Accessor = NewMapAccessor()
-		return c.Accessor.UnmarshalJSON(generic.Data)
+		if unmarshaler, ok := c.Accessor.(json.Unmarshaler); ok {
+			return unmarshaler.UnmarshalJSON(generic.Data)
+		}
+		return errors.New("Chunk.UnmarshalJSON: this chunk doesn't support JSON unmarshaling")
 	default:
 		return fmt.Errorf("Chunk.UnmarshalJSON: unsupported chunk type '%d'", c.Type)
 	}
@@ -407,6 +394,11 @@ func (c *Chunk) UnmarshalBinary(b []byte) error {
 	switch chunkType {
 	case MapType:
 		c.Accessor = NewMapAccessor()
+		c.Accessor.SetChunkCoordinate(c.Point, c.Size)
+		return c.Accessor.UnmarshalBinary(reader.Bytes())
+	case RLEType:
+		c.Accessor = NewRLEAccessor()
+		c.Accessor.SetChunkCoordinate(c.Point, c.Size)
 		return c.Accessor.UnmarshalBinary(reader.Bytes())
 	default:
 		return fmt.Errorf("Chunk.UnmarshalJSON: unsupported chunk type '%d'", c.Type)

pkg/level/chunk_map.go

@@ -16,8 +16,10 @@ import (
 // MapAccessor implements a chunk accessor by using a map of points to their
 // palette indexes. This is the simplest accessor and is best for sparse chunks.
 type MapAccessor struct {
-	grid map[render.Point]*Swatch
-	mu   sync.RWMutex
+	coord render.Point `json:"-"` // chunk coordinate, assigned by Chunker
+	size  uint8        `json:"-"` // chunk size, assigned by Chunker
+	grid  map[render.Point]*Swatch
+	mu    sync.RWMutex
 }
 
 // NewMapAccessor initializes a MapAccessor.
@@ -27,6 +29,12 @@ func NewMapAccessor() *MapAccessor {
 	}
 }
 
+// SetChunkCoordinate receives our chunk's coordinate from the Chunker.
+func (a *MapAccessor) SetChunkCoordinate(p render.Point, size uint8) {
+	a.coord = p
+	a.size = size
+}
+
 // Inflate the sparse swatches from their palette indexes.
 func (a *MapAccessor) Inflate(pal *Palette) error {
 	for point, swatch := range a.grid {
@@ -271,7 +279,7 @@ func (a *MapAccessor) UnmarshalBinary(compressed []byte) error {
 	defer a.mu.Unlock()
 
 	// New format: decompress the byte stream.
-	//log.Debug("MapAccessor.Unmarshal: Reading %d bytes of compressed chunk data", len(compressed))
+	log.Debug("MapAccessor.Unmarshal: Reading %d bytes of compressed chunk data", len(compressed))
 
 	var reader = bytes.NewBuffer(compressed)
 

pkg/level/chunk_rle.go

@@ -0,0 +1,201 @@
+package level
+
+import (
+	"bytes"
+	"encoding/binary"
+	"errors"
+
+	"git.kirsle.net/SketchyMaze/doodle/pkg/log"
+	"git.kirsle.net/go/render"
+)
+
+// RLEAccessor implements a chunk accessor which stores its on-disk format using
+// Run Length Encoding (RLE), but in memory behaves equivalently to the MapAccessor.
+type RLEAccessor struct {
+	acc *MapAccessor
+}
+
+// NewRLEAccessor initializes a RLEAccessor.
+func NewRLEAccessor() *RLEAccessor {
+	return &RLEAccessor{
+		acc: NewMapAccessor(),
+	}
+}
+
+// SetChunkCoordinate receives our chunk's coordinate from the Chunker.
+func (a *RLEAccessor) SetChunkCoordinate(p render.Point, size uint8) {
+	a.acc.coord = p
+	a.acc.size = size
+}
+
+// Inflate the sparse swatches from their palette indexes.
+func (a *RLEAccessor) Inflate(pal *Palette) error {
+	return a.acc.Inflate(pal)
+}
+
+// Len returns the current size of the map, or number of pixels registered.
+func (a *RLEAccessor) Len() int {
+	return a.acc.Len()
+}
+
+// IterViewport returns a channel to loop over pixels in the viewport.
+func (a *RLEAccessor) IterViewport(viewport render.Rect) <-chan Pixel {
+	return a.acc.IterViewport(viewport)
+}
+
+// Iter returns a channel to loop over all points in this chunk.
+func (a *RLEAccessor) Iter() <-chan Pixel {
+	return a.acc.Iter()
+}
+
+// Get a pixel from the map.
+func (a *RLEAccessor) Get(p render.Point) (*Swatch, error) {
+	return a.acc.Get(p)
+}
+
+// Set a pixel on the map.
+func (a *RLEAccessor) Set(p render.Point, sw *Swatch) error {
+	return a.acc.Set(p, sw)
+}
+
+// Delete a pixel from the map.
+func (a *RLEAccessor) Delete(p render.Point) error {
+	return a.acc.Delete(p)
+}
+
+// Make2DChunkGrid creates a 2D map of uint64 pointers matching the square dimensions of the given size.
+//
+// It is used by the RLEAccessor to flatten a chunk into a grid for run-length encoding.
+func Make2DChunkGrid(size int) ([][]*uint64, error) {
+	// Sanity check if the chunk was properly initialized.
+	if size == 0 {
+		return nil, errors.New("chunk not initialized correctly with its size and coordinate")
+	}
+
+	var grid = make([][]*uint64, size)
+	for i := 0; i < size; i++ {
+		grid[i] = make([]*uint64, size)
+	}
+
+	return grid, nil
+}
+
+/*
+MarshalBinary converts the chunk data to a binary representation.
+
+This accessor uses Run Length Encoding (RLE) in its binary format. Starting
+with the top-left pixel of this chunk, the binary format is a stream of bytes
+formatted as such:
+
+- UVarint for the palette index number (0-255), with 0xFF meaning void
+- UVarint for the length of repetition of that palette index
+*/
+func (a *RLEAccessor) MarshalBinary() ([]byte, error) {
+	// Flatten the chunk out into a full 2D array of all its points.
+	var (
+		size      = int(a.acc.size)
+		grid, err = Make2DChunkGrid(size)
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	// Populate the dense 2D array of its pixels.
+	for px := range a.Iter() {
+		var (
+			point    = render.NewPoint(px.X, px.Y)
+			relative = RelativeCoordinate(point, a.acc.coord, a.acc.size)
+			ptr      = uint64(px.PaletteIndex)
+		)
+		grid[relative.Y][relative.X] = &ptr
+	}
+
+	// log.Error("2D GRID:\n%+v", grid)
+
+	// Run-length encode the grid.
+	var (
+		compressed []byte
+		firstColor = true
+		lastColor  uint64
+		runLength  uint64
+	)
+	for _, row := range grid {
+		for _, color := range row {
+			var index uint64
+			if color == nil {
+				index = 0xFF
+			}
+
+			if firstColor {
+				lastColor = index
+				runLength = 1
+				firstColor = false
+				continue
+			}
+
+			if index != lastColor {
+				compressed = binary.AppendUvarint(compressed, index)
+				compressed = binary.AppendUvarint(compressed, runLength)
+				lastColor = index
+				runLength = 1
+				continue
+			}
+
+			runLength++
+		}
+	}
+
+	log.Error("RLE compressed: %v", compressed)
+
+	return compressed, nil
+}
+
+// UnmarshalBinary will decode a compressed RLEAccessor byte stream.
+func (a *RLEAccessor) UnmarshalBinary(compressed []byte) error {
+	a.acc.mu.Lock()
+	defer a.acc.mu.Unlock()
+
+	// New format: decompress the byte stream.
+	log.Debug("RLEAccessor.Unmarshal: Reading %d bytes of compressed chunk data", len(compressed))
+
+	// Prepare the 2D grid to decompress the RLE stream into.
+	var (
+		size         = int(a.acc.size)
+		_, err       = Make2DChunkGrid(size)
+		x, y, cursor int
+	)
+	if err != nil {
+		return err
+	}
+
+	var reader = bytes.NewBuffer(compressed)
+
+	for {
+		var (
+			paletteIndex, err1 = binary.ReadUvarint(reader)
+			repeatCount, err2  = binary.ReadUvarint(reader)
+		)
+
+		if err1 != nil || err2 != nil {
+			log.Error("reading Uvarints from compressed data: {%s, %s}", err1, err2)
+			break
+		}
+
+		for i := uint64(0); i < repeatCount; i++ {
+			cursor++
+			if cursor%size == 0 {
+				y++
+				x = 0
+			} else {
+				x++
+			}
+
+			point := render.NewPoint(int(x), int(y))
+			if paletteIndex != 0xFF {
+				a.acc.grid[point] = NewSparseSwatch(int(paletteIndex))
+			}
+		}
+	}
+
+	return nil
+}

pkg/level/chunk_test.go

@@ -242,54 +242,88 @@ func TestChunkCoordinates(t *testing.T) {
 	c := level.NewChunker(128)
 
 	type testCase struct {
-		In     render.Point
-		Expect render.Point
+		WorldCoordinate    render.Point
+		ChunkCoordinate    render.Point
+		RelativeCoordinate render.Point
 	}
 	tests := []testCase{
 		testCase{
-			In:     render.NewPoint(0, 0),
-			Expect: render.NewPoint(0, 0),
+			WorldCoordinate:    render.NewPoint(0, 0),
+			ChunkCoordinate:    render.NewPoint(0, 0),
+			RelativeCoordinate: render.NewPoint(0, 0),
 		},
 		testCase{
-			In:     render.NewPoint(128, 128),
-			Expect: render.NewPoint(0, 0),
+			WorldCoordinate:    render.NewPoint(4, 8),
+			ChunkCoordinate:    render.NewPoint(0, 0),
+			RelativeCoordinate: render.NewPoint(4, 8),
 		},
 		testCase{
-			In:     render.NewPoint(1024, 128),
-			Expect: render.NewPoint(1, 0),
+			WorldCoordinate:    render.NewPoint(128, 128),
+			ChunkCoordinate:    render.NewPoint(1, 1),
+			RelativeCoordinate: render.NewPoint(0, 0),
 		},
 		testCase{
-			In:     render.NewPoint(3600, 1228),
-			Expect: render.NewPoint(3, 1),
+			WorldCoordinate:    render.NewPoint(130, 156),
+			ChunkCoordinate:    render.NewPoint(1, 1),
+			RelativeCoordinate: render.NewPoint(2, 28),
 		},
 		testCase{
-			In:     render.NewPoint(-100, -1),
-			Expect: render.NewPoint(-1, -1),
+			WorldCoordinate:    render.NewPoint(1024, 128),
+			ChunkCoordinate:    render.NewPoint(8, 1),
+			RelativeCoordinate: render.NewPoint(0, 0),
 		},
 		testCase{
-			In:     render.NewPoint(-950, 100),
-			Expect: render.NewPoint(-1, 0),
+			WorldCoordinate:    render.NewPoint(3600, 1228),
+			ChunkCoordinate:    render.NewPoint(28, 9),
+			RelativeCoordinate: render.NewPoint(16, 76),
 		},
 		testCase{
-			In:     render.NewPoint(-1001, -856),
-			Expect: render.NewPoint(-2, -1),
+			WorldCoordinate:    render.NewPoint(-100, -1),
+			ChunkCoordinate:    render.NewPoint(-1, -1),
+			RelativeCoordinate: render.NewPoint(28, 127),
 		},
 		testCase{
-			In:     render.NewPoint(-3600, -4800),
-			Expect: render.NewPoint(-4, -5),
+			WorldCoordinate:    render.NewPoint(-950, 100),
+			ChunkCoordinate:    render.NewPoint(-8, 0),
+			RelativeCoordinate: render.NewPoint(74, 100),
+		},
+		testCase{
+			WorldCoordinate:    render.NewPoint(-1001, -856),
+			ChunkCoordinate:    render.NewPoint(-8, -7),
+			RelativeCoordinate: render.NewPoint(23, 40),
+		},
+		testCase{
+			WorldCoordinate:    render.NewPoint(-3600, -4800),
+			ChunkCoordinate:    render.NewPoint(-29, -38),
+			RelativeCoordinate: render.NewPoint(112, 64),
 		},
 	}
 
 	for _, test := range tests {
-		actual := c.ChunkCoordinate(test.In)
-		if actual != test.Expect {
+		// Test conversion from world to chunk coordinate.
+		actual := c.ChunkCoordinate(test.WorldCoordinate)
+		if actual != test.ChunkCoordinate {
 			t.Errorf(
 				"Failed ChunkCoordinate conversion:\n"+
 					"   Input: %s\n"+
 					"Expected: %s\n"+
 					"     Got: %s",
-				test.In,
-				test.Expect,
+				test.WorldCoordinate,
+				test.ChunkCoordinate,
+				actual,
+			)
+		}
+
+		// Test the relative (inside-chunk) coordinate.
+		actual = level.RelativeCoordinate(test.WorldCoordinate, actual, c.Size)
+		if actual != test.RelativeCoordinate {
+			t.Errorf(
+				"Failed RelativeCoordinate conversion:\n"+
+					"   Input: %s\n"+
+					"Expected: %s\n"+
+					"     Got: %s",
+				test.WorldCoordinate,
+				test.RelativeCoordinate,
 				actual,
 			)
 		}

pkg/level/chunker.go

@@ -74,6 +74,7 @@ func (c *Chunker) Inflate(pal *Palette) error {
 	for coord, chunk := range c.Chunks {
 		chunk.Point = coord
 		chunk.Size = c.Size
+		chunk.SetChunkCoordinate(chunk.Point, chunk.Size)
 		chunk.Inflate(pal)
 	}
 	return nil
@@ -445,6 +446,7 @@ func (c *Chunker) FreeCaches() int {
 // This function should be the singular writer to the chunk cache.
 func (c *Chunker) SetChunk(p render.Point, chunk *Chunk) {
 	c.chunkMu.Lock()
+	chunk.SetChunkCoordinate(p, chunk.Size)
 	c.Chunks[p] = chunk
 	c.chunkMu.Unlock()
 
@@ -605,6 +607,30 @@ func (c *Chunker) ChunkCoordinate(abs render.Point) render.Point {
 	)
 }
 
+// RelativeCoordinate will translate from an absolute world coordinate, into one that
+// is relative to fit inside of the chunk with the given chunk coordinate and size.
+//
+// Example:
+//
+// - With 128x128 chunks and a world coordinate of (280,-600)
+// - The ChunkCoordinate would be (2,-4) which encompasses (256,-512) to (383,-639)
+// - And relative inside that chunk, the pixel is at (24,)
+func RelativeCoordinate(abs render.Point, chunkCoord render.Point, chunkSize uint8) render.Point {
+	// Pixel coordinate offset.
+	var (
+		size   = int(chunkSize)
+		offset = render.Point{
+			X: chunkCoord.X * size,
+			Y: chunkCoord.Y * size,
+		}
+	)
+
+	return render.Point{
+		X: abs.X - offset.X,
+		Y: abs.Y - offset.Y,
+	}
+}
+
 // ChunkMap maps a chunk coordinate to its chunk data.
 type ChunkMap map[render.Point]*Chunk
 

pkg/level/chunker_zipfile.go

@@ -2,6 +2,7 @@ package level
 
 import (
 	"archive/zip"
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"regexp"
@@ -190,11 +191,7 @@ func (c *Chunk) ToZipfile(zf *zip.Writer, layer int, coord render.Point) error {
 			data = bytes
 		}
 	} else {
-		if json, err := c.MarshalJSON(); err != nil {
-			return err
-		} else {
-			data = json
-		}
+		return errors.New("Chunk.ToZipfile: JSON chunk format no longer supported for writing")
 	}
 
 	// Write the file contents to zip whether binary or json.
@@ -226,6 +223,7 @@ func ChunkFromZipfile(zf *zip.Reader, layer int, coord render.Point) (*Chunk, er
 
 		err = chunk.UnmarshalBinary(bin)
 		if err != nil {
+			log.Error("ChunkFromZipfile(%s): %s", coord, err)
 			return nil, err
 		}
 	} else if file, err := zf.Open(jsonfile); err == nil {