From b1d7c7a3848adbfe06737907958a523f68fd662e Mon Sep 17 00:00:00 2001 From: Noah Petherbridge Date: Thu, 23 May 2024 19:15:10 -0700 Subject: [PATCH] WIP Run Length Encoding for Levels --- pkg/level/chunk.go | 36 +++---- pkg/level/chunk_map.go | 14 ++- pkg/level/chunk_rle.go | 201 +++++++++++++++++++++++++++++++++++ pkg/level/chunk_test.go | 78 ++++++++++---- pkg/level/chunker.go | 26 +++++ pkg/level/chunker_zipfile.go | 8 +- 6 files changed, 311 insertions(+), 52 deletions(-) create mode 100644 pkg/level/chunk_rle.go diff --git a/pkg/level/chunk.go b/pkg/level/chunk.go index 85e2cb7..fbee1c8 100644 --- a/pkg/level/chunk.go +++ b/pkg/level/chunk.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/binary" "encoding/json" + "errors" "fmt" "image" "math" @@ -19,6 +20,7 @@ import ( // Types of chunks. const ( MapType uint64 = iota + RLEType GridType ) @@ -53,6 +55,7 @@ type JSONChunk struct { // Accessor provides a high-level API to interact with absolute pixel coordinates // while abstracting away the details of how they're stored. type Accessor interface { + SetChunkCoordinate(render.Point, uint8) Inflate(*Palette) error Iter() <-chan Pixel IterViewport(viewport render.Rect) <-chan Pixel @@ -62,15 +65,13 @@ type Accessor interface { Len() int MarshalBinary() ([]byte, error) UnmarshalBinary([]byte) error - MarshalJSON() ([]byte, error) - UnmarshalJSON([]byte) error } // NewChunk creates a new chunk. func NewChunk() *Chunk { return &Chunk{ - Type: MapType, - Accessor: NewMapAccessor(), + Type: RLEType, + Accessor: NewRLEAccessor(), } } @@ -330,23 +331,6 @@ func (c *Chunk) Usage(size int) float64 { return float64(c.Len()) / float64(size) } -// MarshalJSON writes the chunk to JSON. -// -// DEPRECATED: MarshalBinary will encode chunks to a tighter binary format. -func (c *Chunk) MarshalJSON() ([]byte, error) { - data, err := c.Accessor.MarshalJSON() - if err != nil { - return []byte{}, err - } - - generic := &JSONChunk{ - Type: c.Type, - Data: data, - } - b, err := json.Marshal(generic) - return b, err -} - // UnmarshalJSON loads the chunk from JSON and uses the correct accessor to // parse the inner details. // @@ -363,7 +347,10 @@ func (c *Chunk) UnmarshalJSON(b []byte) error { switch c.Type { case MapType: c.Accessor = NewMapAccessor() - return c.Accessor.UnmarshalJSON(generic.Data) + if unmarshaler, ok := c.Accessor.(json.Unmarshaler); ok { + return unmarshaler.UnmarshalJSON(generic.Data) + } + return errors.New("Chunk.UnmarshalJSON: this chunk doesn't support JSON unmarshaling") default: return fmt.Errorf("Chunk.UnmarshalJSON: unsupported chunk type '%d'", c.Type) } @@ -407,6 +394,11 @@ func (c *Chunk) UnmarshalBinary(b []byte) error { switch chunkType { case MapType: c.Accessor = NewMapAccessor() + c.Accessor.SetChunkCoordinate(c.Point, c.Size) + return c.Accessor.UnmarshalBinary(reader.Bytes()) + case RLEType: + c.Accessor = NewRLEAccessor() + c.Accessor.SetChunkCoordinate(c.Point, c.Size) return c.Accessor.UnmarshalBinary(reader.Bytes()) default: return fmt.Errorf("Chunk.UnmarshalJSON: unsupported chunk type '%d'", c.Type) diff --git a/pkg/level/chunk_map.go b/pkg/level/chunk_map.go index 652b50c..ebbd839 100644 --- a/pkg/level/chunk_map.go +++ b/pkg/level/chunk_map.go @@ -16,8 +16,10 @@ import ( // MapAccessor implements a chunk accessor by using a map of points to their // palette indexes. This is the simplest accessor and is best for sparse chunks. type MapAccessor struct { - grid map[render.Point]*Swatch - mu sync.RWMutex + coord render.Point `json:"-"` // chunk coordinate, assigned by Chunker + size uint8 `json:"-"` // chunk size, assigned by Chunker + grid map[render.Point]*Swatch + mu sync.RWMutex } // NewMapAccessor initializes a MapAccessor. @@ -27,6 +29,12 @@ func NewMapAccessor() *MapAccessor { } } +// SetChunkCoordinate receives our chunk's coordinate from the Chunker. +func (a *MapAccessor) SetChunkCoordinate(p render.Point, size uint8) { + a.coord = p + a.size = size +} + // Inflate the sparse swatches from their palette indexes. func (a *MapAccessor) Inflate(pal *Palette) error { for point, swatch := range a.grid { @@ -271,7 +279,7 @@ func (a *MapAccessor) UnmarshalBinary(compressed []byte) error { defer a.mu.Unlock() // New format: decompress the byte stream. - //log.Debug("MapAccessor.Unmarshal: Reading %d bytes of compressed chunk data", len(compressed)) + log.Debug("MapAccessor.Unmarshal: Reading %d bytes of compressed chunk data", len(compressed)) var reader = bytes.NewBuffer(compressed) diff --git a/pkg/level/chunk_rle.go b/pkg/level/chunk_rle.go new file mode 100644 index 0000000..bf0ee68 --- /dev/null +++ b/pkg/level/chunk_rle.go @@ -0,0 +1,201 @@ +package level + +import ( + "bytes" + "encoding/binary" + "errors" + + "git.kirsle.net/SketchyMaze/doodle/pkg/log" + "git.kirsle.net/go/render" +) + +// RLEAccessor implements a chunk accessor which stores its on-disk format using +// Run Length Encoding (RLE), but in memory behaves equivalently to the MapAccessor. +type RLEAccessor struct { + acc *MapAccessor +} + +// NewRLEAccessor initializes a RLEAccessor. +func NewRLEAccessor() *RLEAccessor { + return &RLEAccessor{ + acc: NewMapAccessor(), + } +} + +// SetChunkCoordinate receives our chunk's coordinate from the Chunker. +func (a *RLEAccessor) SetChunkCoordinate(p render.Point, size uint8) { + a.acc.coord = p + a.acc.size = size +} + +// Inflate the sparse swatches from their palette indexes. +func (a *RLEAccessor) Inflate(pal *Palette) error { + return a.acc.Inflate(pal) +} + +// Len returns the current size of the map, or number of pixels registered. +func (a *RLEAccessor) Len() int { + return a.acc.Len() +} + +// IterViewport returns a channel to loop over pixels in the viewport. +func (a *RLEAccessor) IterViewport(viewport render.Rect) <-chan Pixel { + return a.acc.IterViewport(viewport) +} + +// Iter returns a channel to loop over all points in this chunk. +func (a *RLEAccessor) Iter() <-chan Pixel { + return a.acc.Iter() +} + +// Get a pixel from the map. +func (a *RLEAccessor) Get(p render.Point) (*Swatch, error) { + return a.acc.Get(p) +} + +// Set a pixel on the map. +func (a *RLEAccessor) Set(p render.Point, sw *Swatch) error { + return a.acc.Set(p, sw) +} + +// Delete a pixel from the map. +func (a *RLEAccessor) Delete(p render.Point) error { + return a.acc.Delete(p) +} + +// Make2DChunkGrid creates a 2D map of uint64 pointers matching the square dimensions of the given size. +// +// It is used by the RLEAccessor to flatten a chunk into a grid for run-length encoding. +func Make2DChunkGrid(size int) ([][]*uint64, error) { + // Sanity check if the chunk was properly initialized. + if size == 0 { + return nil, errors.New("chunk not initialized correctly with its size and coordinate") + } + + var grid = make([][]*uint64, size) + for i := 0; i < size; i++ { + grid[i] = make([]*uint64, size) + } + + return grid, nil +} + +/* +MarshalBinary converts the chunk data to a binary representation. + +This accessor uses Run Length Encoding (RLE) in its binary format. Starting +with the top-left pixel of this chunk, the binary format is a stream of bytes +formatted as such: + +- UVarint for the palette index number (0-255), with 0xFF meaning void +- UVarint for the length of repetition of that palette index +*/ +func (a *RLEAccessor) MarshalBinary() ([]byte, error) { + // Flatten the chunk out into a full 2D array of all its points. + var ( + size = int(a.acc.size) + grid, err = Make2DChunkGrid(size) + ) + if err != nil { + return nil, err + } + + // Populate the dense 2D array of its pixels. + for px := range a.Iter() { + var ( + point = render.NewPoint(px.X, px.Y) + relative = RelativeCoordinate(point, a.acc.coord, a.acc.size) + ptr = uint64(px.PaletteIndex) + ) + grid[relative.Y][relative.X] = &ptr + } + + // log.Error("2D GRID:\n%+v", grid) + + // Run-length encode the grid. + var ( + compressed []byte + firstColor = true + lastColor uint64 + runLength uint64 + ) + for _, row := range grid { + for _, color := range row { + var index uint64 + if color == nil { + index = 0xFF + } + + if firstColor { + lastColor = index + runLength = 1 + firstColor = false + continue + } + + if index != lastColor { + compressed = binary.AppendUvarint(compressed, index) + compressed = binary.AppendUvarint(compressed, runLength) + lastColor = index + runLength = 1 + continue + } + + runLength++ + } + } + + log.Error("RLE compressed: %v", compressed) + + return compressed, nil +} + +// UnmarshalBinary will decode a compressed RLEAccessor byte stream. +func (a *RLEAccessor) UnmarshalBinary(compressed []byte) error { + a.acc.mu.Lock() + defer a.acc.mu.Unlock() + + // New format: decompress the byte stream. + log.Debug("RLEAccessor.Unmarshal: Reading %d bytes of compressed chunk data", len(compressed)) + + // Prepare the 2D grid to decompress the RLE stream into. + var ( + size = int(a.acc.size) + _, err = Make2DChunkGrid(size) + x, y, cursor int + ) + if err != nil { + return err + } + + var reader = bytes.NewBuffer(compressed) + + for { + var ( + paletteIndex, err1 = binary.ReadUvarint(reader) + repeatCount, err2 = binary.ReadUvarint(reader) + ) + + if err1 != nil || err2 != nil { + log.Error("reading Uvarints from compressed data: {%s, %s}", err1, err2) + break + } + + for i := uint64(0); i < repeatCount; i++ { + cursor++ + if cursor%size == 0 { + y++ + x = 0 + } else { + x++ + } + + point := render.NewPoint(int(x), int(y)) + if paletteIndex != 0xFF { + a.acc.grid[point] = NewSparseSwatch(int(paletteIndex)) + } + } + } + + return nil +} diff --git a/pkg/level/chunk_test.go b/pkg/level/chunk_test.go index 43c4300..e9c614f 100644 --- a/pkg/level/chunk_test.go +++ b/pkg/level/chunk_test.go @@ -242,54 +242,88 @@ func TestChunkCoordinates(t *testing.T) { c := level.NewChunker(128) type testCase struct { - In render.Point - Expect render.Point + WorldCoordinate render.Point + ChunkCoordinate render.Point + RelativeCoordinate render.Point } tests := []testCase{ testCase{ - In: render.NewPoint(0, 0), - Expect: render.NewPoint(0, 0), + WorldCoordinate: render.NewPoint(0, 0), + ChunkCoordinate: render.NewPoint(0, 0), + RelativeCoordinate: render.NewPoint(0, 0), }, testCase{ - In: render.NewPoint(128, 128), - Expect: render.NewPoint(0, 0), + WorldCoordinate: render.NewPoint(4, 8), + ChunkCoordinate: render.NewPoint(0, 0), + RelativeCoordinate: render.NewPoint(4, 8), }, testCase{ - In: render.NewPoint(1024, 128), - Expect: render.NewPoint(1, 0), + WorldCoordinate: render.NewPoint(128, 128), + ChunkCoordinate: render.NewPoint(1, 1), + RelativeCoordinate: render.NewPoint(0, 0), }, testCase{ - In: render.NewPoint(3600, 1228), - Expect: render.NewPoint(3, 1), + WorldCoordinate: render.NewPoint(130, 156), + ChunkCoordinate: render.NewPoint(1, 1), + RelativeCoordinate: render.NewPoint(2, 28), }, testCase{ - In: render.NewPoint(-100, -1), - Expect: render.NewPoint(-1, -1), + WorldCoordinate: render.NewPoint(1024, 128), + ChunkCoordinate: render.NewPoint(8, 1), + RelativeCoordinate: render.NewPoint(0, 0), }, testCase{ - In: render.NewPoint(-950, 100), - Expect: render.NewPoint(-1, 0), + WorldCoordinate: render.NewPoint(3600, 1228), + ChunkCoordinate: render.NewPoint(28, 9), + RelativeCoordinate: render.NewPoint(16, 76), }, testCase{ - In: render.NewPoint(-1001, -856), - Expect: render.NewPoint(-2, -1), + WorldCoordinate: render.NewPoint(-100, -1), + ChunkCoordinate: render.NewPoint(-1, -1), + RelativeCoordinate: render.NewPoint(28, 127), }, testCase{ - In: render.NewPoint(-3600, -4800), - Expect: render.NewPoint(-4, -5), + WorldCoordinate: render.NewPoint(-950, 100), + ChunkCoordinate: render.NewPoint(-8, 0), + RelativeCoordinate: render.NewPoint(74, 100), + }, + testCase{ + WorldCoordinate: render.NewPoint(-1001, -856), + ChunkCoordinate: render.NewPoint(-8, -7), + RelativeCoordinate: render.NewPoint(23, 40), + }, + testCase{ + WorldCoordinate: render.NewPoint(-3600, -4800), + ChunkCoordinate: render.NewPoint(-29, -38), + RelativeCoordinate: render.NewPoint(112, 64), }, } for _, test := range tests { - actual := c.ChunkCoordinate(test.In) - if actual != test.Expect { + // Test conversion from world to chunk coordinate. + actual := c.ChunkCoordinate(test.WorldCoordinate) + if actual != test.ChunkCoordinate { t.Errorf( "Failed ChunkCoordinate conversion:\n"+ " Input: %s\n"+ "Expected: %s\n"+ " Got: %s", - test.In, - test.Expect, + test.WorldCoordinate, + test.ChunkCoordinate, + actual, + ) + } + + // Test the relative (inside-chunk) coordinate. + actual = level.RelativeCoordinate(test.WorldCoordinate, actual, c.Size) + if actual != test.RelativeCoordinate { + t.Errorf( + "Failed RelativeCoordinate conversion:\n"+ + " Input: %s\n"+ + "Expected: %s\n"+ + " Got: %s", + test.WorldCoordinate, + test.RelativeCoordinate, actual, ) } diff --git a/pkg/level/chunker.go b/pkg/level/chunker.go index e1c00c8..72e5763 100644 --- a/pkg/level/chunker.go +++ b/pkg/level/chunker.go @@ -74,6 +74,7 @@ func (c *Chunker) Inflate(pal *Palette) error { for coord, chunk := range c.Chunks { chunk.Point = coord chunk.Size = c.Size + chunk.SetChunkCoordinate(chunk.Point, chunk.Size) chunk.Inflate(pal) } return nil @@ -445,6 +446,7 @@ func (c *Chunker) FreeCaches() int { // This function should be the singular writer to the chunk cache. func (c *Chunker) SetChunk(p render.Point, chunk *Chunk) { c.chunkMu.Lock() + chunk.SetChunkCoordinate(p, chunk.Size) c.Chunks[p] = chunk c.chunkMu.Unlock() @@ -605,6 +607,30 @@ func (c *Chunker) ChunkCoordinate(abs render.Point) render.Point { ) } +// RelativeCoordinate will translate from an absolute world coordinate, into one that +// is relative to fit inside of the chunk with the given chunk coordinate and size. +// +// Example: +// +// - With 128x128 chunks and a world coordinate of (280,-600) +// - The ChunkCoordinate would be (2,-4) which encompasses (256,-512) to (383,-639) +// - And relative inside that chunk, the pixel is at (24,) +func RelativeCoordinate(abs render.Point, chunkCoord render.Point, chunkSize uint8) render.Point { + // Pixel coordinate offset. + var ( + size = int(chunkSize) + offset = render.Point{ + X: chunkCoord.X * size, + Y: chunkCoord.Y * size, + } + ) + + return render.Point{ + X: abs.X - offset.X, + Y: abs.Y - offset.Y, + } +} + // ChunkMap maps a chunk coordinate to its chunk data. type ChunkMap map[render.Point]*Chunk diff --git a/pkg/level/chunker_zipfile.go b/pkg/level/chunker_zipfile.go index 53c1810..8eb04d6 100644 --- a/pkg/level/chunker_zipfile.go +++ b/pkg/level/chunker_zipfile.go @@ -2,6 +2,7 @@ package level import ( "archive/zip" + "errors" "fmt" "io/ioutil" "regexp" @@ -190,11 +191,7 @@ func (c *Chunk) ToZipfile(zf *zip.Writer, layer int, coord render.Point) error { data = bytes } } else { - if json, err := c.MarshalJSON(); err != nil { - return err - } else { - data = json - } + return errors.New("Chunk.ToZipfile: JSON chunk format no longer supported for writing") } // Write the file contents to zip whether binary or json. @@ -226,6 +223,7 @@ func ChunkFromZipfile(zf *zip.Reader, layer int, coord render.Point) (*Chunk, er err = chunk.UnmarshalBinary(bin) if err != nil { + log.Error("ChunkFromZipfile(%s): %s", coord, err) return nil, err } } else if file, err := zf.Open(jsonfile); err == nil {