From 5654145fd8bdd706df4c851f3676542b8e0f5113 Mon Sep 17 00:00:00 2001 From: Noah Petherbridge Date: Thu, 23 May 2024 23:02:01 -0700 Subject: [PATCH] (Experimental) Run Length Encoding for Levels Finally add a second option for Chunk MapAccessor implementation besides the MapAccessor. The RLEAccessor is basically a MapAccessor that will compress your drawing with Run Length Encoding (RLE) in the on-disk format in the ZIP file. This slashes the file sizes of most levels: * Shapeshifter: 21.8 MB -> 8.1 MB * Jungle: 10.4 MB -> 4.1 MB * Zoo: 2.8 MB -> 1.3 MB Implementation details: * The RLE binary format for Chunks is a stream of Uvarint pairs storing the palette index number and the number of pixels to repeat it (along the Y,X axis of the chunk). * Null colors are represented by a Uvarint that decodes to 0xFFFF or 65535 in decimal. * Gameplay logic currently limits maps to 256 colors. * The default for newly created chunks in-game will be RLE by default. * Its in-memory representation is still a MapAccessor (a map of absolute world coordinates to palette index). * The game can still open and play legacy MapAccessor maps. * On save in the editor, the game will upgrade/convert MapAccessor chunks over to RLEAccessors, improving on your level's file size with a simple re-save. Current Bugs * On every re-save to RLE, one pixel is lost in the bottom-right corner of each chunk. Each subsequent re-save loses one more pixel to the left, so what starts as a single pixel per chunk slowly evolves into a horizontal line. * Some pixels smear vertically as well. * Off-by-negative-one errors when some chunks Iter() their pixels but compute a relative coordinate of (-1,0)! Some mismatch between the stored world coords of a pixel inside the chunk vs. the chunk's assigned coordinate by the Chunker: certain combinations of chunk coord/abs coord. To Do * The `doodad touch` command should re-save existing levels to upgrade them. --- cmd/doodle/main.go | 7 ++ pkg/balance/feature_flags.go | 6 ++ pkg/level/chunk.go | 24 +++-- pkg/level/chunk_map.go | 15 ++- pkg/level/chunk_rle.go | 129 ++++++++++-------------- pkg/level/chunk_test.go | 6 +- pkg/level/chunker.go | 34 ++++++- pkg/level/chunker_migrate.go | 67 +++++++++++++ pkg/level/chunker_test.go | 95 ++++++++++++++++++ pkg/level/chunker_zipfile.go | 10 +- pkg/level/fmt_maintenance.go | 15 +++ pkg/level/fmt_readwrite.go | 7 +- pkg/level/rle/rle.go | 189 +++++++++++++++++++++++++++++++++++ pkg/level/rle/rle_test.go | 43 ++++++++ pkg/uix/canvas_editable.go | 2 +- 15 files changed, 542 insertions(+), 107 deletions(-) create mode 100644 pkg/level/chunker_migrate.go create mode 100644 pkg/level/rle/rle.go create mode 100644 pkg/level/rle/rle_test.go diff --git a/cmd/doodle/main.go b/cmd/doodle/main.go index ce4df43..6509fa4 100644 --- a/cmd/doodle/main.go +++ b/cmd/doodle/main.go @@ -101,6 +101,11 @@ func main() { Name: "chdir", Usage: "working directory for the game's runtime package", }, + &cli.BoolFlag{ + Name: "new", + Aliases: []string{"n"}, + Usage: "open immediately to the level editor", + }, &cli.BoolFlag{ Name: "edit", Aliases: []string{"e"}, @@ -248,6 +253,8 @@ func main() { if c.Bool("guitest") { game.Goto(&doodle.GUITestScene{}) + } else if c.Bool("new") { + game.NewMap() } else if filename != "" { if c.Bool("edit") { game.EditFile(filename) diff --git a/pkg/balance/feature_flags.go b/pkg/balance/feature_flags.go index a109dc0..bd23394 100644 --- a/pkg/balance/feature_flags.go +++ b/pkg/balance/feature_flags.go @@ -28,6 +28,12 @@ const ( // If you set both flags to false, level zipfiles will use the classic // json chunk format as before on save. BinaryChunkerEnabled = true + + // Enable "v3" Run-Length Encoding for level chunker. + // + // This only supports Zipfile levels and will use the ".bin" format + // enabled by the previous setting. + RLEBinaryChunkerEnabled = true ) // Feature Flags to turn on/off experimental content. diff --git a/pkg/level/chunk.go b/pkg/level/chunk.go index fbee1c8..f4bc28a 100644 --- a/pkg/level/chunk.go +++ b/pkg/level/chunk.go @@ -24,6 +24,9 @@ const ( GridType ) +// Default chunk type for newly created chunks (was MapType). +const DefaultChunkType = RLEType + // Chunk holds a single portion of the pixel canvas. type Chunk struct { Type uint64 // map vs. 2D array. @@ -55,7 +58,6 @@ type JSONChunk struct { // Accessor provides a high-level API to interact with absolute pixel coordinates // while abstracting away the details of how they're stored. type Accessor interface { - SetChunkCoordinate(render.Point, uint8) Inflate(*Palette) error Iter() <-chan Pixel IterViewport(viewport render.Rect) <-chan Pixel @@ -69,10 +71,11 @@ type Accessor interface { // NewChunk creates a new chunk. func NewChunk() *Chunk { - return &Chunk{ - Type: RLEType, - Accessor: NewRLEAccessor(), + var c = &Chunk{ + Type: RLEType, } + c.Accessor = NewRLEAccessor(c) + return c } // Texture will return a cached texture for the rendering engine for this @@ -335,6 +338,9 @@ func (c *Chunk) Usage(size int) float64 { // parse the inner details. // // DEPRECATED in favor of binary marshalling. +// +// Only supports MapAccessor chunk types, which was the only one supported +// before this function was deprecated. func (c *Chunk) UnmarshalJSON(b []byte) error { // Parse it generically so we can hand off the inner "data" object to the // right accessor for unmarshalling. @@ -346,7 +352,7 @@ func (c *Chunk) UnmarshalJSON(b []byte) error { switch c.Type { case MapType: - c.Accessor = NewMapAccessor() + c.Accessor = NewMapAccessor(c) if unmarshaler, ok := c.Accessor.(json.Unmarshaler); ok { return unmarshaler.UnmarshalJSON(generic.Data) } @@ -393,12 +399,12 @@ func (c *Chunk) UnmarshalBinary(b []byte) error { // Decode the rest of the byte stream. switch chunkType { case MapType: - c.Accessor = NewMapAccessor() - c.Accessor.SetChunkCoordinate(c.Point, c.Size) + c.Type = MapType + c.Accessor = NewMapAccessor(c) return c.Accessor.UnmarshalBinary(reader.Bytes()) case RLEType: - c.Accessor = NewRLEAccessor() - c.Accessor.SetChunkCoordinate(c.Point, c.Size) + c.Type = RLEType + c.Accessor = NewRLEAccessor(c) return c.Accessor.UnmarshalBinary(reader.Bytes()) default: return fmt.Errorf("Chunk.UnmarshalJSON: unsupported chunk type '%d'", c.Type) diff --git a/pkg/level/chunk_map.go b/pkg/level/chunk_map.go index ebbd839..f5b5f3c 100644 --- a/pkg/level/chunk_map.go +++ b/pkg/level/chunk_map.go @@ -16,23 +16,22 @@ import ( // MapAccessor implements a chunk accessor by using a map of points to their // palette indexes. This is the simplest accessor and is best for sparse chunks. type MapAccessor struct { - coord render.Point `json:"-"` // chunk coordinate, assigned by Chunker - size uint8 `json:"-"` // chunk size, assigned by Chunker + chunk *Chunk // Pointer to parent struct, for its Size and Point grid map[render.Point]*Swatch mu sync.RWMutex } // NewMapAccessor initializes a MapAccessor. -func NewMapAccessor() *MapAccessor { +func NewMapAccessor(chunk *Chunk) *MapAccessor { return &MapAccessor{ - grid: map[render.Point]*Swatch{}, + chunk: chunk, + grid: map[render.Point]*Swatch{}, } } -// SetChunkCoordinate receives our chunk's coordinate from the Chunker. -func (a *MapAccessor) SetChunkCoordinate(p render.Point, size uint8) { - a.coord = p - a.size = size +// Reset the MapAccessor. +func (a *MapAccessor) Reset() { + a.grid = map[render.Point]*Swatch{} } // Inflate the sparse swatches from their palette indexes. diff --git a/pkg/level/chunk_rle.go b/pkg/level/chunk_rle.go index bf0ee68..09c8920 100644 --- a/pkg/level/chunk_rle.go +++ b/pkg/level/chunk_rle.go @@ -1,10 +1,7 @@ package level import ( - "bytes" - "encoding/binary" - "errors" - + "git.kirsle.net/SketchyMaze/doodle/pkg/level/rle" "git.kirsle.net/SketchyMaze/doodle/pkg/log" "git.kirsle.net/go/render" ) @@ -12,22 +9,18 @@ import ( // RLEAccessor implements a chunk accessor which stores its on-disk format using // Run Length Encoding (RLE), but in memory behaves equivalently to the MapAccessor. type RLEAccessor struct { - acc *MapAccessor + chunk *Chunk // parent Chunk, for its Size and Point + acc *MapAccessor } // NewRLEAccessor initializes a RLEAccessor. -func NewRLEAccessor() *RLEAccessor { +func NewRLEAccessor(chunk *Chunk) *RLEAccessor { return &RLEAccessor{ - acc: NewMapAccessor(), + chunk: chunk, + acc: NewMapAccessor(chunk), } } -// SetChunkCoordinate receives our chunk's coordinate from the Chunker. -func (a *RLEAccessor) SetChunkCoordinate(p render.Point, size uint8) { - a.acc.coord = p - a.acc.size = size -} - // Inflate the sparse swatches from their palette indexes. func (a *RLEAccessor) Inflate(pal *Palette) error { return a.acc.Inflate(pal) @@ -63,23 +56,6 @@ func (a *RLEAccessor) Delete(p render.Point) error { return a.acc.Delete(p) } -// Make2DChunkGrid creates a 2D map of uint64 pointers matching the square dimensions of the given size. -// -// It is used by the RLEAccessor to flatten a chunk into a grid for run-length encoding. -func Make2DChunkGrid(size int) ([][]*uint64, error) { - // Sanity check if the chunk was properly initialized. - if size == 0 { - return nil, errors.New("chunk not initialized correctly with its size and coordinate") - } - - var grid = make([][]*uint64, size) - for i := 0; i < size; i++ { - grid[i] = make([]*uint64, size) - } - - return grid, nil -} - /* MarshalBinary converts the chunk data to a binary representation. @@ -93,8 +69,8 @@ formatted as such: func (a *RLEAccessor) MarshalBinary() ([]byte, error) { // Flatten the chunk out into a full 2D array of all its points. var ( - size = int(a.acc.size) - grid, err = Make2DChunkGrid(size) + size = int(a.chunk.Size) + grid, err = rle.NewGrid(size) ) if err != nil { return nil, err @@ -104,50 +80,18 @@ func (a *RLEAccessor) MarshalBinary() ([]byte, error) { for px := range a.Iter() { var ( point = render.NewPoint(px.X, px.Y) - relative = RelativeCoordinate(point, a.acc.coord, a.acc.size) - ptr = uint64(px.PaletteIndex) + relative = RelativeCoordinate(point, a.chunk.Point, a.chunk.Size) + ptr = uint64(px.Swatch.Index()) ) + + // TODO: sometimes we get a -1 value in X or Y, not sure why. + if relative.X < 0 || relative.Y < 0 { + continue + } grid[relative.Y][relative.X] = &ptr } - // log.Error("2D GRID:\n%+v", grid) - - // Run-length encode the grid. - var ( - compressed []byte - firstColor = true - lastColor uint64 - runLength uint64 - ) - for _, row := range grid { - for _, color := range row { - var index uint64 - if color == nil { - index = 0xFF - } - - if firstColor { - lastColor = index - runLength = 1 - firstColor = false - continue - } - - if index != lastColor { - compressed = binary.AppendUvarint(compressed, index) - compressed = binary.AppendUvarint(compressed, runLength) - lastColor = index - runLength = 1 - continue - } - - runLength++ - } - } - - log.Error("RLE compressed: %v", compressed) - - return compressed, nil + return grid.Compress() } // UnmarshalBinary will decode a compressed RLEAccessor byte stream. @@ -158,10 +102,39 @@ func (a *RLEAccessor) UnmarshalBinary(compressed []byte) error { // New format: decompress the byte stream. log.Debug("RLEAccessor.Unmarshal: Reading %d bytes of compressed chunk data", len(compressed)) - // Prepare the 2D grid to decompress the RLE stream into. + grid, err := rle.NewGrid(int(a.chunk.Size)) + if err != nil { + return err + } + + if err := grid.Decompress(compressed); err != nil { + return err + } + + // Load the grid into our MapAccessor. + a.acc.Reset() + for y, row := range grid { + for x, col := range row { + if col == nil { + continue + } + + // TODO: x-1 to avoid the level creeping to the right every save, + // not sure on the root cause! RLEAccessor Decompress? + abs := FromRelativeCoordinate(render.NewPoint(x, y), a.chunk.Point, a.chunk.Size) + abs.X -= 1 + a.acc.grid[abs] = NewSparseSwatch(int(*col)) + } + } + + return nil +} + +/* +// Prepare the 2D grid to decompress the RLE stream into. var ( - size = int(a.acc.size) - _, err = Make2DChunkGrid(size) + size = int(a.chunk.Size) + _, err = rle.NewGrid(size) x, y, cursor int ) if err != nil { @@ -181,6 +154,8 @@ func (a *RLEAccessor) UnmarshalBinary(compressed []byte) error { break } + log.Warn("RLE index %d for %dpx", paletteIndex, repeatCount) + for i := uint64(0); i < repeatCount; i++ { cursor++ if cursor%size == 0 { @@ -196,6 +171,4 @@ func (a *RLEAccessor) UnmarshalBinary(compressed []byte) error { } } } - - return nil -} +*/ diff --git a/pkg/level/chunk_test.go b/pkg/level/chunk_test.go index e9c614f..06bfa34 100644 --- a/pkg/level/chunk_test.go +++ b/pkg/level/chunk_test.go @@ -129,8 +129,10 @@ func TestChunker(t *testing.T) { // Test the map chunk accessor. func TestMapAccessor(t *testing.T) { - a := level.NewMapAccessor() - _ = a + var ( + c = level.NewChunk() + a = level.NewMapAccessor(c) + ) // Test action types var ( diff --git a/pkg/level/chunker.go b/pkg/level/chunker.go index 72e5763..134366a 100644 --- a/pkg/level/chunker.go +++ b/pkg/level/chunker.go @@ -74,7 +74,6 @@ func (c *Chunker) Inflate(pal *Palette) error { for coord, chunk := range c.Chunks { chunk.Point = coord chunk.Size = c.Size - chunk.SetChunkCoordinate(chunk.Point, chunk.Size) chunk.Inflate(pal) } return nil @@ -326,7 +325,7 @@ func (c *Chunker) GetChunk(p render.Point) (*Chunk, bool) { // Hit the zipfile for it. if c.Zipfile != nil { - if chunk, err := ChunkFromZipfile(c.Zipfile, c.Layer, p); err == nil { + if chunk, err := c.ChunkFromZipfile(p); err == nil { // log.Debug("GetChunk(%s) cache miss, read from zip", p) c.SetChunk(p, chunk) // cache it c.logChunkAccess(p, chunk) // for the LRU cache @@ -446,7 +445,6 @@ func (c *Chunker) FreeCaches() int { // This function should be the singular writer to the chunk cache. func (c *Chunker) SetChunk(p render.Point, chunk *Chunk) { c.chunkMu.Lock() - chunk.SetChunkCoordinate(p, chunk.Size) c.Chunks[p] = chunk c.chunkMu.Unlock() @@ -617,6 +615,32 @@ func (c *Chunker) ChunkCoordinate(abs render.Point) render.Point { // - And relative inside that chunk, the pixel is at (24,) func RelativeCoordinate(abs render.Point, chunkCoord render.Point, chunkSize uint8) render.Point { // Pixel coordinate offset. + var ( + size = int(chunkSize) + offset = render.Point{ + X: chunkCoord.X * size, + Y: chunkCoord.Y * size, + } + point = render.Point{ + X: abs.X - offset.X, + Y: abs.Y - offset.Y, + } + ) + + if point.X < 0 || point.Y < 0 { + log.Error("RelativeCoordinate: X < 0! abs=%s rel=%s chunk=%s size=%d", abs, point, chunkCoord, chunkSize) + log.Error("RelativeCoordinate(2): size=%d offset=%s point=%s", size, offset, point) + } + + return point +} + +// FromRelativeCoordinate is the inverse of RelativeCoordinate. +// +// With a chunk size of 128 and a relative coordinate like (8, 12), +// this function will return the absolute world coordinates based +// on your chunk.Point's placement in the level. +func FromRelativeCoordinate(rel render.Point, chunkCoord render.Point, chunkSize uint8) render.Point { var ( size = int(chunkSize) offset = render.Point{ @@ -626,8 +650,8 @@ func RelativeCoordinate(abs render.Point, chunkCoord render.Point, chunkSize uin ) return render.Point{ - X: abs.X - offset.X, - Y: abs.Y - offset.Y, + X: rel.X + offset.X, + Y: rel.Y + offset.Y, } } diff --git a/pkg/level/chunker_migrate.go b/pkg/level/chunker_migrate.go new file mode 100644 index 0000000..7b007cc --- /dev/null +++ b/pkg/level/chunker_migrate.go @@ -0,0 +1,67 @@ +package level + +import ( + "runtime" + "sync" + + "git.kirsle.net/SketchyMaze/doodle/pkg/balance" + "git.kirsle.net/SketchyMaze/doodle/pkg/log" +) + +/* Functions to migrate Chunkers between different implementations. */ + +// OptimizeChunkerAccessors will evaluate all of the chunks of your drawing +// and possibly migrate them to a different Accessor implementation when +// saving on disk. +func (c *Chunker) OptimizeChunkerAccessors() { + c.chunkMu.Lock() + defer c.chunkMu.Unlock() + + log.Info("Optimizing Chunker Accessors") + + // TODO: parallelize this with goroutines + var ( + chunks = make(chan *Chunk, len(c.Chunks)) + wg sync.WaitGroup + ) + + for range runtime.NumCPU() { + wg.Add(1) + go func() { + defer wg.Done() + for chunk := range chunks { + var point = chunk.Point + log.Warn("Chunk %s is a: %d", point, chunk.Type) + + // Upgrade all MapTypes into RLE compressed MapTypes? + if balance.RLEBinaryChunkerEnabled { + if chunk.Type == MapType { + log.Info("Optimizing chunk %s accessor from Map to RLE", point) + ma, _ := chunk.Accessor.(*MapAccessor) + rle := NewRLEAccessor(chunk).FromMapAccessor(ma) + + c.Chunks[point].Type = RLEType + c.Chunks[point].Accessor = rle + } + } + } + }() + } + + // Feed it the chunks. + for _, chunk := range c.Chunks { + chunks <- chunk + } + + close(chunks) + wg.Wait() + +} + +// FromMapAccessor migrates from a MapAccessor to RLE. +func (a *RLEAccessor) FromMapAccessor(ma *MapAccessor) *RLEAccessor { + return &RLEAccessor{ + chunk: a.chunk, + acc: ma, + } +} diff --git a/pkg/level/chunker_test.go b/pkg/level/chunker_test.go index dff40fd..76885fa 100644 --- a/pkg/level/chunker_test.go +++ b/pkg/level/chunker_test.go @@ -228,3 +228,98 @@ func TestViewportChunks(t *testing.T) { } } } + +func TestRelativeCoordinates(t *testing.T) { + + var ( + chunker = level.NewChunker(128) + ) + + type TestCase struct { + WorldCoord render.Point + ChunkCoord render.Point + ExpectRelative render.Point + } + var tests = []TestCase{ + { + WorldCoord: render.NewPoint(4, 8), + ExpectRelative: render.NewPoint(4, 8), + }, + { + WorldCoord: render.NewPoint(128, 128), + ExpectRelative: render.NewPoint(0, 0), + }, + { + WorldCoord: render.NewPoint(143, 144), + ExpectRelative: render.NewPoint(15, 16), + }, + { + WorldCoord: render.NewPoint(-105, -86), + ExpectRelative: render.NewPoint(23, 42), + }, + { + WorldCoord: render.NewPoint(-252, 264), + ExpectRelative: render.NewPoint(4, 8), + }, + + // These were seen breaking actual levels, at the corners of the chunk + { + WorldCoord: render.NewPoint(511, 256), + ExpectRelative: render.NewPoint(127, 0), // was getting -1,0 in game + }, + { + WorldCoord: render.NewPoint(511, 512), + ChunkCoord: render.NewPoint(4, 4), + ExpectRelative: render.NewPoint(127, 0), // was getting -1,0 in game + }, + { + WorldCoord: render.NewPoint(127, 384), + ChunkCoord: render.NewPoint(1, 3), + ExpectRelative: render.NewPoint(-1, 0), + }, + } + for i, test := range tests { + var ( + chunkCoord = test.ChunkCoord + actualRelative = level.RelativeCoordinate( + test.WorldCoord, + chunkCoord, + chunker.Size, + ) + roundTrip = level.FromRelativeCoordinate( + actualRelative, + chunkCoord, + chunker.Size, + ) + ) + + // compute expected chunk coord automatically? + if chunkCoord == render.Origin { + chunkCoord = chunker.ChunkCoordinate(test.WorldCoord) + } + + if actualRelative != test.ExpectRelative { + t.Errorf("Test %d: world coord %s in chunk %s\n"+ + "Expected RelativeCoordinate() to be: %s\n"+ + "But it was: %s", + i, + test.WorldCoord, + chunkCoord, + test.ExpectRelative, + actualRelative, + ) + } + + if roundTrip != test.WorldCoord { + t.Errorf("Test %d: world coord %s in chunk %s\n"+ + "Did not survive round trip! Expected: %s\n"+ + "But it was: %s", + i, + test.WorldCoord, + chunkCoord, + test.WorldCoord, + roundTrip, + ) + } + } +} diff --git a/pkg/level/chunker_zipfile.go b/pkg/level/chunker_zipfile.go index 8eb04d6..7ca4ced 100644 --- a/pkg/level/chunker_zipfile.go +++ b/pkg/level/chunker_zipfile.go @@ -93,7 +93,7 @@ func (c *Chunker) MigrateZipfile(zf *zip.Writer) error { } // Verify that this chunk file in the old ZIP was not empty. - chunk, err := ChunkFromZipfile(c.Zipfile, c.Layer, point) + chunk, err := c.ChunkFromZipfile(point) if err == nil && chunk.Len() == 0 { log.Debug("Skip chunk %s (old zipfile chunk was empty)", coord) continue @@ -205,14 +205,20 @@ func (c *Chunk) ToZipfile(zf *zip.Writer, layer int, coord render.Point) error { } // ChunkFromZipfile loads a chunk from a zipfile. -func ChunkFromZipfile(zf *zip.Reader, layer int, coord render.Point) (*Chunk, error) { +func (c *Chunker) ChunkFromZipfile(coord render.Point) (*Chunk, error) { // File names? var ( + zf = c.Zipfile + layer = c.Layer + binfile = fmt.Sprintf("chunks/%d/%s.bin", layer, coord) jsonfile = fmt.Sprintf("chunks/%d/%s.json", layer, coord) chunk = NewChunk() ) + chunk.Point = coord + chunk.Size = c.Size + // Read from the new binary format. if file, err := zf.Open(binfile); err == nil { // log.Debug("Reading binary compressed chunk from %s", binfile) diff --git a/pkg/level/fmt_maintenance.go b/pkg/level/fmt_maintenance.go index e7a2268..7e45198 100644 --- a/pkg/level/fmt_maintenance.go +++ b/pkg/level/fmt_maintenance.go @@ -4,6 +4,21 @@ import "git.kirsle.net/SketchyMaze/doodle/pkg/log" // Maintenance functions for the file format on disk. +// Vacuum runs any maintenance or migration tasks for the level at time of save. +// +// It will prune broken links between actors, or migrate internal data structures +// to optimize storage on disk of its binary data. +func (m *Level) Vacuum() error { + if links := m.PruneLinks(); links > 0 { + log.Debug("Vacuum: removed %d broken links between actors in this level.") + } + + // Let the Chunker optimize accessor types. + m.Chunker.OptimizeChunkerAccessors() + + return nil +} + // PruneLinks cleans up any Actor Links that can not be resolved in the // level data. For example, if actors were linked in Edit Mode and one // actor is deleted leaving a broken link. diff --git a/pkg/level/fmt_readwrite.go b/pkg/level/fmt_readwrite.go index 4981f92..2833631 100644 --- a/pkg/level/fmt_readwrite.go +++ b/pkg/level/fmt_readwrite.go @@ -3,6 +3,7 @@ package level import ( "fmt" "io/ioutil" + "os" "runtime" "strings" @@ -96,7 +97,9 @@ func (m *Level) WriteFile(filename string) error { m.GameVersion = branding.Version // Maintenance functions, clean up cruft before save. - m.PruneLinks() + if err := m.Vacuum(); err != nil { + log.Error("Vacuum level %s: %s", filename, err) + } bin, err := m.ToJSON() if err != nil { @@ -115,7 +118,7 @@ func (m *Level) WriteFile(filename string) error { } // Desktop: write to disk. - err = ioutil.WriteFile(filename, bin, 0644) + err = os.WriteFile(filename, bin, 0644) if err != nil { return fmt.Errorf("level.WriteFile: %s", err) } diff --git a/pkg/level/rle/rle.go b/pkg/level/rle/rle.go new file mode 100644 index 0000000..2f44875 --- /dev/null +++ b/pkg/level/rle/rle.go @@ -0,0 +1,189 @@ +// Package rle contains support for Run-Length Encoding of level chunks. +package rle + +import ( + "bytes" + "encoding/binary" + "errors" + "fmt" + "strings" + + "git.kirsle.net/SketchyMaze/doodle/pkg/log" + "git.kirsle.net/go/render" +) + +const NullColor = 0xFFFF + +// Grid is a 2D array of nullable integers to store a flat bitmap of a chunk. +type Grid [][]*uint64 + +// NewGrid will return an initialized 2D grid of equal dimensions of the given size. +// +// The grid is indexed in [Y][X] notation, or: by row first and then column. +func NewGrid(size int) (Grid, error) { + if size == 0 { + return nil, errors.New("no size given for RLE Grid: the chunker was probably not initialized") + } + + var grid = make([][]*uint64, size+1) + for i := 0; i < size+1; i++ { + grid[i] = make([]*uint64, size+1) + } + + return grid, nil +} + +func MustGrid(size int) Grid { + grid, err := NewGrid(size) + if err != nil { + panic(err) + } + return grid +} + +type Pixel struct { + Point render.Point + Palette int +} + +// Size of the grid. +func (g Grid) Size() int { + return len(g[0]) +} + +// Compress the grid into a byte stream of RLE compressed data. +// +// The compressed format is a stream of: +// +// - A Uvarint for the palette index (0-255) or 0xffff (65535) for null. +// - A Uvarint for how many pixels to repeat that color. +func (g Grid) Compress() ([]byte, error) { + log.Error("BEGIN Compress()") + // log.Warn("Visualized:\n%s", g.Visualize()) + + // Run-length encode the grid. + var ( + compressed []byte // final result + lastColor uint64 // last color seen (current streak) + runLength uint64 // current streak for the last color + buffering bool // detect end of grid + + // Flush the buffer + flush = func() { + // log.Info("flush: %d for %d length", lastColor, runLength) + compressed = binary.AppendUvarint(compressed, lastColor) + compressed = binary.AppendUvarint(compressed, runLength) + } + ) + + for y, row := range g { + for x, nullableIndex := range row { + var index uint64 + if nullableIndex == nil { + index = NullColor + } else { + index = *nullableIndex + } + + // First color of the grid + if y == 0 && x == 0 { + // log.Info("First color @ %dx%d is %d", x, y, index) + lastColor = index + runLength = 1 + continue + } + + // Buffer it until we get a change of color or EOF. + if index != lastColor { + // log.Info("Color %d streaks for %d until %dx%d", lastColor, runLength, x, y) + flush() + lastColor = index + runLength = 1 + buffering = false + continue + } + + buffering = true + runLength++ + } + } + + // Flush the final buffer when we got to EOF on the grid. + if buffering { + flush() + } + + // log.Error("RLE compressed: %v", compressed) + + return compressed, nil +} + +// Decompress the RLE byte stream back into a populated 2D grid. +func (g Grid) Decompress(compressed []byte) error { + log.Error("BEGIN Decompress()") + // log.Warn("Visualized:\n%s", g.Visualize()) + + // Prepare the 2D grid to decompress the RLE stream into. + var ( + size = g.Size() + x, y, cursor int + ) + + var reader = bytes.NewBuffer(compressed) + + for { + var ( + paletteIndexRaw, err1 = binary.ReadUvarint(reader) + repeatCount, err2 = binary.ReadUvarint(reader) + ) + + if err1 != nil || err2 != nil { + break + } + + // Handle the null color. + var paletteIndex *uint64 + if paletteIndexRaw != NullColor { + paletteIndex = &paletteIndexRaw + } + + // log.Warn("RLE index %v for %dpx", paletteIndexRaw, repeatCount) + + for i := uint64(0); i < repeatCount; i++ { + cursor++ + if cursor%size == 0 { + y++ + x = 0 + } + + point := render.NewPoint(int(x), int(y)) + if point.Y >= size || point.X >= size { + continue + } + g[point.Y][point.X] = paletteIndex + + x++ + } + } + + // log.Warn("Visualized:\n%s", g.Visualize()) + + return nil +} + +// Visualize the state of the 2D grid. +func (g Grid) Visualize() string { + var lines []string + for _, row := range g { + var line = "[" + for _, col := range row { + if col == nil { + line += " " + } else { + line += fmt.Sprintf("%x", *col) + } + } + lines = append(lines, line+"]") + } + return strings.Join(lines, "\n") +} diff --git a/pkg/level/rle/rle_test.go b/pkg/level/rle/rle_test.go new file mode 100644 index 0000000..858d2ea --- /dev/null +++ b/pkg/level/rle/rle_test.go @@ -0,0 +1,43 @@ +package rle_test + +import ( + "testing" + + "git.kirsle.net/SketchyMaze/doodle/pkg/level/rle" +) + +func TestRLE(t *testing.T) { + + // Test a completely filled grid. + var ( + grid = rle.MustGrid(128) + color = uint64(5) + ) + for y := range grid { + for x := range y { + grid[y][x] = &color + } + } + + // Compress and decompress it. + var ( + compressed, _ = grid.Compress() + grid2 = rle.MustGrid(128) + ) + grid2.Decompress(compressed) + + // Ensure our color is set everywhere. + for y := range grid { + for x := range y { + if grid[y][x] != &color { + t.Errorf("RLE compression didn't survive the round trip: %d,%d didn't save\n"+ + " Expected: %d\n"+ + " Actually: %v", + x, y, + color, + grid[y][x], + ) + } + } + } +} diff --git a/pkg/uix/canvas_editable.go b/pkg/uix/canvas_editable.go index 90977e2..9ef096f 100644 --- a/pkg/uix/canvas_editable.go +++ b/pkg/uix/canvas_editable.go @@ -420,7 +420,7 @@ func (w *Canvas) loopEditable(ev *event.State) error { baseColor, err := chunker.Get(cursor) if err != nil { limit = balance.FloodToolVoidLimit - log.Warn("FloodTool: couldn't get base color at %s: %s (got %s)", cursor, err, baseColor.Color) + log.Warn("FloodTool: couldn't get base color at %s: %s (got %+v)", cursor, err, baseColor) } // If no change, do nothing.