From 4851730ccf6b449f22bec478d92b4d60cd5e4eb1 Mon Sep 17 00:00:00 2001 From: Noah Petherbridge Date: Fri, 24 May 2024 13:54:41 -0700 Subject: [PATCH] Fix RLE Encoding Off-by-One Errors [PTO] Levels can now be converted to RLE encoded chunk accessors and be re-saved continuously without any loss of information. Off-by-one errors resolved: * The rle.NewGrid() was adding a +1 everywhere making the 2D grids have 129 elements to a side for a 128 chunk size. * In rle.Decompress() the cursor value and translation to X,Y coordinates is fixed to avoid a pixel going missing at the end of the first row (128,0) * The abs.X-- hack in UnmarshalBinary is no longer needed to prevent the chunks from scooting a pixel to the right on every save. Doodad tool updates: * Remove unused CLI flags in `doodad resave` (actors, chunks, script, attachment, verbose) and add a `--output` flag to save to a different file name to the original. * Update `doodad show` to allow debugging of RLE compressed chunks: * CLI flag `--chunk=1,2` to specify a single chunk coordinate to debug * CLI flag `--visualize-rle` will Visualize() RLE compressed chunks in their 2D grid form in your terminal window (VERY noisy for large levels! Use the --chunk option to narrow to one chunk). Bug fixes and misc changes: * Chunk.Usage() to return a better percentage of chunk utilization. * Chunker.ChunkFromZipfile() was split out into two functions: * RawChunkFromZipfile retrieves the raw bytes of the chunk as well as the file extension discovered (.bin or .json) so the caller can interpret the bytes correctly. * ChunkFromZipfile calls the former function and then depending on file extension, unmarshals from binary or json. * The Raw function enables the `doodad show` command to debug and visualize the raw contents of the RLE compressed chunks. * Updated the Visualize() function for the RLE encoder: instead of converting palette indexes to hex (0-F) which would begin causing problems for palette indexes above 16 (as they would use two+ characters), indexes are mapped to a wider range of symbols (0-9A-Z) and roll over if you have more than 36 colors on your level. This at least keeps the Visualize() grid an easy to read 128x128 characters in your terminal. --- cmd/doodad/commands/resave.go | 41 ++++++++++---------- cmd/doodad/commands/show.go | 57 +++++++++++++++++++++++++++- pkg/level/chunk.go | 5 ++- pkg/level/chunk_rle.go | 26 ++++++------- pkg/level/chunker_migrate.go | 10 ++--- pkg/level/chunker_zipfile.go | 71 +++++++++++++++++++++-------------- pkg/level/rle/rle.go | 42 +++++++++++++-------- 7 files changed, 167 insertions(+), 85 deletions(-) diff --git a/cmd/doodad/commands/resave.go b/cmd/doodad/commands/resave.go index 75f6837..495b162 100644 --- a/cmd/doodad/commands/resave.go +++ b/cmd/doodad/commands/resave.go @@ -21,27 +21,10 @@ func init() { Usage: "load and re-save a level or doodad file to migrate to newer file format versions", ArgsUsage: "<.level or .doodad>", Flags: []cli.Flag{ - &cli.BoolFlag{ - Name: "actors", - Usage: "print verbose actor data in Level files", - }, - &cli.BoolFlag{ - Name: "chunks", - Usage: "print verbose data about all the pixel chunks in a file", - }, - &cli.BoolFlag{ - Name: "script", - Usage: "print the script from a doodad file and exit", - }, &cli.StringFlag{ - Name: "attachment", - Aliases: []string{"a"}, - Usage: "print the contents of the attached filename to terminal", - }, - &cli.BoolFlag{ - Name: "verbose", - Aliases: []string{"v"}, - Usage: "print verbose output (all verbose flags enabled)", + Name: "output", + Aliases: []string{"o"}, + Usage: "write to a different file than the input", }, }, Action: func(c *cli.Context) error { @@ -84,6 +67,18 @@ func resaveLevel(c *cli.Context, filename string) error { log.Info("Loaded level from file: %s", filename) log.Info("Last saved game version: %s", lvl.GameVersion) + // Different output filename? + if output := c.String("output"); output != "" { + log.Info("Output will be saved to: %s", output) + filename = output + } + + if err := lvl.Vacuum(); err != nil { + log.Error("Vacuum error: %s", err) + } else { + log.Info("Run vacuum on level file.") + } + log.Info("Saving back to disk") if err := lvl.WriteJSON(filename); err != nil { return fmt.Errorf("couldn't write %s: %s", filename, err) @@ -100,6 +95,12 @@ func resaveDoodad(c *cli.Context, filename string) error { log.Info("Loaded doodad from file: %s", filename) log.Info("Last saved game version: %s", dd.GameVersion) + // Different output filename? + if output := c.String("output"); output != "" { + log.Info("Output will be saved to: %s", output) + filename = output + } + log.Info("Saving back to disk") if err := dd.WriteJSON(filename); err != nil { return fmt.Errorf("couldn't write %s: %s", filename, err) diff --git a/cmd/doodad/commands/show.go b/cmd/doodad/commands/show.go index e43839c..9093607 100644 --- a/cmd/doodad/commands/show.go +++ b/cmd/doodad/commands/show.go @@ -1,6 +1,8 @@ package commands import ( + "bytes" + "encoding/binary" "fmt" "path/filepath" "sort" @@ -9,6 +11,7 @@ import ( "git.kirsle.net/SketchyMaze/doodle/pkg/doodads" "git.kirsle.net/SketchyMaze/doodle/pkg/enum" "git.kirsle.net/SketchyMaze/doodle/pkg/level" + "git.kirsle.net/SketchyMaze/doodle/pkg/level/rle" "git.kirsle.net/SketchyMaze/doodle/pkg/log" "github.com/urfave/cli/v2" ) @@ -44,6 +47,14 @@ func init() { Aliases: []string{"v"}, Usage: "print verbose output (all verbose flags enabled)", }, + &cli.BoolFlag{ + Name: "visualize-rle", + Usage: "visually dump RLE encoded chunks to the terminal (VERY noisy for large drawings!)", + }, + &cli.StringFlag{ + Name: "chunk", + Usage: "specific chunk coordinate; when debugging chunks, only show this chunk (example: 2,-1)", + }, }, Action: func(c *cli.Context) error { if c.NArg() < 1 { @@ -263,6 +274,10 @@ func showChunker(c *cli.Context, ch *level.Chunker) { chunkSize = int(ch.Size) width = worldSize.W - worldSize.X height = worldSize.H - worldSize.Y + + // Chunk debugging CLI options. + visualize = c.Bool("visualize-rle") + specificChunk = c.String("chunk") ) fmt.Println("Chunks:") fmt.Printf(" Pixels Per Chunk: %d^2\n", ch.Size) @@ -278,7 +293,18 @@ func showChunker(c *cli.Context, ch *level.Chunker) { // Verbose chunk information. if c.Bool("chunks") || c.Bool("verbose") { fmt.Println(" Chunk Details:") - for point, chunk := range ch.Chunks { + for point := range ch.IterChunks() { + // Debugging specific chunk coordinate? + if specificChunk != "" && point.String() != specificChunk { + log.Warn("Skip chunk %s: not the specific chunk you're looking for", point) + continue + } + + chunk, ok := ch.GetChunk(point) + if !ok { + continue + } + fmt.Printf(" - Coord: %s\n", point) fmt.Printf(" Type: %s\n", chunkTypeToName(chunk.Type)) fmt.Printf(" Range: (%d,%d) ... (%d,%d)\n", @@ -287,6 +313,33 @@ func showChunker(c *cli.Context, ch *level.Chunker) { (int(point.X)*chunkSize)+chunkSize, (int(point.Y)*chunkSize)+chunkSize, ) + fmt.Printf(" Usage: %f (%d len of %d)\n", chunk.Usage(), chunk.Len(), chunkSize*chunkSize) + + // Visualize the RLE encoded chunks? + if visualize && chunk.Type == level.RLEType { + ext, bin, err := ch.RawChunkFromZipfile(point) + if err != nil { + log.Error(err.Error()) + continue + } else if ext != ".bin" { + log.Error("Unexpected filetype for RLE compressed chunk (expected .bin, got %s)", ext) + continue + } + + // Read off the first byte (chunk type) + var reader = bytes.NewBuffer(bin) + binary.ReadUvarint(reader) + bin = reader.Bytes() + + grid, err := rle.NewGrid(chunkSize) + if err != nil { + log.Error(err.Error()) + continue + } + + grid.Decompress(bin) + fmt.Println(grid.Visualize()) + } } } else { fmt.Println(" Use -chunks or -verbose to serialize Chunks") @@ -298,6 +351,8 @@ func chunkTypeToName(v uint64) string { switch v { case level.MapType: return "map" + case level.RLEType: + return "rle map" case level.GridType: return "grid" default: diff --git a/pkg/level/chunk.go b/pkg/level/chunk.go index f4bc28a..80d5d53 100644 --- a/pkg/level/chunk.go +++ b/pkg/level/chunk.go @@ -330,8 +330,9 @@ func (c *Chunk) SizePositive() render.Rect { } // Usage returns the percent of free space vs. allocated pixels in the chunk. -func (c *Chunk) Usage(size int) float64 { - return float64(c.Len()) / float64(size) +func (c *Chunk) Usage() float64 { + size := float64(c.Size) + return float64(c.Len()) / (size * size) } // UnmarshalJSON loads the chunk from JSON and uses the correct accessor to diff --git a/pkg/level/chunk_rle.go b/pkg/level/chunk_rle.go index 09c8920..0744552 100644 --- a/pkg/level/chunk_rle.go +++ b/pkg/level/chunk_rle.go @@ -77,18 +77,21 @@ func (a *RLEAccessor) MarshalBinary() ([]byte, error) { } // Populate the dense 2D array of its pixels. - for px := range a.Iter() { - var ( - point = render.NewPoint(px.X, px.Y) - relative = RelativeCoordinate(point, a.chunk.Point, a.chunk.Size) - ptr = uint64(px.Swatch.Index()) - ) + for y, row := range grid { + for x := range row { + var ( + relative = render.NewPoint(x, y) + absolute = FromRelativeCoordinate(relative, a.chunk.Point, a.chunk.Size) + swatch, err = a.Get(absolute) + ) - // TODO: sometimes we get a -1 value in X or Y, not sure why. - if relative.X < 0 || relative.Y < 0 { - continue + if err != nil { + continue + } + + var ptr = uint64(swatch.Index()) + grid[relative.Y][relative.X] = &ptr } - grid[relative.Y][relative.X] = &ptr } return grid.Compress() @@ -119,10 +122,7 @@ func (a *RLEAccessor) UnmarshalBinary(compressed []byte) error { continue } - // TODO: x-1 to avoid the level creeping to the right every save, - // not sure on the root cause! RLEAccessor Decompress? abs := FromRelativeCoordinate(render.NewPoint(x, y), a.chunk.Point, a.chunk.Size) - abs.X -= 1 a.acc.grid[abs] = NewSparseSwatch(int(*col)) } } diff --git a/pkg/level/chunker_migrate.go b/pkg/level/chunker_migrate.go index 7b007cc..5b70fe2 100644 --- a/pkg/level/chunker_migrate.go +++ b/pkg/level/chunker_migrate.go @@ -14,9 +14,6 @@ import ( // and possibly migrate them to a different Accessor implementation when // saving on disk. func (c *Chunker) OptimizeChunkerAccessors() { - c.chunkMu.Lock() - defer c.chunkMu.Unlock() - log.Info("Optimizing Chunker Accessors") // TODO: parallelize this with goroutines @@ -31,7 +28,6 @@ func (c *Chunker) OptimizeChunkerAccessors() { defer wg.Done() for chunk := range chunks { var point = chunk.Point - log.Warn("Chunk %s is a: %d", point, chunk.Type) // Upgrade all MapTypes into RLE compressed MapTypes? if balance.RLEBinaryChunkerEnabled { @@ -49,7 +45,11 @@ func (c *Chunker) OptimizeChunkerAccessors() { } // Feed it the chunks. - for _, chunk := range c.Chunks { + for point := range c.IterChunks() { + chunk, ok := c.GetChunk(point) + if !ok { + continue + } chunks <- chunk } diff --git a/pkg/level/chunker_zipfile.go b/pkg/level/chunker_zipfile.go index 7ca4ced..7413515 100644 --- a/pkg/level/chunker_zipfile.go +++ b/pkg/level/chunker_zipfile.go @@ -4,7 +4,7 @@ import ( "archive/zip" "errors" "fmt" - "io/ioutil" + "io" "regexp" "strconv" @@ -206,6 +206,42 @@ func (c *Chunk) ToZipfile(zf *zip.Writer, layer int, coord render.Point) error { // ChunkFromZipfile loads a chunk from a zipfile. func (c *Chunker) ChunkFromZipfile(coord render.Point) (*Chunk, error) { + // Grab the chunk (bin or json) from the Zipfile. + ext, bin, err := c.RawChunkFromZipfile(coord) + if err != nil { + return nil, err + } + + var chunk = NewChunk() + chunk.Point = coord + chunk.Size = c.Size + + switch ext { + case ".bin": + // New style .bin compressed format: + // Either a MapAccessor compressed bin, or RLE compressed. + err = chunk.UnmarshalBinary(bin) + if err != nil { + log.Error("ChunkFromZipfile(%s): %s", coord, err) + return nil, err + } + case ".json": + // Legacy style plain .json file (MapAccessor only). + err = chunk.UnmarshalJSON(bin) + if err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("unexpected filetype found for this chunk: %s", ext) + } + + return chunk, nil +} + +// RawChunkFromZipfile loads a chunk from a zipfile and returns its raw binary content. +// +// Returns the file extension (".bin" or ".json"), raw bytes, and an error. +func (c *Chunker) RawChunkFromZipfile(coord render.Point) (string, []byte, error) { // File names? var ( zf = c.Zipfile @@ -213,41 +249,18 @@ func (c *Chunker) ChunkFromZipfile(coord render.Point) (*Chunk, error) { binfile = fmt.Sprintf("chunks/%d/%s.bin", layer, coord) jsonfile = fmt.Sprintf("chunks/%d/%s.json", layer, coord) - chunk = NewChunk() ) - chunk.Point = coord - chunk.Size = c.Size - // Read from the new binary format. if file, err := zf.Open(binfile); err == nil { - // log.Debug("Reading binary compressed chunk from %s", binfile) - bin, err := ioutil.ReadAll(file) - if err != nil { - return nil, err - } - - err = chunk.UnmarshalBinary(bin) - if err != nil { - log.Error("ChunkFromZipfile(%s): %s", coord, err) - return nil, err - } + data, err := io.ReadAll(file) + return ".bin", data, err } else if file, err := zf.Open(jsonfile); err == nil { - // log.Debug("Reading JSON encoded chunk from %s", jsonfile) - bin, err := ioutil.ReadAll(file) - if err != nil { - return nil, err - } - - err = chunk.UnmarshalJSON(bin) - if err != nil { - return nil, err - } - } else { - return nil, err + data, err := io.ReadAll(file) + return ".json", data, err } - return chunk, nil + return "", nil, errors.New("not found in zipfile") } // ChunksInZipfile returns the list of chunk coordinates in a zipfile. diff --git a/pkg/level/rle/rle.go b/pkg/level/rle/rle.go index 2f44875..dfd2230 100644 --- a/pkg/level/rle/rle.go +++ b/pkg/level/rle/rle.go @@ -5,10 +5,8 @@ import ( "bytes" "encoding/binary" "errors" - "fmt" "strings" - "git.kirsle.net/SketchyMaze/doodle/pkg/log" "git.kirsle.net/go/render" ) @@ -25,9 +23,9 @@ func NewGrid(size int) (Grid, error) { return nil, errors.New("no size given for RLE Grid: the chunker was probably not initialized") } - var grid = make([][]*uint64, size+1) - for i := 0; i < size+1; i++ { - grid[i] = make([]*uint64, size+1) + var grid = make([][]*uint64, size) + for i := 0; i < size; i++ { + grid[i] = make([]*uint64, size) } return grid, nil @@ -58,7 +56,7 @@ func (g Grid) Size() int { // - A Uvarint for the palette index (0-255) or 0xffff (65535) for null. // - A Uvarint for how many pixels to repeat that color. func (g Grid) Compress() ([]byte, error) { - log.Error("BEGIN Compress()") + // log.Error("BEGIN Compress()") // log.Warn("Visualized:\n%s", g.Visualize()) // Run-length encode the grid. @@ -120,13 +118,14 @@ func (g Grid) Compress() ([]byte, error) { // Decompress the RLE byte stream back into a populated 2D grid. func (g Grid) Decompress(compressed []byte) error { - log.Error("BEGIN Decompress()") + // log.Error("BEGIN Decompress() Length of stream: %d", len(compressed)) // log.Warn("Visualized:\n%s", g.Visualize()) // Prepare the 2D grid to decompress the RLE stream into. var ( - size = g.Size() - x, y, cursor int + size = g.Size() + x, y = -1, -1 + cursor int ) var reader = bytes.NewBuffer(compressed) @@ -147,22 +146,19 @@ func (g Grid) Decompress(compressed []byte) error { paletteIndex = &paletteIndexRaw } - // log.Warn("RLE index %v for %dpx", paletteIndexRaw, repeatCount) + // log.Warn("RLE index %v for %dpx - coord=%d,%d", paletteIndexRaw, repeatCount, x, y) for i := uint64(0); i < repeatCount; i++ { - cursor++ if cursor%size == 0 { y++ x = 0 } point := render.NewPoint(int(x), int(y)) - if point.Y >= size || point.X >= size { - continue - } g[point.Y][point.X] = paletteIndex x++ + cursor++ } } @@ -180,10 +176,26 @@ func (g Grid) Visualize() string { if col == nil { line += " " } else { - line += fmt.Sprintf("%x", *col) + line += Alphabetize(col) } } lines = append(lines, line+"]") } return strings.Join(lines, "\n") } + +const alphabet = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" + +// Alphabetize converts a palette index value into a single character for +// Visualize to display. +// +// It supports up to 36 palette indexes before it will wrap back around and +// begin reusing symbols. +func Alphabetize(value *uint64) string { + if value == nil { + return " " + } + + var i = int(*value) + return string(alphabet[i%len(alphabet)]) +}