Fix RLE Encoding Off-by-One Errors [PTO]

Levels can now be converted to RLE encoded chunk accessors and be re-saved
continuously without any loss of information.

Off-by-one errors resolved:

* The rle.NewGrid() was adding a +1 everywhere making the 2D grids have 129
  elements to a side for a 128 chunk size.
* In rle.Decompress() the cursor value and translation to X,Y coordinates is
  fixed to avoid a pixel going missing at the end of the first row (128,0)
* The abs.X-- hack in UnmarshalBinary is no longer needed to prevent the
  chunks from scooting a pixel to the right on every save.

Doodad tool updates:

* Remove unused CLI flags in `doodad resave` (actors, chunks, script,
  attachment, verbose) and add a `--output` flag to save to a different file
  name to the original.
* Update `doodad show` to allow debugging of RLE compressed chunks:
    * CLI flag `--chunk=1,2` to specify a single chunk coordinate to debug
    * CLI flag `--visualize-rle` will Visualize() RLE compressed chunks in
      their 2D grid form in your terminal window (VERY noisy for large
      levels! Use the --chunk option to narrow to one chunk).

Bug fixes and misc changes:

* Chunk.Usage() to return a better percentage of chunk utilization.
* Chunker.ChunkFromZipfile() was split out into two functions:
    * RawChunkFromZipfile retrieves the raw bytes of the chunk as well as the
      file extension discovered (.bin or .json) so the caller can interpret
      the bytes correctly.
    * ChunkFromZipfile calls the former function and then depending on file
      extension, unmarshals from binary or json.
    * The Raw function enables the `doodad show` command to debug and visualize
      the raw contents of the RLE compressed chunks.
* Updated the Visualize() function for the RLE encoder: instead of converting
  palette indexes to hex (0-F) which would begin causing problems for palette
  indexes above 16 (as they would use two+ characters), indexes are mapped to
  a wider range of symbols (0-9A-Z) and roll over if you have more than 36
  colors on your level. This at least keeps the Visualize() grid an easy to
  read 128x128 characters in your terminal.
This commit is contained in:
Noah 2024-05-24 13:54:41 -07:00
parent 5654145fd8
commit 4851730ccf
7 changed files with 167 additions and 85 deletions

View File

@ -21,27 +21,10 @@ func init() {
Usage: "load and re-save a level or doodad file to migrate to newer file format versions", Usage: "load and re-save a level or doodad file to migrate to newer file format versions",
ArgsUsage: "<.level or .doodad>", ArgsUsage: "<.level or .doodad>",
Flags: []cli.Flag{ Flags: []cli.Flag{
&cli.BoolFlag{
Name: "actors",
Usage: "print verbose actor data in Level files",
},
&cli.BoolFlag{
Name: "chunks",
Usage: "print verbose data about all the pixel chunks in a file",
},
&cli.BoolFlag{
Name: "script",
Usage: "print the script from a doodad file and exit",
},
&cli.StringFlag{ &cli.StringFlag{
Name: "attachment", Name: "output",
Aliases: []string{"a"}, Aliases: []string{"o"},
Usage: "print the contents of the attached filename to terminal", Usage: "write to a different file than the input",
},
&cli.BoolFlag{
Name: "verbose",
Aliases: []string{"v"},
Usage: "print verbose output (all verbose flags enabled)",
}, },
}, },
Action: func(c *cli.Context) error { Action: func(c *cli.Context) error {
@ -84,6 +67,18 @@ func resaveLevel(c *cli.Context, filename string) error {
log.Info("Loaded level from file: %s", filename) log.Info("Loaded level from file: %s", filename)
log.Info("Last saved game version: %s", lvl.GameVersion) log.Info("Last saved game version: %s", lvl.GameVersion)
// Different output filename?
if output := c.String("output"); output != "" {
log.Info("Output will be saved to: %s", output)
filename = output
}
if err := lvl.Vacuum(); err != nil {
log.Error("Vacuum error: %s", err)
} else {
log.Info("Run vacuum on level file.")
}
log.Info("Saving back to disk") log.Info("Saving back to disk")
if err := lvl.WriteJSON(filename); err != nil { if err := lvl.WriteJSON(filename); err != nil {
return fmt.Errorf("couldn't write %s: %s", filename, err) return fmt.Errorf("couldn't write %s: %s", filename, err)
@ -100,6 +95,12 @@ func resaveDoodad(c *cli.Context, filename string) error {
log.Info("Loaded doodad from file: %s", filename) log.Info("Loaded doodad from file: %s", filename)
log.Info("Last saved game version: %s", dd.GameVersion) log.Info("Last saved game version: %s", dd.GameVersion)
// Different output filename?
if output := c.String("output"); output != "" {
log.Info("Output will be saved to: %s", output)
filename = output
}
log.Info("Saving back to disk") log.Info("Saving back to disk")
if err := dd.WriteJSON(filename); err != nil { if err := dd.WriteJSON(filename); err != nil {
return fmt.Errorf("couldn't write %s: %s", filename, err) return fmt.Errorf("couldn't write %s: %s", filename, err)

View File

@ -1,6 +1,8 @@
package commands package commands
import ( import (
"bytes"
"encoding/binary"
"fmt" "fmt"
"path/filepath" "path/filepath"
"sort" "sort"
@ -9,6 +11,7 @@ import (
"git.kirsle.net/SketchyMaze/doodle/pkg/doodads" "git.kirsle.net/SketchyMaze/doodle/pkg/doodads"
"git.kirsle.net/SketchyMaze/doodle/pkg/enum" "git.kirsle.net/SketchyMaze/doodle/pkg/enum"
"git.kirsle.net/SketchyMaze/doodle/pkg/level" "git.kirsle.net/SketchyMaze/doodle/pkg/level"
"git.kirsle.net/SketchyMaze/doodle/pkg/level/rle"
"git.kirsle.net/SketchyMaze/doodle/pkg/log" "git.kirsle.net/SketchyMaze/doodle/pkg/log"
"github.com/urfave/cli/v2" "github.com/urfave/cli/v2"
) )
@ -44,6 +47,14 @@ func init() {
Aliases: []string{"v"}, Aliases: []string{"v"},
Usage: "print verbose output (all verbose flags enabled)", Usage: "print verbose output (all verbose flags enabled)",
}, },
&cli.BoolFlag{
Name: "visualize-rle",
Usage: "visually dump RLE encoded chunks to the terminal (VERY noisy for large drawings!)",
},
&cli.StringFlag{
Name: "chunk",
Usage: "specific chunk coordinate; when debugging chunks, only show this chunk (example: 2,-1)",
},
}, },
Action: func(c *cli.Context) error { Action: func(c *cli.Context) error {
if c.NArg() < 1 { if c.NArg() < 1 {
@ -263,6 +274,10 @@ func showChunker(c *cli.Context, ch *level.Chunker) {
chunkSize = int(ch.Size) chunkSize = int(ch.Size)
width = worldSize.W - worldSize.X width = worldSize.W - worldSize.X
height = worldSize.H - worldSize.Y height = worldSize.H - worldSize.Y
// Chunk debugging CLI options.
visualize = c.Bool("visualize-rle")
specificChunk = c.String("chunk")
) )
fmt.Println("Chunks:") fmt.Println("Chunks:")
fmt.Printf(" Pixels Per Chunk: %d^2\n", ch.Size) fmt.Printf(" Pixels Per Chunk: %d^2\n", ch.Size)
@ -278,7 +293,18 @@ func showChunker(c *cli.Context, ch *level.Chunker) {
// Verbose chunk information. // Verbose chunk information.
if c.Bool("chunks") || c.Bool("verbose") { if c.Bool("chunks") || c.Bool("verbose") {
fmt.Println(" Chunk Details:") fmt.Println(" Chunk Details:")
for point, chunk := range ch.Chunks { for point := range ch.IterChunks() {
// Debugging specific chunk coordinate?
if specificChunk != "" && point.String() != specificChunk {
log.Warn("Skip chunk %s: not the specific chunk you're looking for", point)
continue
}
chunk, ok := ch.GetChunk(point)
if !ok {
continue
}
fmt.Printf(" - Coord: %s\n", point) fmt.Printf(" - Coord: %s\n", point)
fmt.Printf(" Type: %s\n", chunkTypeToName(chunk.Type)) fmt.Printf(" Type: %s\n", chunkTypeToName(chunk.Type))
fmt.Printf(" Range: (%d,%d) ... (%d,%d)\n", fmt.Printf(" Range: (%d,%d) ... (%d,%d)\n",
@ -287,6 +313,33 @@ func showChunker(c *cli.Context, ch *level.Chunker) {
(int(point.X)*chunkSize)+chunkSize, (int(point.X)*chunkSize)+chunkSize,
(int(point.Y)*chunkSize)+chunkSize, (int(point.Y)*chunkSize)+chunkSize,
) )
fmt.Printf(" Usage: %f (%d len of %d)\n", chunk.Usage(), chunk.Len(), chunkSize*chunkSize)
// Visualize the RLE encoded chunks?
if visualize && chunk.Type == level.RLEType {
ext, bin, err := ch.RawChunkFromZipfile(point)
if err != nil {
log.Error(err.Error())
continue
} else if ext != ".bin" {
log.Error("Unexpected filetype for RLE compressed chunk (expected .bin, got %s)", ext)
continue
}
// Read off the first byte (chunk type)
var reader = bytes.NewBuffer(bin)
binary.ReadUvarint(reader)
bin = reader.Bytes()
grid, err := rle.NewGrid(chunkSize)
if err != nil {
log.Error(err.Error())
continue
}
grid.Decompress(bin)
fmt.Println(grid.Visualize())
}
} }
} else { } else {
fmt.Println(" Use -chunks or -verbose to serialize Chunks") fmt.Println(" Use -chunks or -verbose to serialize Chunks")
@ -298,6 +351,8 @@ func chunkTypeToName(v uint64) string {
switch v { switch v {
case level.MapType: case level.MapType:
return "map" return "map"
case level.RLEType:
return "rle map"
case level.GridType: case level.GridType:
return "grid" return "grid"
default: default:

View File

@ -330,8 +330,9 @@ func (c *Chunk) SizePositive() render.Rect {
} }
// Usage returns the percent of free space vs. allocated pixels in the chunk. // Usage returns the percent of free space vs. allocated pixels in the chunk.
func (c *Chunk) Usage(size int) float64 { func (c *Chunk) Usage() float64 {
return float64(c.Len()) / float64(size) size := float64(c.Size)
return float64(c.Len()) / (size * size)
} }
// UnmarshalJSON loads the chunk from JSON and uses the correct accessor to // UnmarshalJSON loads the chunk from JSON and uses the correct accessor to

View File

@ -77,19 +77,22 @@ func (a *RLEAccessor) MarshalBinary() ([]byte, error) {
} }
// Populate the dense 2D array of its pixels. // Populate the dense 2D array of its pixels.
for px := range a.Iter() { for y, row := range grid {
for x := range row {
var ( var (
point = render.NewPoint(px.X, px.Y) relative = render.NewPoint(x, y)
relative = RelativeCoordinate(point, a.chunk.Point, a.chunk.Size) absolute = FromRelativeCoordinate(relative, a.chunk.Point, a.chunk.Size)
ptr = uint64(px.Swatch.Index()) swatch, err = a.Get(absolute)
) )
// TODO: sometimes we get a -1 value in X or Y, not sure why. if err != nil {
if relative.X < 0 || relative.Y < 0 {
continue continue
} }
var ptr = uint64(swatch.Index())
grid[relative.Y][relative.X] = &ptr grid[relative.Y][relative.X] = &ptr
} }
}
return grid.Compress() return grid.Compress()
} }
@ -119,10 +122,7 @@ func (a *RLEAccessor) UnmarshalBinary(compressed []byte) error {
continue continue
} }
// TODO: x-1 to avoid the level creeping to the right every save,
// not sure on the root cause! RLEAccessor Decompress?
abs := FromRelativeCoordinate(render.NewPoint(x, y), a.chunk.Point, a.chunk.Size) abs := FromRelativeCoordinate(render.NewPoint(x, y), a.chunk.Point, a.chunk.Size)
abs.X -= 1
a.acc.grid[abs] = NewSparseSwatch(int(*col)) a.acc.grid[abs] = NewSparseSwatch(int(*col))
} }
} }

View File

@ -14,9 +14,6 @@ import (
// and possibly migrate them to a different Accessor implementation when // and possibly migrate them to a different Accessor implementation when
// saving on disk. // saving on disk.
func (c *Chunker) OptimizeChunkerAccessors() { func (c *Chunker) OptimizeChunkerAccessors() {
c.chunkMu.Lock()
defer c.chunkMu.Unlock()
log.Info("Optimizing Chunker Accessors") log.Info("Optimizing Chunker Accessors")
// TODO: parallelize this with goroutines // TODO: parallelize this with goroutines
@ -31,7 +28,6 @@ func (c *Chunker) OptimizeChunkerAccessors() {
defer wg.Done() defer wg.Done()
for chunk := range chunks { for chunk := range chunks {
var point = chunk.Point var point = chunk.Point
log.Warn("Chunk %s is a: %d", point, chunk.Type)
// Upgrade all MapTypes into RLE compressed MapTypes? // Upgrade all MapTypes into RLE compressed MapTypes?
if balance.RLEBinaryChunkerEnabled { if balance.RLEBinaryChunkerEnabled {
@ -49,7 +45,11 @@ func (c *Chunker) OptimizeChunkerAccessors() {
} }
// Feed it the chunks. // Feed it the chunks.
for _, chunk := range c.Chunks { for point := range c.IterChunks() {
chunk, ok := c.GetChunk(point)
if !ok {
continue
}
chunks <- chunk chunks <- chunk
} }

View File

@ -4,7 +4,7 @@ import (
"archive/zip" "archive/zip"
"errors" "errors"
"fmt" "fmt"
"io/ioutil" "io"
"regexp" "regexp"
"strconv" "strconv"
@ -206,6 +206,42 @@ func (c *Chunk) ToZipfile(zf *zip.Writer, layer int, coord render.Point) error {
// ChunkFromZipfile loads a chunk from a zipfile. // ChunkFromZipfile loads a chunk from a zipfile.
func (c *Chunker) ChunkFromZipfile(coord render.Point) (*Chunk, error) { func (c *Chunker) ChunkFromZipfile(coord render.Point) (*Chunk, error) {
// Grab the chunk (bin or json) from the Zipfile.
ext, bin, err := c.RawChunkFromZipfile(coord)
if err != nil {
return nil, err
}
var chunk = NewChunk()
chunk.Point = coord
chunk.Size = c.Size
switch ext {
case ".bin":
// New style .bin compressed format:
// Either a MapAccessor compressed bin, or RLE compressed.
err = chunk.UnmarshalBinary(bin)
if err != nil {
log.Error("ChunkFromZipfile(%s): %s", coord, err)
return nil, err
}
case ".json":
// Legacy style plain .json file (MapAccessor only).
err = chunk.UnmarshalJSON(bin)
if err != nil {
return nil, err
}
default:
return nil, fmt.Errorf("unexpected filetype found for this chunk: %s", ext)
}
return chunk, nil
}
// RawChunkFromZipfile loads a chunk from a zipfile and returns its raw binary content.
//
// Returns the file extension (".bin" or ".json"), raw bytes, and an error.
func (c *Chunker) RawChunkFromZipfile(coord render.Point) (string, []byte, error) {
// File names? // File names?
var ( var (
zf = c.Zipfile zf = c.Zipfile
@ -213,41 +249,18 @@ func (c *Chunker) ChunkFromZipfile(coord render.Point) (*Chunk, error) {
binfile = fmt.Sprintf("chunks/%d/%s.bin", layer, coord) binfile = fmt.Sprintf("chunks/%d/%s.bin", layer, coord)
jsonfile = fmt.Sprintf("chunks/%d/%s.json", layer, coord) jsonfile = fmt.Sprintf("chunks/%d/%s.json", layer, coord)
chunk = NewChunk()
) )
chunk.Point = coord
chunk.Size = c.Size
// Read from the new binary format. // Read from the new binary format.
if file, err := zf.Open(binfile); err == nil { if file, err := zf.Open(binfile); err == nil {
// log.Debug("Reading binary compressed chunk from %s", binfile) data, err := io.ReadAll(file)
bin, err := ioutil.ReadAll(file) return ".bin", data, err
if err != nil {
return nil, err
}
err = chunk.UnmarshalBinary(bin)
if err != nil {
log.Error("ChunkFromZipfile(%s): %s", coord, err)
return nil, err
}
} else if file, err := zf.Open(jsonfile); err == nil { } else if file, err := zf.Open(jsonfile); err == nil {
// log.Debug("Reading JSON encoded chunk from %s", jsonfile) data, err := io.ReadAll(file)
bin, err := ioutil.ReadAll(file) return ".json", data, err
if err != nil {
return nil, err
} }
err = chunk.UnmarshalJSON(bin) return "", nil, errors.New("not found in zipfile")
if err != nil {
return nil, err
}
} else {
return nil, err
}
return chunk, nil
} }
// ChunksInZipfile returns the list of chunk coordinates in a zipfile. // ChunksInZipfile returns the list of chunk coordinates in a zipfile.

View File

@ -5,10 +5,8 @@ import (
"bytes" "bytes"
"encoding/binary" "encoding/binary"
"errors" "errors"
"fmt"
"strings" "strings"
"git.kirsle.net/SketchyMaze/doodle/pkg/log"
"git.kirsle.net/go/render" "git.kirsle.net/go/render"
) )
@ -25,9 +23,9 @@ func NewGrid(size int) (Grid, error) {
return nil, errors.New("no size given for RLE Grid: the chunker was probably not initialized") return nil, errors.New("no size given for RLE Grid: the chunker was probably not initialized")
} }
var grid = make([][]*uint64, size+1) var grid = make([][]*uint64, size)
for i := 0; i < size+1; i++ { for i := 0; i < size; i++ {
grid[i] = make([]*uint64, size+1) grid[i] = make([]*uint64, size)
} }
return grid, nil return grid, nil
@ -58,7 +56,7 @@ func (g Grid) Size() int {
// - A Uvarint for the palette index (0-255) or 0xffff (65535) for null. // - A Uvarint for the palette index (0-255) or 0xffff (65535) for null.
// - A Uvarint for how many pixels to repeat that color. // - A Uvarint for how many pixels to repeat that color.
func (g Grid) Compress() ([]byte, error) { func (g Grid) Compress() ([]byte, error) {
log.Error("BEGIN Compress()") // log.Error("BEGIN Compress()")
// log.Warn("Visualized:\n%s", g.Visualize()) // log.Warn("Visualized:\n%s", g.Visualize())
// Run-length encode the grid. // Run-length encode the grid.
@ -120,13 +118,14 @@ func (g Grid) Compress() ([]byte, error) {
// Decompress the RLE byte stream back into a populated 2D grid. // Decompress the RLE byte stream back into a populated 2D grid.
func (g Grid) Decompress(compressed []byte) error { func (g Grid) Decompress(compressed []byte) error {
log.Error("BEGIN Decompress()") // log.Error("BEGIN Decompress() Length of stream: %d", len(compressed))
// log.Warn("Visualized:\n%s", g.Visualize()) // log.Warn("Visualized:\n%s", g.Visualize())
// Prepare the 2D grid to decompress the RLE stream into. // Prepare the 2D grid to decompress the RLE stream into.
var ( var (
size = g.Size() size = g.Size()
x, y, cursor int x, y = -1, -1
cursor int
) )
var reader = bytes.NewBuffer(compressed) var reader = bytes.NewBuffer(compressed)
@ -147,22 +146,19 @@ func (g Grid) Decompress(compressed []byte) error {
paletteIndex = &paletteIndexRaw paletteIndex = &paletteIndexRaw
} }
// log.Warn("RLE index %v for %dpx", paletteIndexRaw, repeatCount) // log.Warn("RLE index %v for %dpx - coord=%d,%d", paletteIndexRaw, repeatCount, x, y)
for i := uint64(0); i < repeatCount; i++ { for i := uint64(0); i < repeatCount; i++ {
cursor++
if cursor%size == 0 { if cursor%size == 0 {
y++ y++
x = 0 x = 0
} }
point := render.NewPoint(int(x), int(y)) point := render.NewPoint(int(x), int(y))
if point.Y >= size || point.X >= size {
continue
}
g[point.Y][point.X] = paletteIndex g[point.Y][point.X] = paletteIndex
x++ x++
cursor++
} }
} }
@ -180,10 +176,26 @@ func (g Grid) Visualize() string {
if col == nil { if col == nil {
line += " " line += " "
} else { } else {
line += fmt.Sprintf("%x", *col) line += Alphabetize(col)
} }
} }
lines = append(lines, line+"]") lines = append(lines, line+"]")
} }
return strings.Join(lines, "\n") return strings.Join(lines, "\n")
} }
const alphabet = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
// Alphabetize converts a palette index value into a single character for
// Visualize to display.
//
// It supports up to 36 palette indexes before it will wrap back around and
// begin reusing symbols.
func Alphabetize(value *uint64) string {
if value == nil {
return " "
}
var i = int(*value)
return string(alphabet[i%len(alphabet)])
}