RLE Compression for File Formats #95
29
Changes.md
29
Changes.md
|
@ -1,5 +1,34 @@
|
||||||
# Changes
|
# Changes
|
||||||
|
|
||||||
|
## v0.14.1 (TBD)
|
||||||
|
|
||||||
|
The file format for Levels and Doodads has been optimized to store drawing data
|
||||||
|
with Run Length Encoding (RLE) compression which nets a filesize savings upwards
|
||||||
|
of 90%, especially for levels featuring large areas of solid colors.
|
||||||
|
|
||||||
|
* For example, the Shapeshifter level from the First Quest has shrank from
|
||||||
|
22 MB to only 263 KB.
|
||||||
|
* The complete size of the First Quest levelpack from the previous release of
|
||||||
|
the game shrinks from 50 MB to only 1.8 MB!
|
||||||
|
* The game is still able to load levels and doodads created by previous releases
|
||||||
|
and will automatically convert them into the optimized RLE format when you
|
||||||
|
save them back to disk.
|
||||||
|
* The `doodad resave` command can also optimize your levels and doodads outside
|
||||||
|
of the game's editor.
|
||||||
|
|
||||||
|
Other miscellaneous changes:
|
||||||
|
|
||||||
|
* Command line option `sketchymaze --new` to open the game quickly to a new
|
||||||
|
level in the editor.
|
||||||
|
|
||||||
|
Cleanup of old features and unused code:
|
||||||
|
|
||||||
|
* The game can no longer save any Chunk files in their legacy JSON format: it
|
||||||
|
can still read JSON but all writes will be in the binary chunk format (usually
|
||||||
|
with the new RLE compression). Regular releases of the game have not been
|
||||||
|
writing in the JSON format for a while as it is controlled by hard-coded
|
||||||
|
feature flag constants.
|
||||||
|
|
||||||
## v0.14.0 (May 4 2024)
|
## v0.14.0 (May 4 2024)
|
||||||
|
|
||||||
Level screenshots and thumbnails:
|
Level screenshots and thumbnails:
|
||||||
|
|
|
@ -278,7 +278,7 @@ func (a *MapAccessor) UnmarshalBinary(compressed []byte) error {
|
||||||
defer a.mu.Unlock()
|
defer a.mu.Unlock()
|
||||||
|
|
||||||
// New format: decompress the byte stream.
|
// New format: decompress the byte stream.
|
||||||
log.Debug("MapAccessor.Unmarshal: Reading %d bytes of compressed chunk data", len(compressed))
|
// log.Debug("MapAccessor.Unmarshal: Reading %d bytes of compressed chunk data", len(compressed))
|
||||||
|
|
||||||
var reader = bytes.NewBuffer(compressed)
|
var reader = bytes.NewBuffer(compressed)
|
||||||
|
|
||||||
|
|
|
@ -2,7 +2,6 @@ package level
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"git.kirsle.net/SketchyMaze/doodle/pkg/level/rle"
|
"git.kirsle.net/SketchyMaze/doodle/pkg/level/rle"
|
||||||
"git.kirsle.net/SketchyMaze/doodle/pkg/log"
|
|
||||||
"git.kirsle.net/go/render"
|
"git.kirsle.net/go/render"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -63,7 +62,7 @@ This accessor uses Run Length Encoding (RLE) in its binary format. Starting
|
||||||
with the top-left pixel of this chunk, the binary format is a stream of bytes
|
with the top-left pixel of this chunk, the binary format is a stream of bytes
|
||||||
formatted as such:
|
formatted as such:
|
||||||
|
|
||||||
- UVarint for the palette index number (0-255), with 0xFF meaning void
|
- UVarint for the palette index number (0-255), with 0xFFFF meaning void
|
||||||
- UVarint for the length of repetition of that palette index
|
- UVarint for the length of repetition of that palette index
|
||||||
*/
|
*/
|
||||||
func (a *RLEAccessor) MarshalBinary() ([]byte, error) {
|
func (a *RLEAccessor) MarshalBinary() ([]byte, error) {
|
||||||
|
@ -103,7 +102,7 @@ func (a *RLEAccessor) UnmarshalBinary(compressed []byte) error {
|
||||||
defer a.acc.mu.Unlock()
|
defer a.acc.mu.Unlock()
|
||||||
|
|
||||||
// New format: decompress the byte stream.
|
// New format: decompress the byte stream.
|
||||||
log.Debug("RLEAccessor.Unmarshal: Reading %d bytes of compressed chunk data", len(compressed))
|
// log.Debug("RLEAccessor.Unmarshal: Reading %d bytes of compressed chunk data", len(compressed))
|
||||||
|
|
||||||
grid, err := rle.NewGrid(int(a.chunk.Size))
|
grid, err := rle.NewGrid(int(a.chunk.Size))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -129,46 +128,3 @@ func (a *RLEAccessor) UnmarshalBinary(compressed []byte) error {
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
|
||||||
// Prepare the 2D grid to decompress the RLE stream into.
|
|
||||||
var (
|
|
||||||
size = int(a.chunk.Size)
|
|
||||||
_, err = rle.NewGrid(size)
|
|
||||||
x, y, cursor int
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
var reader = bytes.NewBuffer(compressed)
|
|
||||||
|
|
||||||
for {
|
|
||||||
var (
|
|
||||||
paletteIndex, err1 = binary.ReadUvarint(reader)
|
|
||||||
repeatCount, err2 = binary.ReadUvarint(reader)
|
|
||||||
)
|
|
||||||
|
|
||||||
if err1 != nil || err2 != nil {
|
|
||||||
log.Error("reading Uvarints from compressed data: {%s, %s}", err1, err2)
|
|
||||||
break
|
|
||||||
}
|
|
||||||
|
|
||||||
log.Warn("RLE index %d for %dpx", paletteIndex, repeatCount)
|
|
||||||
|
|
||||||
for i := uint64(0); i < repeatCount; i++ {
|
|
||||||
cursor++
|
|
||||||
if cursor%size == 0 {
|
|
||||||
y++
|
|
||||||
x = 0
|
|
||||||
} else {
|
|
||||||
x++
|
|
||||||
}
|
|
||||||
|
|
||||||
point := render.NewPoint(int(x), int(y))
|
|
||||||
if paletteIndex != 0xFF {
|
|
||||||
a.acc.grid[point] = NewSparseSwatch(int(paletteIndex))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
*/
|
|
||||||
|
|
|
@ -627,11 +627,6 @@ func RelativeCoordinate(abs render.Point, chunkCoord render.Point, chunkSize uin
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
if point.X < 0 || point.Y < 0 {
|
|
||||||
log.Error("RelativeCoordinate: X < 0! abs=%s rel=%s chunk=%s size=%d", abs, point, chunkCoord, chunkSize)
|
|
||||||
log.Error("RelativeCoordinate(2): size=%d offset=%s point=%s", size, offset, point)
|
|
||||||
}
|
|
||||||
|
|
||||||
return point
|
return point
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -14,9 +14,7 @@ import (
|
||||||
// and possibly migrate them to a different Accessor implementation when
|
// and possibly migrate them to a different Accessor implementation when
|
||||||
// saving on disk.
|
// saving on disk.
|
||||||
func (c *Chunker) OptimizeChunkerAccessors() {
|
func (c *Chunker) OptimizeChunkerAccessors() {
|
||||||
log.Info("Optimizing Chunker Accessors")
|
// Parallelize this with goroutines.
|
||||||
|
|
||||||
// TODO: parallelize this with goroutines
|
|
||||||
var (
|
var (
|
||||||
chunks = make(chan *Chunk, len(c.Chunks))
|
chunks = make(chan *Chunk, len(c.Chunks))
|
||||||
wg sync.WaitGroup
|
wg sync.WaitGroup
|
||||||
|
@ -58,7 +56,6 @@ func (c *Chunker) OptimizeChunkerAccessors() {
|
||||||
|
|
||||||
close(chunks)
|
close(chunks)
|
||||||
wg.Wait()
|
wg.Wait()
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// FromMapAccessor migrates from a MapAccessor to RLE.
|
// FromMapAccessor migrates from a MapAccessor to RLE.
|
||||||
|
|
Loading…
Reference in New Issue
Block a user