RLE Compression for File Formats #95
|
@ -101,6 +101,11 @@ func main() {
|
||||||
Name: "chdir",
|
Name: "chdir",
|
||||||
Usage: "working directory for the game's runtime package",
|
Usage: "working directory for the game's runtime package",
|
||||||
},
|
},
|
||||||
|
&cli.BoolFlag{
|
||||||
|
Name: "new",
|
||||||
|
Aliases: []string{"n"},
|
||||||
|
Usage: "open immediately to the level editor",
|
||||||
|
},
|
||||||
&cli.BoolFlag{
|
&cli.BoolFlag{
|
||||||
Name: "edit",
|
Name: "edit",
|
||||||
Aliases: []string{"e"},
|
Aliases: []string{"e"},
|
||||||
|
@ -248,6 +253,8 @@ func main() {
|
||||||
|
|
||||||
if c.Bool("guitest") {
|
if c.Bool("guitest") {
|
||||||
game.Goto(&doodle.GUITestScene{})
|
game.Goto(&doodle.GUITestScene{})
|
||||||
|
} else if c.Bool("new") {
|
||||||
|
game.NewMap()
|
||||||
} else if filename != "" {
|
} else if filename != "" {
|
||||||
if c.Bool("edit") {
|
if c.Bool("edit") {
|
||||||
game.EditFile(filename)
|
game.EditFile(filename)
|
||||||
|
|
|
@ -28,6 +28,12 @@ const (
|
||||||
// If you set both flags to false, level zipfiles will use the classic
|
// If you set both flags to false, level zipfiles will use the classic
|
||||||
// json chunk format as before on save.
|
// json chunk format as before on save.
|
||||||
BinaryChunkerEnabled = true
|
BinaryChunkerEnabled = true
|
||||||
|
|
||||||
|
// Enable "v3" Run-Length Encoding for level chunker.
|
||||||
|
//
|
||||||
|
// This only supports Zipfile levels and will use the ".bin" format
|
||||||
|
// enabled by the previous setting.
|
||||||
|
RLEBinaryChunkerEnabled = true
|
||||||
)
|
)
|
||||||
|
|
||||||
// Feature Flags to turn on/off experimental content.
|
// Feature Flags to turn on/off experimental content.
|
||||||
|
|
|
@ -24,6 +24,9 @@ const (
|
||||||
GridType
|
GridType
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// Default chunk type for newly created chunks (was MapType).
|
||||||
|
const DefaultChunkType = RLEType
|
||||||
|
|
||||||
// Chunk holds a single portion of the pixel canvas.
|
// Chunk holds a single portion of the pixel canvas.
|
||||||
type Chunk struct {
|
type Chunk struct {
|
||||||
Type uint64 // map vs. 2D array.
|
Type uint64 // map vs. 2D array.
|
||||||
|
@ -55,7 +58,6 @@ type JSONChunk struct {
|
||||||
// Accessor provides a high-level API to interact with absolute pixel coordinates
|
// Accessor provides a high-level API to interact with absolute pixel coordinates
|
||||||
// while abstracting away the details of how they're stored.
|
// while abstracting away the details of how they're stored.
|
||||||
type Accessor interface {
|
type Accessor interface {
|
||||||
SetChunkCoordinate(render.Point, uint8)
|
|
||||||
Inflate(*Palette) error
|
Inflate(*Palette) error
|
||||||
Iter() <-chan Pixel
|
Iter() <-chan Pixel
|
||||||
IterViewport(viewport render.Rect) <-chan Pixel
|
IterViewport(viewport render.Rect) <-chan Pixel
|
||||||
|
@ -69,10 +71,11 @@ type Accessor interface {
|
||||||
|
|
||||||
// NewChunk creates a new chunk.
|
// NewChunk creates a new chunk.
|
||||||
func NewChunk() *Chunk {
|
func NewChunk() *Chunk {
|
||||||
return &Chunk{
|
var c = &Chunk{
|
||||||
Type: RLEType,
|
Type: RLEType,
|
||||||
Accessor: NewRLEAccessor(),
|
|
||||||
}
|
}
|
||||||
|
c.Accessor = NewRLEAccessor(c)
|
||||||
|
return c
|
||||||
}
|
}
|
||||||
|
|
||||||
// Texture will return a cached texture for the rendering engine for this
|
// Texture will return a cached texture for the rendering engine for this
|
||||||
|
@ -335,6 +338,9 @@ func (c *Chunk) Usage(size int) float64 {
|
||||||
// parse the inner details.
|
// parse the inner details.
|
||||||
//
|
//
|
||||||
// DEPRECATED in favor of binary marshalling.
|
// DEPRECATED in favor of binary marshalling.
|
||||||
|
//
|
||||||
|
// Only supports MapAccessor chunk types, which was the only one supported
|
||||||
|
// before this function was deprecated.
|
||||||
func (c *Chunk) UnmarshalJSON(b []byte) error {
|
func (c *Chunk) UnmarshalJSON(b []byte) error {
|
||||||
// Parse it generically so we can hand off the inner "data" object to the
|
// Parse it generically so we can hand off the inner "data" object to the
|
||||||
// right accessor for unmarshalling.
|
// right accessor for unmarshalling.
|
||||||
|
@ -346,7 +352,7 @@ func (c *Chunk) UnmarshalJSON(b []byte) error {
|
||||||
|
|
||||||
switch c.Type {
|
switch c.Type {
|
||||||
case MapType:
|
case MapType:
|
||||||
c.Accessor = NewMapAccessor()
|
c.Accessor = NewMapAccessor(c)
|
||||||
if unmarshaler, ok := c.Accessor.(json.Unmarshaler); ok {
|
if unmarshaler, ok := c.Accessor.(json.Unmarshaler); ok {
|
||||||
return unmarshaler.UnmarshalJSON(generic.Data)
|
return unmarshaler.UnmarshalJSON(generic.Data)
|
||||||
}
|
}
|
||||||
|
@ -393,12 +399,12 @@ func (c *Chunk) UnmarshalBinary(b []byte) error {
|
||||||
// Decode the rest of the byte stream.
|
// Decode the rest of the byte stream.
|
||||||
switch chunkType {
|
switch chunkType {
|
||||||
case MapType:
|
case MapType:
|
||||||
c.Accessor = NewMapAccessor()
|
c.Type = MapType
|
||||||
c.Accessor.SetChunkCoordinate(c.Point, c.Size)
|
c.Accessor = NewMapAccessor(c)
|
||||||
return c.Accessor.UnmarshalBinary(reader.Bytes())
|
return c.Accessor.UnmarshalBinary(reader.Bytes())
|
||||||
case RLEType:
|
case RLEType:
|
||||||
c.Accessor = NewRLEAccessor()
|
c.Type = RLEType
|
||||||
c.Accessor.SetChunkCoordinate(c.Point, c.Size)
|
c.Accessor = NewRLEAccessor(c)
|
||||||
return c.Accessor.UnmarshalBinary(reader.Bytes())
|
return c.Accessor.UnmarshalBinary(reader.Bytes())
|
||||||
default:
|
default:
|
||||||
return fmt.Errorf("Chunk.UnmarshalJSON: unsupported chunk type '%d'", c.Type)
|
return fmt.Errorf("Chunk.UnmarshalJSON: unsupported chunk type '%d'", c.Type)
|
||||||
|
|
|
@ -16,23 +16,22 @@ import (
|
||||||
// MapAccessor implements a chunk accessor by using a map of points to their
|
// MapAccessor implements a chunk accessor by using a map of points to their
|
||||||
// palette indexes. This is the simplest accessor and is best for sparse chunks.
|
// palette indexes. This is the simplest accessor and is best for sparse chunks.
|
||||||
type MapAccessor struct {
|
type MapAccessor struct {
|
||||||
coord render.Point `json:"-"` // chunk coordinate, assigned by Chunker
|
chunk *Chunk // Pointer to parent struct, for its Size and Point
|
||||||
size uint8 `json:"-"` // chunk size, assigned by Chunker
|
|
||||||
grid map[render.Point]*Swatch
|
grid map[render.Point]*Swatch
|
||||||
mu sync.RWMutex
|
mu sync.RWMutex
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewMapAccessor initializes a MapAccessor.
|
// NewMapAccessor initializes a MapAccessor.
|
||||||
func NewMapAccessor() *MapAccessor {
|
func NewMapAccessor(chunk *Chunk) *MapAccessor {
|
||||||
return &MapAccessor{
|
return &MapAccessor{
|
||||||
|
chunk: chunk,
|
||||||
grid: map[render.Point]*Swatch{},
|
grid: map[render.Point]*Swatch{},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// SetChunkCoordinate receives our chunk's coordinate from the Chunker.
|
// Reset the MapAccessor.
|
||||||
func (a *MapAccessor) SetChunkCoordinate(p render.Point, size uint8) {
|
func (a *MapAccessor) Reset() {
|
||||||
a.coord = p
|
a.grid = map[render.Point]*Swatch{}
|
||||||
a.size = size
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Inflate the sparse swatches from their palette indexes.
|
// Inflate the sparse swatches from their palette indexes.
|
||||||
|
|
|
@ -1,10 +1,7 @@
|
||||||
package level
|
package level
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"git.kirsle.net/SketchyMaze/doodle/pkg/level/rle"
|
||||||
"encoding/binary"
|
|
||||||
"errors"
|
|
||||||
|
|
||||||
"git.kirsle.net/SketchyMaze/doodle/pkg/log"
|
"git.kirsle.net/SketchyMaze/doodle/pkg/log"
|
||||||
"git.kirsle.net/go/render"
|
"git.kirsle.net/go/render"
|
||||||
)
|
)
|
||||||
|
@ -12,22 +9,18 @@ import (
|
||||||
// RLEAccessor implements a chunk accessor which stores its on-disk format using
|
// RLEAccessor implements a chunk accessor which stores its on-disk format using
|
||||||
// Run Length Encoding (RLE), but in memory behaves equivalently to the MapAccessor.
|
// Run Length Encoding (RLE), but in memory behaves equivalently to the MapAccessor.
|
||||||
type RLEAccessor struct {
|
type RLEAccessor struct {
|
||||||
|
chunk *Chunk // parent Chunk, for its Size and Point
|
||||||
acc *MapAccessor
|
acc *MapAccessor
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewRLEAccessor initializes a RLEAccessor.
|
// NewRLEAccessor initializes a RLEAccessor.
|
||||||
func NewRLEAccessor() *RLEAccessor {
|
func NewRLEAccessor(chunk *Chunk) *RLEAccessor {
|
||||||
return &RLEAccessor{
|
return &RLEAccessor{
|
||||||
acc: NewMapAccessor(),
|
chunk: chunk,
|
||||||
|
acc: NewMapAccessor(chunk),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// SetChunkCoordinate receives our chunk's coordinate from the Chunker.
|
|
||||||
func (a *RLEAccessor) SetChunkCoordinate(p render.Point, size uint8) {
|
|
||||||
a.acc.coord = p
|
|
||||||
a.acc.size = size
|
|
||||||
}
|
|
||||||
|
|
||||||
// Inflate the sparse swatches from their palette indexes.
|
// Inflate the sparse swatches from their palette indexes.
|
||||||
func (a *RLEAccessor) Inflate(pal *Palette) error {
|
func (a *RLEAccessor) Inflate(pal *Palette) error {
|
||||||
return a.acc.Inflate(pal)
|
return a.acc.Inflate(pal)
|
||||||
|
@ -63,23 +56,6 @@ func (a *RLEAccessor) Delete(p render.Point) error {
|
||||||
return a.acc.Delete(p)
|
return a.acc.Delete(p)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Make2DChunkGrid creates a 2D map of uint64 pointers matching the square dimensions of the given size.
|
|
||||||
//
|
|
||||||
// It is used by the RLEAccessor to flatten a chunk into a grid for run-length encoding.
|
|
||||||
func Make2DChunkGrid(size int) ([][]*uint64, error) {
|
|
||||||
// Sanity check if the chunk was properly initialized.
|
|
||||||
if size == 0 {
|
|
||||||
return nil, errors.New("chunk not initialized correctly with its size and coordinate")
|
|
||||||
}
|
|
||||||
|
|
||||||
var grid = make([][]*uint64, size)
|
|
||||||
for i := 0; i < size; i++ {
|
|
||||||
grid[i] = make([]*uint64, size)
|
|
||||||
}
|
|
||||||
|
|
||||||
return grid, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
/*
|
/*
|
||||||
MarshalBinary converts the chunk data to a binary representation.
|
MarshalBinary converts the chunk data to a binary representation.
|
||||||
|
|
||||||
|
@ -93,8 +69,8 @@ formatted as such:
|
||||||
func (a *RLEAccessor) MarshalBinary() ([]byte, error) {
|
func (a *RLEAccessor) MarshalBinary() ([]byte, error) {
|
||||||
// Flatten the chunk out into a full 2D array of all its points.
|
// Flatten the chunk out into a full 2D array of all its points.
|
||||||
var (
|
var (
|
||||||
size = int(a.acc.size)
|
size = int(a.chunk.Size)
|
||||||
grid, err = Make2DChunkGrid(size)
|
grid, err = rle.NewGrid(size)
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
|
@ -104,50 +80,18 @@ func (a *RLEAccessor) MarshalBinary() ([]byte, error) {
|
||||||
for px := range a.Iter() {
|
for px := range a.Iter() {
|
||||||
var (
|
var (
|
||||||
point = render.NewPoint(px.X, px.Y)
|
point = render.NewPoint(px.X, px.Y)
|
||||||
relative = RelativeCoordinate(point, a.acc.coord, a.acc.size)
|
relative = RelativeCoordinate(point, a.chunk.Point, a.chunk.Size)
|
||||||
ptr = uint64(px.PaletteIndex)
|
ptr = uint64(px.Swatch.Index())
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// TODO: sometimes we get a -1 value in X or Y, not sure why.
|
||||||
|
if relative.X < 0 || relative.Y < 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
grid[relative.Y][relative.X] = &ptr
|
grid[relative.Y][relative.X] = &ptr
|
||||||
}
|
}
|
||||||
|
|
||||||
// log.Error("2D GRID:\n%+v", grid)
|
return grid.Compress()
|
||||||
|
|
||||||
// Run-length encode the grid.
|
|
||||||
var (
|
|
||||||
compressed []byte
|
|
||||||
firstColor = true
|
|
||||||
lastColor uint64
|
|
||||||
runLength uint64
|
|
||||||
)
|
|
||||||
for _, row := range grid {
|
|
||||||
for _, color := range row {
|
|
||||||
var index uint64
|
|
||||||
if color == nil {
|
|
||||||
index = 0xFF
|
|
||||||
}
|
|
||||||
|
|
||||||
if firstColor {
|
|
||||||
lastColor = index
|
|
||||||
runLength = 1
|
|
||||||
firstColor = false
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
if index != lastColor {
|
|
||||||
compressed = binary.AppendUvarint(compressed, index)
|
|
||||||
compressed = binary.AppendUvarint(compressed, runLength)
|
|
||||||
lastColor = index
|
|
||||||
runLength = 1
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
runLength++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
log.Error("RLE compressed: %v", compressed)
|
|
||||||
|
|
||||||
return compressed, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// UnmarshalBinary will decode a compressed RLEAccessor byte stream.
|
// UnmarshalBinary will decode a compressed RLEAccessor byte stream.
|
||||||
|
@ -158,10 +102,39 @@ func (a *RLEAccessor) UnmarshalBinary(compressed []byte) error {
|
||||||
// New format: decompress the byte stream.
|
// New format: decompress the byte stream.
|
||||||
log.Debug("RLEAccessor.Unmarshal: Reading %d bytes of compressed chunk data", len(compressed))
|
log.Debug("RLEAccessor.Unmarshal: Reading %d bytes of compressed chunk data", len(compressed))
|
||||||
|
|
||||||
// Prepare the 2D grid to decompress the RLE stream into.
|
grid, err := rle.NewGrid(int(a.chunk.Size))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := grid.Decompress(compressed); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Load the grid into our MapAccessor.
|
||||||
|
a.acc.Reset()
|
||||||
|
for y, row := range grid {
|
||||||
|
for x, col := range row {
|
||||||
|
if col == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// TODO: x-1 to avoid the level creeping to the right every save,
|
||||||
|
// not sure on the root cause! RLEAccessor Decompress?
|
||||||
|
abs := FromRelativeCoordinate(render.NewPoint(x, y), a.chunk.Point, a.chunk.Size)
|
||||||
|
abs.X -= 1
|
||||||
|
a.acc.grid[abs] = NewSparseSwatch(int(*col))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
// Prepare the 2D grid to decompress the RLE stream into.
|
||||||
var (
|
var (
|
||||||
size = int(a.acc.size)
|
size = int(a.chunk.Size)
|
||||||
_, err = Make2DChunkGrid(size)
|
_, err = rle.NewGrid(size)
|
||||||
x, y, cursor int
|
x, y, cursor int
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -181,6 +154,8 @@ func (a *RLEAccessor) UnmarshalBinary(compressed []byte) error {
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
|
|
||||||
|
log.Warn("RLE index %d for %dpx", paletteIndex, repeatCount)
|
||||||
|
|
||||||
for i := uint64(0); i < repeatCount; i++ {
|
for i := uint64(0); i < repeatCount; i++ {
|
||||||
cursor++
|
cursor++
|
||||||
if cursor%size == 0 {
|
if cursor%size == 0 {
|
||||||
|
@ -196,6 +171,4 @@ func (a *RLEAccessor) UnmarshalBinary(compressed []byte) error {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
*/
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
|
@ -129,8 +129,10 @@ func TestChunker(t *testing.T) {
|
||||||
|
|
||||||
// Test the map chunk accessor.
|
// Test the map chunk accessor.
|
||||||
func TestMapAccessor(t *testing.T) {
|
func TestMapAccessor(t *testing.T) {
|
||||||
a := level.NewMapAccessor()
|
var (
|
||||||
_ = a
|
c = level.NewChunk()
|
||||||
|
a = level.NewMapAccessor(c)
|
||||||
|
)
|
||||||
|
|
||||||
// Test action types
|
// Test action types
|
||||||
var (
|
var (
|
||||||
|
|
|
@ -74,7 +74,6 @@ func (c *Chunker) Inflate(pal *Palette) error {
|
||||||
for coord, chunk := range c.Chunks {
|
for coord, chunk := range c.Chunks {
|
||||||
chunk.Point = coord
|
chunk.Point = coord
|
||||||
chunk.Size = c.Size
|
chunk.Size = c.Size
|
||||||
chunk.SetChunkCoordinate(chunk.Point, chunk.Size)
|
|
||||||
chunk.Inflate(pal)
|
chunk.Inflate(pal)
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
|
@ -326,7 +325,7 @@ func (c *Chunker) GetChunk(p render.Point) (*Chunk, bool) {
|
||||||
|
|
||||||
// Hit the zipfile for it.
|
// Hit the zipfile for it.
|
||||||
if c.Zipfile != nil {
|
if c.Zipfile != nil {
|
||||||
if chunk, err := ChunkFromZipfile(c.Zipfile, c.Layer, p); err == nil {
|
if chunk, err := c.ChunkFromZipfile(p); err == nil {
|
||||||
// log.Debug("GetChunk(%s) cache miss, read from zip", p)
|
// log.Debug("GetChunk(%s) cache miss, read from zip", p)
|
||||||
c.SetChunk(p, chunk) // cache it
|
c.SetChunk(p, chunk) // cache it
|
||||||
c.logChunkAccess(p, chunk) // for the LRU cache
|
c.logChunkAccess(p, chunk) // for the LRU cache
|
||||||
|
@ -446,7 +445,6 @@ func (c *Chunker) FreeCaches() int {
|
||||||
// This function should be the singular writer to the chunk cache.
|
// This function should be the singular writer to the chunk cache.
|
||||||
func (c *Chunker) SetChunk(p render.Point, chunk *Chunk) {
|
func (c *Chunker) SetChunk(p render.Point, chunk *Chunk) {
|
||||||
c.chunkMu.Lock()
|
c.chunkMu.Lock()
|
||||||
chunk.SetChunkCoordinate(p, chunk.Size)
|
|
||||||
c.Chunks[p] = chunk
|
c.Chunks[p] = chunk
|
||||||
c.chunkMu.Unlock()
|
c.chunkMu.Unlock()
|
||||||
|
|
||||||
|
@ -617,6 +615,32 @@ func (c *Chunker) ChunkCoordinate(abs render.Point) render.Point {
|
||||||
// - And relative inside that chunk, the pixel is at (24,)
|
// - And relative inside that chunk, the pixel is at (24,)
|
||||||
func RelativeCoordinate(abs render.Point, chunkCoord render.Point, chunkSize uint8) render.Point {
|
func RelativeCoordinate(abs render.Point, chunkCoord render.Point, chunkSize uint8) render.Point {
|
||||||
// Pixel coordinate offset.
|
// Pixel coordinate offset.
|
||||||
|
var (
|
||||||
|
size = int(chunkSize)
|
||||||
|
offset = render.Point{
|
||||||
|
X: chunkCoord.X * size,
|
||||||
|
Y: chunkCoord.Y * size,
|
||||||
|
}
|
||||||
|
point = render.Point{
|
||||||
|
X: abs.X - offset.X,
|
||||||
|
Y: abs.Y - offset.Y,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
if point.X < 0 || point.Y < 0 {
|
||||||
|
log.Error("RelativeCoordinate: X < 0! abs=%s rel=%s chunk=%s size=%d", abs, point, chunkCoord, chunkSize)
|
||||||
|
log.Error("RelativeCoordinate(2): size=%d offset=%s point=%s", size, offset, point)
|
||||||
|
}
|
||||||
|
|
||||||
|
return point
|
||||||
|
}
|
||||||
|
|
||||||
|
// FromRelativeCoordinate is the inverse of RelativeCoordinate.
|
||||||
|
//
|
||||||
|
// With a chunk size of 128 and a relative coordinate like (8, 12),
|
||||||
|
// this function will return the absolute world coordinates based
|
||||||
|
// on your chunk.Point's placement in the level.
|
||||||
|
func FromRelativeCoordinate(rel render.Point, chunkCoord render.Point, chunkSize uint8) render.Point {
|
||||||
var (
|
var (
|
||||||
size = int(chunkSize)
|
size = int(chunkSize)
|
||||||
offset = render.Point{
|
offset = render.Point{
|
||||||
|
@ -626,8 +650,8 @@ func RelativeCoordinate(abs render.Point, chunkCoord render.Point, chunkSize uin
|
||||||
)
|
)
|
||||||
|
|
||||||
return render.Point{
|
return render.Point{
|
||||||
X: abs.X - offset.X,
|
X: rel.X + offset.X,
|
||||||
Y: abs.Y - offset.Y,
|
Y: rel.Y + offset.Y,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
67
pkg/level/chunker_migrate.go
Normal file
67
pkg/level/chunker_migrate.go
Normal file
|
@ -0,0 +1,67 @@
|
||||||
|
package level
|
||||||
|
|
||||||
|
import (
|
||||||
|
"runtime"
|
||||||
|
"sync"
|
||||||
|
|
||||||
|
"git.kirsle.net/SketchyMaze/doodle/pkg/balance"
|
||||||
|
"git.kirsle.net/SketchyMaze/doodle/pkg/log"
|
||||||
|
)
|
||||||
|
|
||||||
|
/* Functions to migrate Chunkers between different implementations. */
|
||||||
|
|
||||||
|
// OptimizeChunkerAccessors will evaluate all of the chunks of your drawing
|
||||||
|
// and possibly migrate them to a different Accessor implementation when
|
||||||
|
// saving on disk.
|
||||||
|
func (c *Chunker) OptimizeChunkerAccessors() {
|
||||||
|
c.chunkMu.Lock()
|
||||||
|
defer c.chunkMu.Unlock()
|
||||||
|
|
||||||
|
log.Info("Optimizing Chunker Accessors")
|
||||||
|
|
||||||
|
// TODO: parallelize this with goroutines
|
||||||
|
var (
|
||||||
|
chunks = make(chan *Chunk, len(c.Chunks))
|
||||||
|
wg sync.WaitGroup
|
||||||
|
)
|
||||||
|
|
||||||
|
for range runtime.NumCPU() {
|
||||||
|
wg.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
for chunk := range chunks {
|
||||||
|
var point = chunk.Point
|
||||||
|
log.Warn("Chunk %s is a: %d", point, chunk.Type)
|
||||||
|
|
||||||
|
// Upgrade all MapTypes into RLE compressed MapTypes?
|
||||||
|
if balance.RLEBinaryChunkerEnabled {
|
||||||
|
if chunk.Type == MapType {
|
||||||
|
log.Info("Optimizing chunk %s accessor from Map to RLE", point)
|
||||||
|
ma, _ := chunk.Accessor.(*MapAccessor)
|
||||||
|
rle := NewRLEAccessor(chunk).FromMapAccessor(ma)
|
||||||
|
|
||||||
|
c.Chunks[point].Type = RLEType
|
||||||
|
c.Chunks[point].Accessor = rle
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Feed it the chunks.
|
||||||
|
for _, chunk := range c.Chunks {
|
||||||
|
chunks <- chunk
|
||||||
|
}
|
||||||
|
|
||||||
|
close(chunks)
|
||||||
|
wg.Wait()
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
// FromMapAccessor migrates from a MapAccessor to RLE.
|
||||||
|
func (a *RLEAccessor) FromMapAccessor(ma *MapAccessor) *RLEAccessor {
|
||||||
|
return &RLEAccessor{
|
||||||
|
chunk: a.chunk,
|
||||||
|
acc: ma,
|
||||||
|
}
|
||||||
|
}
|
|
@ -228,3 +228,98 @@ func TestViewportChunks(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRelativeCoordinates(t *testing.T) {
|
||||||
|
|
||||||
|
var (
|
||||||
|
chunker = level.NewChunker(128)
|
||||||
|
)
|
||||||
|
|
||||||
|
type TestCase struct {
|
||||||
|
WorldCoord render.Point
|
||||||
|
ChunkCoord render.Point
|
||||||
|
ExpectRelative render.Point
|
||||||
|
}
|
||||||
|
var tests = []TestCase{
|
||||||
|
{
|
||||||
|
WorldCoord: render.NewPoint(4, 8),
|
||||||
|
ExpectRelative: render.NewPoint(4, 8),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
WorldCoord: render.NewPoint(128, 128),
|
||||||
|
ExpectRelative: render.NewPoint(0, 0),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
WorldCoord: render.NewPoint(143, 144),
|
||||||
|
ExpectRelative: render.NewPoint(15, 16),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
WorldCoord: render.NewPoint(-105, -86),
|
||||||
|
ExpectRelative: render.NewPoint(23, 42),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
WorldCoord: render.NewPoint(-252, 264),
|
||||||
|
ExpectRelative: render.NewPoint(4, 8),
|
||||||
|
},
|
||||||
|
|
||||||
|
// These were seen breaking actual levels, at the corners of the chunk
|
||||||
|
{
|
||||||
|
WorldCoord: render.NewPoint(511, 256),
|
||||||
|
ExpectRelative: render.NewPoint(127, 0), // was getting -1,0 in game
|
||||||
|
},
|
||||||
|
{
|
||||||
|
WorldCoord: render.NewPoint(511, 512),
|
||||||
|
ChunkCoord: render.NewPoint(4, 4),
|
||||||
|
ExpectRelative: render.NewPoint(127, 0), // was getting -1,0 in game
|
||||||
|
},
|
||||||
|
{
|
||||||
|
WorldCoord: render.NewPoint(127, 384),
|
||||||
|
ChunkCoord: render.NewPoint(1, 3),
|
||||||
|
ExpectRelative: render.NewPoint(-1, 0),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for i, test := range tests {
|
||||||
|
var (
|
||||||
|
chunkCoord = test.ChunkCoord
|
||||||
|
actualRelative = level.RelativeCoordinate(
|
||||||
|
test.WorldCoord,
|
||||||
|
chunkCoord,
|
||||||
|
chunker.Size,
|
||||||
|
)
|
||||||
|
roundTrip = level.FromRelativeCoordinate(
|
||||||
|
actualRelative,
|
||||||
|
chunkCoord,
|
||||||
|
chunker.Size,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
// compute expected chunk coord automatically?
|
||||||
|
if chunkCoord == render.Origin {
|
||||||
|
chunkCoord = chunker.ChunkCoordinate(test.WorldCoord)
|
||||||
|
}
|
||||||
|
|
||||||
|
if actualRelative != test.ExpectRelative {
|
||||||
|
t.Errorf("Test %d: world coord %s in chunk %s\n"+
|
||||||
|
"Expected RelativeCoordinate() to be: %s\n"+
|
||||||
|
"But it was: %s",
|
||||||
|
i,
|
||||||
|
test.WorldCoord,
|
||||||
|
chunkCoord,
|
||||||
|
test.ExpectRelative,
|
||||||
|
actualRelative,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
if roundTrip != test.WorldCoord {
|
||||||
|
t.Errorf("Test %d: world coord %s in chunk %s\n"+
|
||||||
|
"Did not survive round trip! Expected: %s\n"+
|
||||||
|
"But it was: %s",
|
||||||
|
i,
|
||||||
|
test.WorldCoord,
|
||||||
|
chunkCoord,
|
||||||
|
test.WorldCoord,
|
||||||
|
roundTrip,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
|
@ -93,7 +93,7 @@ func (c *Chunker) MigrateZipfile(zf *zip.Writer) error {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Verify that this chunk file in the old ZIP was not empty.
|
// Verify that this chunk file in the old ZIP was not empty.
|
||||||
chunk, err := ChunkFromZipfile(c.Zipfile, c.Layer, point)
|
chunk, err := c.ChunkFromZipfile(point)
|
||||||
if err == nil && chunk.Len() == 0 {
|
if err == nil && chunk.Len() == 0 {
|
||||||
log.Debug("Skip chunk %s (old zipfile chunk was empty)", coord)
|
log.Debug("Skip chunk %s (old zipfile chunk was empty)", coord)
|
||||||
continue
|
continue
|
||||||
|
@ -205,14 +205,20 @@ func (c *Chunk) ToZipfile(zf *zip.Writer, layer int, coord render.Point) error {
|
||||||
}
|
}
|
||||||
|
|
||||||
// ChunkFromZipfile loads a chunk from a zipfile.
|
// ChunkFromZipfile loads a chunk from a zipfile.
|
||||||
func ChunkFromZipfile(zf *zip.Reader, layer int, coord render.Point) (*Chunk, error) {
|
func (c *Chunker) ChunkFromZipfile(coord render.Point) (*Chunk, error) {
|
||||||
// File names?
|
// File names?
|
||||||
var (
|
var (
|
||||||
|
zf = c.Zipfile
|
||||||
|
layer = c.Layer
|
||||||
|
|
||||||
binfile = fmt.Sprintf("chunks/%d/%s.bin", layer, coord)
|
binfile = fmt.Sprintf("chunks/%d/%s.bin", layer, coord)
|
||||||
jsonfile = fmt.Sprintf("chunks/%d/%s.json", layer, coord)
|
jsonfile = fmt.Sprintf("chunks/%d/%s.json", layer, coord)
|
||||||
chunk = NewChunk()
|
chunk = NewChunk()
|
||||||
)
|
)
|
||||||
|
|
||||||
|
chunk.Point = coord
|
||||||
|
chunk.Size = c.Size
|
||||||
|
|
||||||
// Read from the new binary format.
|
// Read from the new binary format.
|
||||||
if file, err := zf.Open(binfile); err == nil {
|
if file, err := zf.Open(binfile); err == nil {
|
||||||
// log.Debug("Reading binary compressed chunk from %s", binfile)
|
// log.Debug("Reading binary compressed chunk from %s", binfile)
|
||||||
|
|
|
@ -4,6 +4,21 @@ import "git.kirsle.net/SketchyMaze/doodle/pkg/log"
|
||||||
|
|
||||||
// Maintenance functions for the file format on disk.
|
// Maintenance functions for the file format on disk.
|
||||||
|
|
||||||
|
// Vacuum runs any maintenance or migration tasks for the level at time of save.
|
||||||
|
//
|
||||||
|
// It will prune broken links between actors, or migrate internal data structures
|
||||||
|
// to optimize storage on disk of its binary data.
|
||||||
|
func (m *Level) Vacuum() error {
|
||||||
|
if links := m.PruneLinks(); links > 0 {
|
||||||
|
log.Debug("Vacuum: removed %d broken links between actors in this level.")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Let the Chunker optimize accessor types.
|
||||||
|
m.Chunker.OptimizeChunkerAccessors()
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// PruneLinks cleans up any Actor Links that can not be resolved in the
|
// PruneLinks cleans up any Actor Links that can not be resolved in the
|
||||||
// level data. For example, if actors were linked in Edit Mode and one
|
// level data. For example, if actors were linked in Edit Mode and one
|
||||||
// actor is deleted leaving a broken link.
|
// actor is deleted leaving a broken link.
|
||||||
|
|
|
@ -3,6 +3,7 @@ package level
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"io/ioutil"
|
"io/ioutil"
|
||||||
|
"os"
|
||||||
"runtime"
|
"runtime"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
|
@ -96,7 +97,9 @@ func (m *Level) WriteFile(filename string) error {
|
||||||
m.GameVersion = branding.Version
|
m.GameVersion = branding.Version
|
||||||
|
|
||||||
// Maintenance functions, clean up cruft before save.
|
// Maintenance functions, clean up cruft before save.
|
||||||
m.PruneLinks()
|
if err := m.Vacuum(); err != nil {
|
||||||
|
log.Error("Vacuum level %s: %s", filename, err)
|
||||||
|
}
|
||||||
|
|
||||||
bin, err := m.ToJSON()
|
bin, err := m.ToJSON()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -115,7 +118,7 @@ func (m *Level) WriteFile(filename string) error {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Desktop: write to disk.
|
// Desktop: write to disk.
|
||||||
err = ioutil.WriteFile(filename, bin, 0644)
|
err = os.WriteFile(filename, bin, 0644)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("level.WriteFile: %s", err)
|
return fmt.Errorf("level.WriteFile: %s", err)
|
||||||
}
|
}
|
||||||
|
|
189
pkg/level/rle/rle.go
Normal file
189
pkg/level/rle/rle.go
Normal file
|
@ -0,0 +1,189 @@
|
||||||
|
// Package rle contains support for Run-Length Encoding of level chunks.
|
||||||
|
package rle
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/binary"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"git.kirsle.net/SketchyMaze/doodle/pkg/log"
|
||||||
|
"git.kirsle.net/go/render"
|
||||||
|
)
|
||||||
|
|
||||||
|
const NullColor = 0xFFFF
|
||||||
|
|
||||||
|
// Grid is a 2D array of nullable integers to store a flat bitmap of a chunk.
|
||||||
|
type Grid [][]*uint64
|
||||||
|
|
||||||
|
// NewGrid will return an initialized 2D grid of equal dimensions of the given size.
|
||||||
|
//
|
||||||
|
// The grid is indexed in [Y][X] notation, or: by row first and then column.
|
||||||
|
func NewGrid(size int) (Grid, error) {
|
||||||
|
if size == 0 {
|
||||||
|
return nil, errors.New("no size given for RLE Grid: the chunker was probably not initialized")
|
||||||
|
}
|
||||||
|
|
||||||
|
var grid = make([][]*uint64, size+1)
|
||||||
|
for i := 0; i < size+1; i++ {
|
||||||
|
grid[i] = make([]*uint64, size+1)
|
||||||
|
}
|
||||||
|
|
||||||
|
return grid, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func MustGrid(size int) Grid {
|
||||||
|
grid, err := NewGrid(size)
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
return grid
|
||||||
|
}
|
||||||
|
|
||||||
|
type Pixel struct {
|
||||||
|
Point render.Point
|
||||||
|
Palette int
|
||||||
|
}
|
||||||
|
|
||||||
|
// Size of the grid.
|
||||||
|
func (g Grid) Size() int {
|
||||||
|
return len(g[0])
|
||||||
|
}
|
||||||
|
|
||||||
|
// Compress the grid into a byte stream of RLE compressed data.
|
||||||
|
//
|
||||||
|
// The compressed format is a stream of:
|
||||||
|
//
|
||||||
|
// - A Uvarint for the palette index (0-255) or 0xffff (65535) for null.
|
||||||
|
// - A Uvarint for how many pixels to repeat that color.
|
||||||
|
func (g Grid) Compress() ([]byte, error) {
|
||||||
|
log.Error("BEGIN Compress()")
|
||||||
|
// log.Warn("Visualized:\n%s", g.Visualize())
|
||||||
|
|
||||||
|
// Run-length encode the grid.
|
||||||
|
var (
|
||||||
|
compressed []byte // final result
|
||||||
|
lastColor uint64 // last color seen (current streak)
|
||||||
|
runLength uint64 // current streak for the last color
|
||||||
|
buffering bool // detect end of grid
|
||||||
|
|
||||||
|
// Flush the buffer
|
||||||
|
flush = func() {
|
||||||
|
// log.Info("flush: %d for %d length", lastColor, runLength)
|
||||||
|
compressed = binary.AppendUvarint(compressed, lastColor)
|
||||||
|
compressed = binary.AppendUvarint(compressed, runLength)
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
for y, row := range g {
|
||||||
|
for x, nullableIndex := range row {
|
||||||
|
var index uint64
|
||||||
|
if nullableIndex == nil {
|
||||||
|
index = NullColor
|
||||||
|
} else {
|
||||||
|
index = *nullableIndex
|
||||||
|
}
|
||||||
|
|
||||||
|
// First color of the grid
|
||||||
|
if y == 0 && x == 0 {
|
||||||
|
// log.Info("First color @ %dx%d is %d", x, y, index)
|
||||||
|
lastColor = index
|
||||||
|
runLength = 1
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// Buffer it until we get a change of color or EOF.
|
||||||
|
if index != lastColor {
|
||||||
|
// log.Info("Color %d streaks for %d until %dx%d", lastColor, runLength, x, y)
|
||||||
|
flush()
|
||||||
|
lastColor = index
|
||||||
|
runLength = 1
|
||||||
|
buffering = false
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
buffering = true
|
||||||
|
runLength++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Flush the final buffer when we got to EOF on the grid.
|
||||||
|
if buffering {
|
||||||
|
flush()
|
||||||
|
}
|
||||||
|
|
||||||
|
// log.Error("RLE compressed: %v", compressed)
|
||||||
|
|
||||||
|
return compressed, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Decompress the RLE byte stream back into a populated 2D grid.
|
||||||
|
func (g Grid) Decompress(compressed []byte) error {
|
||||||
|
log.Error("BEGIN Decompress()")
|
||||||
|
// log.Warn("Visualized:\n%s", g.Visualize())
|
||||||
|
|
||||||
|
// Prepare the 2D grid to decompress the RLE stream into.
|
||||||
|
var (
|
||||||
|
size = g.Size()
|
||||||
|
x, y, cursor int
|
||||||
|
)
|
||||||
|
|
||||||
|
var reader = bytes.NewBuffer(compressed)
|
||||||
|
|
||||||
|
for {
|
||||||
|
var (
|
||||||
|
paletteIndexRaw, err1 = binary.ReadUvarint(reader)
|
||||||
|
repeatCount, err2 = binary.ReadUvarint(reader)
|
||||||
|
)
|
||||||
|
|
||||||
|
if err1 != nil || err2 != nil {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
// Handle the null color.
|
||||||
|
var paletteIndex *uint64
|
||||||
|
if paletteIndexRaw != NullColor {
|
||||||
|
paletteIndex = &paletteIndexRaw
|
||||||
|
}
|
||||||
|
|
||||||
|
// log.Warn("RLE index %v for %dpx", paletteIndexRaw, repeatCount)
|
||||||
|
|
||||||
|
for i := uint64(0); i < repeatCount; i++ {
|
||||||
|
cursor++
|
||||||
|
if cursor%size == 0 {
|
||||||
|
y++
|
||||||
|
x = 0
|
||||||
|
}
|
||||||
|
|
||||||
|
point := render.NewPoint(int(x), int(y))
|
||||||
|
if point.Y >= size || point.X >= size {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
g[point.Y][point.X] = paletteIndex
|
||||||
|
|
||||||
|
x++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// log.Warn("Visualized:\n%s", g.Visualize())
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Visualize the state of the 2D grid.
|
||||||
|
func (g Grid) Visualize() string {
|
||||||
|
var lines []string
|
||||||
|
for _, row := range g {
|
||||||
|
var line = "["
|
||||||
|
for _, col := range row {
|
||||||
|
if col == nil {
|
||||||
|
line += " "
|
||||||
|
} else {
|
||||||
|
line += fmt.Sprintf("%x", *col)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
lines = append(lines, line+"]")
|
||||||
|
}
|
||||||
|
return strings.Join(lines, "\n")
|
||||||
|
}
|
43
pkg/level/rle/rle_test.go
Normal file
43
pkg/level/rle/rle_test.go
Normal file
|
@ -0,0 +1,43 @@
|
||||||
|
package rle_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"git.kirsle.net/SketchyMaze/doodle/pkg/level/rle"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestRLE(t *testing.T) {
|
||||||
|
|
||||||
|
// Test a completely filled grid.
|
||||||
|
var (
|
||||||
|
grid = rle.MustGrid(128)
|
||||||
|
color = uint64(5)
|
||||||
|
)
|
||||||
|
for y := range grid {
|
||||||
|
for x := range y {
|
||||||
|
grid[y][x] = &color
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Compress and decompress it.
|
||||||
|
var (
|
||||||
|
compressed, _ = grid.Compress()
|
||||||
|
grid2 = rle.MustGrid(128)
|
||||||
|
)
|
||||||
|
grid2.Decompress(compressed)
|
||||||
|
|
||||||
|
// Ensure our color is set everywhere.
|
||||||
|
for y := range grid {
|
||||||
|
for x := range y {
|
||||||
|
if grid[y][x] != &color {
|
||||||
|
t.Errorf("RLE compression didn't survive the round trip: %d,%d didn't save\n"+
|
||||||
|
" Expected: %d\n"+
|
||||||
|
" Actually: %v",
|
||||||
|
x, y,
|
||||||
|
color,
|
||||||
|
grid[y][x],
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
|
@ -420,7 +420,7 @@ func (w *Canvas) loopEditable(ev *event.State) error {
|
||||||
baseColor, err := chunker.Get(cursor)
|
baseColor, err := chunker.Get(cursor)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
limit = balance.FloodToolVoidLimit
|
limit = balance.FloodToolVoidLimit
|
||||||
log.Warn("FloodTool: couldn't get base color at %s: %s (got %s)", cursor, err, baseColor.Color)
|
log.Warn("FloodTool: couldn't get base color at %s: %s (got %+v)", cursor, err, baseColor)
|
||||||
}
|
}
|
||||||
|
|
||||||
// If no change, do nothing.
|
// If no change, do nothing.
|
||||||
|
|
Loading…
Reference in New Issue
Block a user