doodle/pkg/level/chunker_zipfile.go

289 lines
7.4 KiB
Go

package level
import (
"archive/zip"
"fmt"
"io/ioutil"
"regexp"
"strconv"
"git.kirsle.net/SketchyMaze/doodle/pkg/balance"
"git.kirsle.net/SketchyMaze/doodle/pkg/log"
"git.kirsle.net/go/render"
)
// Zipfile interactions for the Chunker to cache and manage which
// chunks of large levels need be in active memory.
var (
zipChunkfileRegexp = regexp.MustCompile(`^chunks/(\d+)/(.+?)\.(bin|json)$`)
)
// MigrateZipfile is called on save to migrate old-style ChunkMap
// chunks into external zipfile members and free up space in the
// master Level or Doodad struct.
func (c *Chunker) MigrateZipfile(zf *zip.Writer) error {
// Identify if any chunks in active memory had been completely erased.
var (
// Chunks that have become empty and are to be REMOVED from zip.
erasedChunks = map[render.Point]interface{}{}
// Unique chunks we added to the zip file so we don't add duplicates.
chunksZipped = map[render.Point]interface{}{}
)
for coord, chunk := range c.Chunks {
if chunk.Len() == 0 {
log.Info("Chunker.MigrateZipfile: %s has become empty, remove from zip", coord)
erasedChunks[coord] = nil
}
}
// Copy all COLD STORED chunks from our original zipfile into the new one.
// These are chunks that are NOT actively loaded (those are written next),
// and erasedChunks are not written to the zipfile at all.
if c.Zipfile != nil {
log.Info("MigrateZipfile: Copying chunk files from old zip to new zip")
for _, file := range c.Zipfile.File {
m := zipChunkfileRegexp.FindStringSubmatch(file.Name)
if len(m) > 0 {
var (
mLayer, _ = strconv.Atoi(m[1])
coord = m[2]
ext = m[3]
)
// Will we need to do a format conversion now?
var reencode bool
if ext == "json" && balance.BinaryChunkerEnabled {
reencode = true
} else if ext == "bin" && !balance.BinaryChunkerEnabled {
reencode = true
}
// Not our layer, not our problem.
if mLayer != c.Layer {
continue
}
point, err := render.ParsePoint(coord)
if err != nil {
return err
}
// Don't create zip files for empty (0 pixel) chunks.
if _, ok := erasedChunks[point]; ok {
log.Debug("Skip copying %s: chunk is empty", coord)
continue
}
// Don't ever write duplicate files.
if _, ok := chunksZipped[point]; ok {
log.Debug("Skip copying duplicate chunk %s", coord)
continue
}
chunksZipped[point] = nil
// Don't copy the chunks we have currently in memory: those
// are written next. Apparently zip files are allowed to
// have duplicate named members!
if _, ok := c.Chunks[point]; ok {
log.Debug("Skip chunk %s (in memory)", coord)
continue
}
// Verify that this chunk file in the old ZIP was not empty.
chunk, err := ChunkFromZipfile(c.Zipfile, c.Layer, point)
if err == nil && chunk.Len() == 0 {
log.Debug("Skip chunk %s (old zipfile chunk was empty)", coord)
continue
}
// Are we simply copying the existing chunk, or re-encoding it too?
if reencode {
log.Debug("Re-encoding existing chunk %s into target format", file.Name)
if err := chunk.Inflate(c.pal); err != nil {
return fmt.Errorf("couldn't inflate cold storage chunk for reencode: %s", err)
}
if err := chunk.ToZipfile(zf, mLayer, point); err != nil {
return err
}
} else {
log.Debug("Copy existing chunk %s", file.Name)
if err := zf.Copy(file); err != nil {
return err
}
}
}
}
} else {
log.Debug("Chunker.MigrateZipfile: the drawing did not give me a zipfile!")
}
if len(c.Chunks) == 0 {
return nil
}
log.Info("MigrateZipfile: chunker has %d in memory, exporting to zipfile", len(c.Chunks))
// Flush in-memory chunks out to zipfile.
for coord, chunk := range c.Chunks {
if _, ok := erasedChunks[coord]; ok {
continue
}
// Are we encoding chunks as JSON?
log.Debug("Flush in-memory chunks %s to zip", coord)
chunk.ToZipfile(zf, c.Layer, coord)
}
// Flush the chunkmap out.
// TODO: do similar to move old attached files (wallpapers) too
c.Chunks = ChunkMap{}
return nil
}
// ClearChunkCache completely flushes the ChunkMap from memory. BE CAREFUL.
// If the level is a Zipfile the chunks will reload as needed, but old style
// levels this will nuke the whole drawing!
func (c *Chunker) ClearChunkCache() {
c.chunkMu.Lock()
c.Chunks = ChunkMap{}
c.chunkMu.Unlock()
}
// CacheSize returns the number of chunks in memory.
func (c *Chunker) CacheSize() int {
return len(c.Chunks)
}
// GCSize returns the number of chunks pending free (not accessed in 2+ ticks)
func (c *Chunker) GCSize() int {
return len(c.chunksToFree)
}
// ToZipfile writes just a chunk's data into a zipfile.
//
// It will write a file like "chunks/{layer}/{coord}.json" if using JSON
// format or a .bin file for binary format based on the BinaryChunkerEnabled
// game config constant.
func (c *Chunk) ToZipfile(zf *zip.Writer, layer int, coord render.Point) error {
// File name?
ext := ".json"
if balance.BinaryChunkerEnabled {
ext = ".bin"
}
filename := fmt.Sprintf("chunks/%d/%s%s", layer, coord, ext)
writer, err := zf.Create(filename)
if err != nil {
return err
}
// Are we writing it as binary format?
var data []byte
if balance.BinaryChunkerEnabled {
if bytes, err := c.MarshalBinary(); err != nil {
return err
} else {
data = bytes
}
} else {
if json, err := c.MarshalJSON(); err != nil {
return err
} else {
data = json
}
}
// Write the file contents to zip whether binary or json.
n, err := writer.Write(data)
if err != nil {
return err
}
log.Debug("Written chunk to zipfile: %s (%d bytes)", filename, n)
return nil
}
// ChunkFromZipfile loads a chunk from a zipfile.
func ChunkFromZipfile(zf *zip.Reader, layer int, coord render.Point) (*Chunk, error) {
// File names?
var (
binfile = fmt.Sprintf("chunks/%d/%s.bin", layer, coord)
jsonfile = fmt.Sprintf("chunks/%d/%s.json", layer, coord)
chunk = NewChunk()
)
// Read from the new binary format.
if file, err := zf.Open(binfile); err == nil {
log.Debug("Reading binary compressed chunk from %s", binfile)
bin, err := ioutil.ReadAll(file)
if err != nil {
return nil, err
}
err = chunk.UnmarshalBinary(bin)
if err != nil {
return nil, err
}
} else if file, err := zf.Open(jsonfile); err == nil {
log.Debug("Reading JSON encoded chunk from %s", jsonfile)
bin, err := ioutil.ReadAll(file)
if err != nil {
return nil, err
}
err = chunk.UnmarshalJSON(bin)
if err != nil {
return nil, err
}
} else {
return nil, err
}
return chunk, nil
}
// ChunksInZipfile returns the list of chunk coordinates in a zipfile.
func ChunksInZipfile(zf *zip.Reader, layer int) []render.Point {
var (
result = []render.Point{}
sLayer = fmt.Sprintf("%d", layer)
)
for _, file := range zf.File {
m := zipChunkfileRegexp.FindStringSubmatch(file.Name)
if len(m) > 0 {
var (
mLayer = m[1]
mPoint = m[2]
)
// Not our layer?
if mLayer != sLayer {
continue
}
if point, err := render.ParsePoint(mPoint); err == nil {
result = append(result, point)
} else {
log.Error("ChunksInZipfile: file '%s' didn't parse as a point: %s", file.Name, err)
}
}
}
return result
}
// ChunkInZipfile tests whether the chunk exists in the zipfile.
func ChunkInZipfile(zf *zip.Reader, layer int, coord render.Point) bool {
for _, chunk := range ChunksInZipfile(zf, layer) {
if chunk == coord {
return true
}
}
return false
}