doodle/pkg/level/chunker_zipfile.go

package level

import (
	"archive/zip"
	"fmt"
	"io/ioutil"
	"regexp"
	"strconv"

	"git.kirsle.net/SketchyMaze/doodle/pkg/balance"
	"git.kirsle.net/SketchyMaze/doodle/pkg/log"
	"git.kirsle.net/go/render"
)

// Zipfile interactions for the Chunker to cache and manage which
// chunks of large levels need be in active memory.

var (
	zipChunkfileRegexp = regexp.MustCompile(`^chunks/(\d+)/(.+?)\.(bin|json)$`)
)

// MigrateZipfile is called on save to migrate old-style ChunkMap
// chunks into external zipfile members and free up space in the
// master Level or Doodad struct.
func (c *Chunker) MigrateZipfile(zf *zip.Writer) error {
	// Identify if any chunks in active memory had been completely erased.
	var (
		// Chunks that have become empty and are to be REMOVED from zip.
		erasedChunks = map[render.Point]interface{}{}

		// Unique chunks we added to the zip file so we don't add duplicates.
		chunksZipped = map[render.Point]interface{}{}
	)
	for coord, chunk := range c.Chunks {
		if chunk.Len() == 0 {
			log.Info("Chunker.MigrateZipfile: %s has become empty, remove from zip", coord)
			erasedChunks[coord] = nil
		}
	}

	// Copy all COLD STORED chunks from our original zipfile into the new one.
	// These are chunks that are NOT actively loaded (those are written next),
	// and erasedChunks are not written to the zipfile at all.
	if c.Zipfile != nil {
		log.Info("MigrateZipfile: Copying chunk files from old zip to new zip")
		for _, file := range c.Zipfile.File {
			m := zipChunkfileRegexp.FindStringSubmatch(file.Name)
			if len(m) > 0 {
				var (
					mLayer, _ = strconv.Atoi(m[1])
					coord     = m[2]
					ext       = m[3]
				)

				// Will we need to do a format conversion now?
				var reencode bool
				if ext == "json" && balance.BinaryChunkerEnabled {
					reencode = true
				} else if ext == "bin" && !balance.BinaryChunkerEnabled {
					reencode = true
				}

				// Not our layer, not our problem.
				if mLayer != c.Layer {
					continue
				}

				point, err := render.ParsePoint(coord)
				if err != nil {
					return err
				}

				// Don't create zip files for empty (0 pixel) chunks.
				if _, ok := erasedChunks[point]; ok {
					log.Debug("Skip copying %s: chunk is empty", coord)
					continue
				}

				// Don't ever write duplicate files.
				if _, ok := chunksZipped[point]; ok {
					log.Debug("Skip copying duplicate chunk %s", coord)
					continue
				}
				chunksZipped[point] = nil

				// Don't copy the chunks we have currently in memory: those
				// are written next. Apparently zip files are allowed to
				// have duplicate named members!
				if _, ok := c.Chunks[point]; ok {
					log.Debug("Skip chunk %s (in memory)", coord)
					continue
				}

				// Verify that this chunk file in the old ZIP was not empty.
				chunk, err := ChunkFromZipfile(c.Zipfile, c.Layer, point)
				if err == nil && chunk.Len() == 0 {
					log.Debug("Skip chunk %s (old zipfile chunk was empty)", coord)
					continue
				}

				// Are we simply copying the existing chunk, or re-encoding it too?
				if reencode {
					log.Debug("Re-encoding existing chunk %s into target format", file.Name)
					if err := chunk.Inflate(c.pal); err != nil {
						return fmt.Errorf("couldn't inflate cold storage chunk for reencode: %s", err)
					}

					if err := chunk.ToZipfile(zf, mLayer, point); err != nil {
						return err
					}
				} else {
					log.Debug("Copy existing chunk %s", file.Name)
					if err := zf.Copy(file); err != nil {
						return err
					}
				}
			}
		}
	} else {
		log.Debug("Chunker.MigrateZipfile: the drawing did not give me a zipfile!")
	}

	if len(c.Chunks) == 0 {
		return nil
	}

	log.Info("MigrateZipfile: chunker has %d in memory, exporting to zipfile", len(c.Chunks))

	// Flush in-memory chunks out to zipfile.
	for coord, chunk := range c.Chunks {
		if _, ok := erasedChunks[coord]; ok {
			continue
		}

		// Are we encoding chunks as JSON?
		log.Debug("Flush in-memory chunks %s to zip", coord)
		chunk.ToZipfile(zf, c.Layer, coord)
	}

	// Flush the chunkmap out.
	// TODO: do similar to move old attached files (wallpapers) too
	c.Chunks = ChunkMap{}

	return nil
}

// ClearChunkCache completely flushes the ChunkMap from memory. BE CAREFUL.
// If the level is a Zipfile the chunks will reload as needed, but old style
// levels this will nuke the whole drawing!
func (c *Chunker) ClearChunkCache() {
	c.chunkMu.Lock()
	c.Chunks = ChunkMap{}
	c.chunkMu.Unlock()
}

// CacheSize returns the number of chunks in memory.
func (c *Chunker) CacheSize() int {
	return len(c.Chunks)
}

// GCSize returns the number of chunks pending free (not accessed in 2+ ticks)
func (c *Chunker) GCSize() int {
	return len(c.chunksToFree)
}

// ToZipfile writes just a chunk's data into a zipfile.
//
// It will write a file like "chunks/{layer}/{coord}.json" if using JSON
// format or a .bin file for binary format based on the BinaryChunkerEnabled
// game config constant.
func (c *Chunk) ToZipfile(zf *zip.Writer, layer int, coord render.Point) error {
	// File name?
	ext := ".json"
	if balance.BinaryChunkerEnabled {
		ext = ".bin"
	}
	filename := fmt.Sprintf("chunks/%d/%s%s", layer, coord, ext)

	writer, err := zf.Create(filename)
	if err != nil {
		return err
	}

	// Are we writing it as binary format?
	var data []byte
	if balance.BinaryChunkerEnabled {
		if bytes, err := c.MarshalBinary(); err != nil {
			return err
		} else {
			data = bytes
		}
	} else {
		if json, err := c.MarshalJSON(); err != nil {
			return err
		} else {
			data = json
		}
	}

	// Write the file contents to zip whether binary or json.
	n, err := writer.Write(data)
	if err != nil {
		return err
	}

	log.Debug("Written chunk to zipfile: %s (%d bytes)", filename, n)
	return nil
}

// ChunkFromZipfile loads a chunk from a zipfile.
func ChunkFromZipfile(zf *zip.Reader, layer int, coord render.Point) (*Chunk, error) {
	// File names?
	var (
		binfile  = fmt.Sprintf("chunks/%d/%s.bin", layer, coord)
		jsonfile = fmt.Sprintf("chunks/%d/%s.json", layer, coord)
		chunk    = NewChunk()
	)

	// Read from the new binary format.
	if file, err := zf.Open(binfile); err == nil {
		// log.Debug("Reading binary compressed chunk from %s", binfile)
		bin, err := ioutil.ReadAll(file)
		if err != nil {
			return nil, err
		}

		err = chunk.UnmarshalBinary(bin)
		if err != nil {
			return nil, err
		}
	} else if file, err := zf.Open(jsonfile); err == nil {
		// log.Debug("Reading JSON encoded chunk from %s", jsonfile)
		bin, err := ioutil.ReadAll(file)
		if err != nil {
			return nil, err
		}

		err = chunk.UnmarshalJSON(bin)
		if err != nil {
			return nil, err
		}
	} else {
		return nil, err
	}

	return chunk, nil
}

// ChunksInZipfile returns the list of chunk coordinates in a zipfile.
func ChunksInZipfile(zf *zip.Reader, layer int) []render.Point {
	var (
		result = []render.Point{}
		sLayer = fmt.Sprintf("%d", layer)
	)

	for _, file := range zf.File {
		m := zipChunkfileRegexp.FindStringSubmatch(file.Name)
		if len(m) > 0 {
			var (
				mLayer = m[1]
				mPoint = m[2]
			)

			// Not our layer?
			if mLayer != sLayer {
				continue
			}

			if point, err := render.ParsePoint(mPoint); err == nil {
				result = append(result, point)
			} else {
				log.Error("ChunksInZipfile: file '%s' didn't parse as a point: %s", file.Name, err)
			}
		}
	}

	return result
}

// ChunkInZipfile tests whether the chunk exists in the zipfile.
func ChunkInZipfile(zf *zip.Reader, layer int, coord render.Point) bool {
	for _, chunk := range ChunksInZipfile(zf, layer) {
		if chunk == coord {
			return true
		}
	}
	return false
}