Zipfiles for Attached Files Too

* The level.FileSystem type has updated to support ZIP files too.
* Legacy levels loaded from gz/json have their old FileSystem as a
  simple map[filename]data and this parses from JSON OK.
* On save to zip, the legacy loaded file data gets exported to ZIP.
* Going forward: newly added or deleted files during runtime are kept in
  the legacy file map until the next save when the filemap is again
  flushed out to ZIP.
* For regular read-access, the FileSystem reads from the ZIP file if the
  data is not in the hot map (legacy file or recently modified
  attachment).
* Bugfix: be sure to Inflate() the Level/Doodad after loading from
  zipfile - it used to be that directly after a save, trying to play the
  level failed because the Level.Actors struct was missing their IDs,
  and similarly recently written chunks would error out (become black
  voids) on levels/doodads so we Inflate() both after save/replacing
  their zip handle.

pkg/commands.go

@@ -337,13 +337,14 @@ func (c Command) RunScript(d *Doodle, code string) (goja.Value, error) {
 		}
 	}()
 
+	out, err := d.shell.js.RunString(code)
+
 	// If we're in Play Mode, consider it cheating if the player is
 	// messing with any in-game structures.
 	if scene, ok := d.Scene.(*PlayScene); ok {
 		scene.SetCheated()
 	}
 
-	out, err := d.shell.js.RunString(code)
 	return out, err
 }
 

pkg/doodads/fmt_zipfile.go

@@ -23,6 +23,13 @@ func (d *Doodad) ToZipfile() ([]byte, error) {
 		}
 	}
 
+	// Migrate attached files to ZIP.
+	if d.Files != nil {
+		if err := d.Files.MigrateZipfile(zipper); err != nil {
+			return nil, fmt.Errorf("FileSystem.MigrateZipfile: %s", err)
+		}
+	}
+
 	// Write the header json.
 	{
 		header, err := d.AsJSON()
@@ -91,11 +98,17 @@ func (d *Doodad) populateFromZipfile(data []byte) error {
 
 	// Keep the zipfile reader handy.
 	d.Zipfile = zf
+	if d.Files != nil {
+		d.Files.Zipfile = zf
+	}
 	for i, layer := range d.Layers {
 		layer.Chunker.Layer = i
 		layer.Chunker.Zipfile = zf
 	}
 
+	// Re-inflate data after saving a new zipfile.
+	d.Inflate()
+
 	return err
 }
 

pkg/level/filesystem.go

@@ -1,93 +1,281 @@
 package level
 
 import (
+	"archive/zip"
+	"encoding/json"
 	"errors"
+	"fmt"
+	"io/ioutil"
 	"sort"
 	"strings"
+
+	"git.kirsle.net/apps/doodle/pkg/log"
 )
 
-// FileSystem embeds a map of files inside a parent drawing.
+/*
-type FileSystem map[string]File
+FileSystem embeds a map of files inside a parent drawing.
+
+Old-style drawings this was a map of filenames to their data in the JSON.
+New-style drawings this just holds the filenames and the data is read
+from the zipfile on demand.
+*/
+type FileSystem struct {
+	filemap map[string]File `json:"-"` // Legacy JSON format
+	Zipfile *zip.Reader     `json:"-"` // New Zipfile format accessor
+}
 
 // File holds details about a file in the FileSystem.
 type File struct {
-	Data []byte `json:"data"`
+	Data []byte `json:"data,omitempty"`
 }
 
+// NewFileSystem initializes the FileSystem struct.
+func NewFileSystem() *FileSystem {
+	return &FileSystem{
+		filemap: map[string]File{},
+	}
+}
+
+// Get a file from the FileSystem.
+func (fs *FileSystem) Get(filename string) ([]byte, error) {
+	if fs.filemap == nil {
+		fs.filemap = map[string]File{}
+	}
+
+	// Legacy file map.
+	if file, ok := fs.filemap[filename]; ok {
+		if len(file.Data) > 0 {
+			return file.Data, nil
+		}
+	}
+
+	// Check in the zipfile.
+	if fs.Zipfile != nil {
+		file, err := fs.Zipfile.Open(filename)
+		if err != nil {
+			return []byte{}, fmt.Errorf("%s: not in zipfile: %s", filename, err)
+		}
+
+		bin, err := ioutil.ReadAll(file)
+		if err != nil {
+			return bin, fmt.Errorf("%s: couldn't read zipfile member: %s", filename, err)
+		}
+
+		return bin, nil
+	}
+
+	return []byte{}, fmt.Errorf("no such file")
+}
+
+// Set a file into the FileSystem. Note: it will go into the legacy map
+// structure until the next save to disk, at which point queued files
+// are committed to ZIP.
+func (fs *FileSystem) Set(filename string, data []byte) {
+	if fs.filemap == nil {
+		fs.filemap = map[string]File{}
+	}
+
+	fs.filemap[filename] = File{
+		Data: data,
+	}
+}
+
+// Delete a file from the FileSystem. This will store zero bytes in the
+// legacy file map structure to mark it for deletion. On the next save,
+// filemap files with zero bytes skip the ZIP archive.
+func (fs *FileSystem) Delete(filename string) {
+	if fs.filemap == nil {
+		fs.filemap = map[string]File{}
+	}
+
+	fs.filemap[filename] = File{
+		Data: []byte{},
+	}
+}
+
+// List files in the FileSystem, including the ZIP file.
+//
+// In the ZIP file, attachments are under the "assets/" prefix so this
+// function won't mistakenly return chunks or level.json/doodad.json.
+func (fs *FileSystem) List() []string {
+	var (
+		result = []string{}
+		seen   = map[string]interface{}{}
+	)
+
+	// List the legacy or recently modified files first.
+	if fs.filemap != nil {
+		for filename := range fs.filemap {
+			result = append(result, filename)
+			seen[filename] = nil
+		}
+	}
+
+	// List the zipfile members.
+	if fs.Zipfile != nil {
+		for _, file := range fs.Zipfile.File {
+			if !strings.HasPrefix(file.Name, "assets/") {
+				continue
+			}
+
+			if _, ok := seen[file.Name]; !ok {
+				result = append(result, file.Name)
+				seen[file.Name] = nil
+			}
+		}
+	}
+
+	sort.Strings(result)
+	return result
+}
+
+// ListPrefix returns a list of files starting with the prefix.
+func (fs *FileSystem) ListPrefix(prefix string) []string {
+	var result = []string{}
+	for _, name := range fs.List() {
+		if strings.HasPrefix(name, prefix) {
+			result = append(result, name)
+		}
+	}
+	return result
+}
+
+// UnmarshalJSON reads in a FileSystem from its legacy JSON representation.
+func (fs *FileSystem) UnmarshalJSON(text []byte) error {
+	// Legacy format was a simple map[string]File.
+	var legacy map[string]File
+	err := json.Unmarshal(text, &legacy)
+	if err != nil {
+		return err
+	}
+
+	fs.filemap = legacy
+	return nil
+}
+
+// MigrateZipfile is called on save to write attached files to the ZIP
+// file format.
+func (fs *FileSystem) MigrateZipfile(zf *zip.Writer) error {
+	// Identify the files that we have marked for deletion.
+	var (
+		filesDeleted = map[string]interface{}{}
+		filesZipped  = map[string]interface{}{}
+	)
+	if fs.filemap != nil {
+		for filename, data := range fs.filemap {
+			if len(data.Data) == 0 {
+				log.Info("FileSystem.MigrateZipfile: %s has become empty, remove from zip", filename)
+				filesDeleted[filename] = nil
+			}
+		}
+	}
+
+	// Copy all COLD STORED files from the old Zipfile into the new Zipfile
+	// except for the ones marked for deletion OR the ones currently in the
+	// warm cache which will be written next.
+	if fs.Zipfile != nil {
+		log.Info("FileSystem.MigrateZipfile: Copying files from old zip to new zip")
+		for _, file := range fs.Zipfile.File {
+			if !strings.HasPrefix(file.Name, "assets/") {
+				continue
+			}
+
+			if _, ok := filesDeleted[file.Name]; ok {
+				log.Debug("Skip copying attachment %s: was marked for deletion")
+				continue
+			}
+
+			// Skip files currently in memory.
+			if fs.filemap != nil {
+				if _, ok := fs.filemap[file.Name]; ok {
+					log.Debug("Skip copying attachment %s: one is loaded in memory")
+					continue
+				}
+			}
+
+			log.Debug("Copy zipfile attachment %s", file.Name)
+			filesZipped[file.Name] = nil
+
+			if err := zf.Copy(file); err != nil {
+				return err
+			}
+		}
+	}
+
+	// Export currently warmed up files to ZIP, these will be ones that
+	// were updated recently OR legacy files from an old level read.
+	if fs.filemap != nil {
+		log.Info("FileSystem.MigrateZipfile: has %d files in memory to write to ZIP", len(fs.filemap))
+		for filename, file := range fs.filemap {
+			if _, ok := filesZipped[filename]; ok {
+				continue
+			}
+
+			writer, err := zf.Create(filename)
+			if err != nil {
+				return err
+			}
+
+			n, err := writer.Write(file.Data)
+			if err != nil {
+				return err
+			}
+
+			log.Debug("Exported file to zip: %s (%d bytes)", filename, n)
+		}
+	}
+
+	return nil
+}
+
+////////////////
+// Level class methods for its filesystem access
+////////////////
+
 // SetFile sets a file's data in the level.
 func (l *Level) SetFile(filename string, data []byte) {
-	if l.Files == nil {
+	l.Files.Set(filename, data)
-		l.Files = map[string]File{}
-	}
-
-	l.Files[filename] = File{
-		Data: data,
-	}
 }
 
 // GetFile looks up an embedded file.
 func (l *Level) GetFile(filename string) ([]byte, error) {
 	if l.Files == nil {
-		l.Files = map[string]File{}
+		return []byte{}, errors.New("filesystem not initialized")
 	}
-
+	return l.Files.Get(filename)
-	if result, ok := l.Files[filename]; ok {
-		return result.Data, nil
-	}
-	return []byte{}, errors.New("not found")
 }
 
 // DeleteFile removes an embedded file.
 func (l *Level) DeleteFile(filename string) bool {
-	if l.Files == nil {
+	l.Files.Delete(filename)
-		l.Files = map[string]File{}
-	}
-
-	if _, ok := l.Files[filename]; ok {
-		delete(l.Files, filename)
 	return true
 }
-	return false
-}
 
 // DeleteFiles removes all files beginning with the prefix.
 func (l *Level) DeleteFiles(prefix string) int {
 	var count int
-	for filename := range l.Files {
+	for _, filename := range l.Files.ListPrefix(prefix) {
-		if strings.HasPrefix(filename, prefix) {
+		l.Files.Delete(filename)
-			delete(l.Files, filename)
 		count++
 	}
-	}
 	return count
 }
 
 // ListFiles returns the list of all embedded file names, alphabetically.
 func (l *Level) ListFiles() []string {
-	var files []string
+	if l == nil {
-
+		log.Error("Level.ListFiles() was called on a nil Level??")
-	if l == nil || l.Files == nil {
+		return []string{}
-		return files
 	}
 
-	for name := range l.Files {
+	if l.Files == nil {
-		files = append(files, name)
+		log.Error("Level(%s).ListFiles: FileSystem not initialized", l.Title)
+		return []string{}
 	}
-
+	return l.Files.List()
-	sort.Strings(files)
-	return files
 }
 
 // ListFilesAt returns the list of files having a common prefix.
 func (l *Level) ListFilesAt(prefix string) []string {
-	var (
+	return l.Files.ListPrefix(prefix)
-		files = l.ListFiles()
-		match = []string{}
-	)
-	for _, name := range files {
-		if strings.HasPrefix(name, prefix) {
-			match = append(match, name)
-		}
-	}
-	return match
 }

pkg/level/fmt_json.go

@@ -114,6 +114,11 @@ func (m *Level) ToZipfile() ([]byte, error) {
 		return nil, fmt.Errorf("MigrateZipfile: %s", err)
 	}
 
+	// Migrate attached files to ZIP.
+	if err := m.Files.MigrateZipfile(zipper); err != nil {
+		return nil, fmt.Errorf("FileSystem.MigrateZipfile: %s", err)
+	}
+
 	// Write the header json.
 	{
 		header, err := m.AsJSON()
@@ -204,6 +209,11 @@ func (m *Level) populateFromZipfile(data []byte) error {
 	// Keep the zipfile reader handy.
 	m.Zipfile = zf
 	m.Chunker.Zipfile = zf
+	m.Files.Zipfile = zf
+
+	// Re-inflate the level: ensures Actor instances get their IDs
+	// and everything is reloaded after saving the level.
+	m.Inflate()
 
 	return err
 }

pkg/level/types.go

@@ -35,7 +35,7 @@ type Base struct {
 	Zipfile *zip.Reader `json:"-"`
 
 	// Every drawing type is able to embed other files inside of itself.
-	Files FileSystem `json:"files"`
+	Files *FileSystem `json:"files,omitempty"`
 }
 
 // Level is the container format for Doodle map drawings.
@@ -81,6 +81,7 @@ func New() *Level {
 			Version: 1,
 			Title:   "Untitled",
 			Author:  os.Getenv("USER"),
+			Files:   NewFileSystem(),
 		},
 		Chunker: NewChunker(balance.ChunkSize),
 		Palette: &Palette{},

pkg/play_scene.go

@@ -403,7 +403,7 @@ func (s *PlayScene) setupPlayer(playerCharacterFilename string) {
 // centerIn is optional, ignored if zero.
 func (s *PlayScene) installPlayerDoodad(filename string, spawn render.Point, centerIn render.Rect) {
 	// Load in the player character.
-	player, err := doodads.LoadFile(filename)
+	player, err := doodads.LoadFromEmbeddable(filename, s.Level)
 	if err != nil {
 		log.Error("PlayScene.Setup: failed to load player doodad: %s", err)
 		player = doodads.NewDummy(32)
@@ -569,6 +569,12 @@ func (s *PlayScene) SetCheated() {
 	}
 }
 
+// GetCheated gives read-only access to tell if you have been cheating. However, by
+// querying this in the dev console during gameplay, you would be marked as cheating. ;)
+func (s *PlayScene) GetCheated() bool {
+	return s.cheated
+}
+
 // ShowEndLevelModal centralizes the EndLevel modal config.
 // This is the common handler function between easy methods such as
 // BeatLevel, FailLevel, and DieByFire.

pkg/scripting/scripting.go

@@ -83,7 +83,7 @@ func (s *Supervisor) InstallScripts(level *level.Level) error {
 // The `name` is used to name the VM for debug logging.
 func (s *Supervisor) AddLevelScript(id string, name string) error {
 	if _, ok := s.scripts[id]; ok {
-		return fmt.Errorf("duplicate actor ID %s in level", id)
+		return fmt.Errorf("AddLevelScript: duplicate actor ID '%s' in level", id)
 	}
 
 	s.scripts[id] = NewVM(fmt.Sprintf("%s#%s", name, id))