From 402b5efa7e64a07f80b471fbd39138ac1af805f2 Mon Sep 17 00:00:00 2001 From: Noah Petherbridge Date: Sat, 30 Apr 2022 12:47:35 -0700 Subject: [PATCH] Zipfiles for Attached Files Too * The level.FileSystem type has updated to support ZIP files too. * Legacy levels loaded from gz/json have their old FileSystem as a simple map[filename]data and this parses from JSON OK. * On save to zip, the legacy loaded file data gets exported to ZIP. * Going forward: newly added or deleted files during runtime are kept in the legacy file map until the next save when the filemap is again flushed out to ZIP. * For regular read-access, the FileSystem reads from the ZIP file if the data is not in the hot map (legacy file or recently modified attachment). * Bugfix: be sure to Inflate() the Level/Doodad after loading from zipfile - it used to be that directly after a save, trying to play the level failed because the Level.Actors struct was missing their IDs, and similarly recently written chunks would error out (become black voids) on levels/doodads so we Inflate() both after save/replacing their zip handle. --- pkg/commands.go | 3 +- pkg/doodads/fmt_zipfile.go | 13 ++ pkg/level/filesystem.go | 286 ++++++++++++++++++++++++++++++------- pkg/level/fmt_json.go | 10 ++ pkg/level/types.go | 3 +- pkg/play_scene.go | 8 +- pkg/scripting/scripting.go | 2 +- 7 files changed, 272 insertions(+), 53 deletions(-) diff --git a/pkg/commands.go b/pkg/commands.go index ae770f9..3c92342 100644 --- a/pkg/commands.go +++ b/pkg/commands.go @@ -337,13 +337,14 @@ func (c Command) RunScript(d *Doodle, code string) (goja.Value, error) { } }() + out, err := d.shell.js.RunString(code) + // If we're in Play Mode, consider it cheating if the player is // messing with any in-game structures. if scene, ok := d.Scene.(*PlayScene); ok { scene.SetCheated() } - out, err := d.shell.js.RunString(code) return out, err } diff --git a/pkg/doodads/fmt_zipfile.go b/pkg/doodads/fmt_zipfile.go index 1baeb2e..81a692e 100644 --- a/pkg/doodads/fmt_zipfile.go +++ b/pkg/doodads/fmt_zipfile.go @@ -23,6 +23,13 @@ func (d *Doodad) ToZipfile() ([]byte, error) { } } + // Migrate attached files to ZIP. + if d.Files != nil { + if err := d.Files.MigrateZipfile(zipper); err != nil { + return nil, fmt.Errorf("FileSystem.MigrateZipfile: %s", err) + } + } + // Write the header json. { header, err := d.AsJSON() @@ -91,11 +98,17 @@ func (d *Doodad) populateFromZipfile(data []byte) error { // Keep the zipfile reader handy. d.Zipfile = zf + if d.Files != nil { + d.Files.Zipfile = zf + } for i, layer := range d.Layers { layer.Chunker.Layer = i layer.Chunker.Zipfile = zf } + // Re-inflate data after saving a new zipfile. + d.Inflate() + return err } diff --git a/pkg/level/filesystem.go b/pkg/level/filesystem.go index 191a645..77ad4b6 100644 --- a/pkg/level/filesystem.go +++ b/pkg/level/filesystem.go @@ -1,93 +1,281 @@ package level import ( + "archive/zip" + "encoding/json" "errors" + "fmt" + "io/ioutil" "sort" "strings" + + "git.kirsle.net/apps/doodle/pkg/log" ) -// FileSystem embeds a map of files inside a parent drawing. -type FileSystem map[string]File +/* +FileSystem embeds a map of files inside a parent drawing. + +Old-style drawings this was a map of filenames to their data in the JSON. +New-style drawings this just holds the filenames and the data is read +from the zipfile on demand. +*/ +type FileSystem struct { + filemap map[string]File `json:"-"` // Legacy JSON format + Zipfile *zip.Reader `json:"-"` // New Zipfile format accessor +} // File holds details about a file in the FileSystem. type File struct { - Data []byte `json:"data"` + Data []byte `json:"data,omitempty"` } +// NewFileSystem initializes the FileSystem struct. +func NewFileSystem() *FileSystem { + return &FileSystem{ + filemap: map[string]File{}, + } +} + +// Get a file from the FileSystem. +func (fs *FileSystem) Get(filename string) ([]byte, error) { + if fs.filemap == nil { + fs.filemap = map[string]File{} + } + + // Legacy file map. + if file, ok := fs.filemap[filename]; ok { + if len(file.Data) > 0 { + return file.Data, nil + } + } + + // Check in the zipfile. + if fs.Zipfile != nil { + file, err := fs.Zipfile.Open(filename) + if err != nil { + return []byte{}, fmt.Errorf("%s: not in zipfile: %s", filename, err) + } + + bin, err := ioutil.ReadAll(file) + if err != nil { + return bin, fmt.Errorf("%s: couldn't read zipfile member: %s", filename, err) + } + + return bin, nil + } + + return []byte{}, fmt.Errorf("no such file") +} + +// Set a file into the FileSystem. Note: it will go into the legacy map +// structure until the next save to disk, at which point queued files +// are committed to ZIP. +func (fs *FileSystem) Set(filename string, data []byte) { + if fs.filemap == nil { + fs.filemap = map[string]File{} + } + + fs.filemap[filename] = File{ + Data: data, + } +} + +// Delete a file from the FileSystem. This will store zero bytes in the +// legacy file map structure to mark it for deletion. On the next save, +// filemap files with zero bytes skip the ZIP archive. +func (fs *FileSystem) Delete(filename string) { + if fs.filemap == nil { + fs.filemap = map[string]File{} + } + + fs.filemap[filename] = File{ + Data: []byte{}, + } +} + +// List files in the FileSystem, including the ZIP file. +// +// In the ZIP file, attachments are under the "assets/" prefix so this +// function won't mistakenly return chunks or level.json/doodad.json. +func (fs *FileSystem) List() []string { + var ( + result = []string{} + seen = map[string]interface{}{} + ) + + // List the legacy or recently modified files first. + if fs.filemap != nil { + for filename := range fs.filemap { + result = append(result, filename) + seen[filename] = nil + } + } + + // List the zipfile members. + if fs.Zipfile != nil { + for _, file := range fs.Zipfile.File { + if !strings.HasPrefix(file.Name, "assets/") { + continue + } + + if _, ok := seen[file.Name]; !ok { + result = append(result, file.Name) + seen[file.Name] = nil + } + } + } + + sort.Strings(result) + return result +} + +// ListPrefix returns a list of files starting with the prefix. +func (fs *FileSystem) ListPrefix(prefix string) []string { + var result = []string{} + for _, name := range fs.List() { + if strings.HasPrefix(name, prefix) { + result = append(result, name) + } + } + return result +} + +// UnmarshalJSON reads in a FileSystem from its legacy JSON representation. +func (fs *FileSystem) UnmarshalJSON(text []byte) error { + // Legacy format was a simple map[string]File. + var legacy map[string]File + err := json.Unmarshal(text, &legacy) + if err != nil { + return err + } + + fs.filemap = legacy + return nil +} + +// MigrateZipfile is called on save to write attached files to the ZIP +// file format. +func (fs *FileSystem) MigrateZipfile(zf *zip.Writer) error { + // Identify the files that we have marked for deletion. + var ( + filesDeleted = map[string]interface{}{} + filesZipped = map[string]interface{}{} + ) + if fs.filemap != nil { + for filename, data := range fs.filemap { + if len(data.Data) == 0 { + log.Info("FileSystem.MigrateZipfile: %s has become empty, remove from zip", filename) + filesDeleted[filename] = nil + } + } + } + + // Copy all COLD STORED files from the old Zipfile into the new Zipfile + // except for the ones marked for deletion OR the ones currently in the + // warm cache which will be written next. + if fs.Zipfile != nil { + log.Info("FileSystem.MigrateZipfile: Copying files from old zip to new zip") + for _, file := range fs.Zipfile.File { + if !strings.HasPrefix(file.Name, "assets/") { + continue + } + + if _, ok := filesDeleted[file.Name]; ok { + log.Debug("Skip copying attachment %s: was marked for deletion") + continue + } + + // Skip files currently in memory. + if fs.filemap != nil { + if _, ok := fs.filemap[file.Name]; ok { + log.Debug("Skip copying attachment %s: one is loaded in memory") + continue + } + } + + log.Debug("Copy zipfile attachment %s", file.Name) + filesZipped[file.Name] = nil + + if err := zf.Copy(file); err != nil { + return err + } + } + } + + // Export currently warmed up files to ZIP, these will be ones that + // were updated recently OR legacy files from an old level read. + if fs.filemap != nil { + log.Info("FileSystem.MigrateZipfile: has %d files in memory to write to ZIP", len(fs.filemap)) + for filename, file := range fs.filemap { + if _, ok := filesZipped[filename]; ok { + continue + } + + writer, err := zf.Create(filename) + if err != nil { + return err + } + + n, err := writer.Write(file.Data) + if err != nil { + return err + } + + log.Debug("Exported file to zip: %s (%d bytes)", filename, n) + } + } + + return nil +} + +//////////////// +// Level class methods for its filesystem access +//////////////// + // SetFile sets a file's data in the level. func (l *Level) SetFile(filename string, data []byte) { - if l.Files == nil { - l.Files = map[string]File{} - } - - l.Files[filename] = File{ - Data: data, - } + l.Files.Set(filename, data) } // GetFile looks up an embedded file. func (l *Level) GetFile(filename string) ([]byte, error) { if l.Files == nil { - l.Files = map[string]File{} + return []byte{}, errors.New("filesystem not initialized") } - - if result, ok := l.Files[filename]; ok { - return result.Data, nil - } - return []byte{}, errors.New("not found") + return l.Files.Get(filename) } // DeleteFile removes an embedded file. func (l *Level) DeleteFile(filename string) bool { - if l.Files == nil { - l.Files = map[string]File{} - } - - if _, ok := l.Files[filename]; ok { - delete(l.Files, filename) - return true - } - return false + l.Files.Delete(filename) + return true } // DeleteFiles removes all files beginning with the prefix. func (l *Level) DeleteFiles(prefix string) int { var count int - for filename := range l.Files { - if strings.HasPrefix(filename, prefix) { - delete(l.Files, filename) - count++ - } + for _, filename := range l.Files.ListPrefix(prefix) { + l.Files.Delete(filename) + count++ } return count } // ListFiles returns the list of all embedded file names, alphabetically. func (l *Level) ListFiles() []string { - var files []string - - if l == nil || l.Files == nil { - return files + if l == nil { + log.Error("Level.ListFiles() was called on a nil Level??") + return []string{} } - for name := range l.Files { - files = append(files, name) + if l.Files == nil { + log.Error("Level(%s).ListFiles: FileSystem not initialized", l.Title) + return []string{} } - - sort.Strings(files) - return files + return l.Files.List() } // ListFilesAt returns the list of files having a common prefix. func (l *Level) ListFilesAt(prefix string) []string { - var ( - files = l.ListFiles() - match = []string{} - ) - for _, name := range files { - if strings.HasPrefix(name, prefix) { - match = append(match, name) - } - } - return match + return l.Files.ListPrefix(prefix) } diff --git a/pkg/level/fmt_json.go b/pkg/level/fmt_json.go index 8ef002a..7b7bc28 100644 --- a/pkg/level/fmt_json.go +++ b/pkg/level/fmt_json.go @@ -114,6 +114,11 @@ func (m *Level) ToZipfile() ([]byte, error) { return nil, fmt.Errorf("MigrateZipfile: %s", err) } + // Migrate attached files to ZIP. + if err := m.Files.MigrateZipfile(zipper); err != nil { + return nil, fmt.Errorf("FileSystem.MigrateZipfile: %s", err) + } + // Write the header json. { header, err := m.AsJSON() @@ -204,6 +209,11 @@ func (m *Level) populateFromZipfile(data []byte) error { // Keep the zipfile reader handy. m.Zipfile = zf m.Chunker.Zipfile = zf + m.Files.Zipfile = zf + + // Re-inflate the level: ensures Actor instances get their IDs + // and everything is reloaded after saving the level. + m.Inflate() return err } diff --git a/pkg/level/types.go b/pkg/level/types.go index 55b0f11..a0de0f4 100644 --- a/pkg/level/types.go +++ b/pkg/level/types.go @@ -35,7 +35,7 @@ type Base struct { Zipfile *zip.Reader `json:"-"` // Every drawing type is able to embed other files inside of itself. - Files FileSystem `json:"files"` + Files *FileSystem `json:"files,omitempty"` } // Level is the container format for Doodle map drawings. @@ -81,6 +81,7 @@ func New() *Level { Version: 1, Title: "Untitled", Author: os.Getenv("USER"), + Files: NewFileSystem(), }, Chunker: NewChunker(balance.ChunkSize), Palette: &Palette{}, diff --git a/pkg/play_scene.go b/pkg/play_scene.go index 91d53a3..2f1ddb3 100644 --- a/pkg/play_scene.go +++ b/pkg/play_scene.go @@ -403,7 +403,7 @@ func (s *PlayScene) setupPlayer(playerCharacterFilename string) { // centerIn is optional, ignored if zero. func (s *PlayScene) installPlayerDoodad(filename string, spawn render.Point, centerIn render.Rect) { // Load in the player character. - player, err := doodads.LoadFile(filename) + player, err := doodads.LoadFromEmbeddable(filename, s.Level) if err != nil { log.Error("PlayScene.Setup: failed to load player doodad: %s", err) player = doodads.NewDummy(32) @@ -569,6 +569,12 @@ func (s *PlayScene) SetCheated() { } } +// GetCheated gives read-only access to tell if you have been cheating. However, by +// querying this in the dev console during gameplay, you would be marked as cheating. ;) +func (s *PlayScene) GetCheated() bool { + return s.cheated +} + // ShowEndLevelModal centralizes the EndLevel modal config. // This is the common handler function between easy methods such as // BeatLevel, FailLevel, and DieByFire. diff --git a/pkg/scripting/scripting.go b/pkg/scripting/scripting.go index 56cccca..82120c7 100644 --- a/pkg/scripting/scripting.go +++ b/pkg/scripting/scripting.go @@ -83,7 +83,7 @@ func (s *Supervisor) InstallScripts(level *level.Level) error { // The `name` is used to name the VM for debug logging. func (s *Supervisor) AddLevelScript(id string, name string) error { if _, ok := s.scripts[id]; ok { - return fmt.Errorf("duplicate actor ID %s in level", id) + return fmt.Errorf("AddLevelScript: duplicate actor ID '%s' in level", id) } s.scripts[id] = NewVM(fmt.Sprintf("%s#%s", name, id))