diff --git a/cmd/doodad/commands/convert.go b/cmd/doodad/commands/convert.go index a0c0cb4..634b1b0 100644 --- a/cmd/doodad/commands/convert.go +++ b/cmd/doodad/commands/convert.go @@ -168,10 +168,7 @@ func imageToDrawing(c *cli.Context, chroma render.Color, inputFiles []string, ou img := images[i] log.Info("Converting extra layer %d", i) _, chunker := imageToChunker(img, chroma, palette, chunkSize) - doodad.Layers = append(doodad.Layers, doodads.Layer{ - Name: toLayerName(inputFiles[i]), - Chunker: chunker, - }) + doodad.AddLayer(toLayerName(inputFiles[i]), chunker) } } diff --git a/cmd/doodad/commands/edit_doodad.go b/cmd/doodad/commands/edit_doodad.go index 18157ed..3d9fb21 100644 --- a/cmd/doodad/commands/edit_doodad.go +++ b/cmd/doodad/commands/edit_doodad.go @@ -59,6 +59,10 @@ func init() { Name: "unlock", Usage: "remove the write-lock on the level file", }, + &cli.BoolFlag{ + Name: "touch", + Usage: "simply load and re-save the doodad, to migrate it to a zipfile", + }, }, Action: func(c *cli.Context) error { if c.NArg() < 1 { @@ -94,6 +98,11 @@ func editDoodad(c *cli.Context, filename string) error { * Update level properties * ***************************/ + if c.Bool("touch") { + log.Info("Just touching and resaving the file") + modified = true + } + if c.String("title") != "" { dd.Title = c.String("title") log.Info("Set title: %s", dd.Title) diff --git a/cmd/doodad/commands/edit_level.go b/cmd/doodad/commands/edit_level.go index 351914f..a1cffcc 100644 --- a/cmd/doodad/commands/edit_level.go +++ b/cmd/doodad/commands/edit_level.go @@ -59,6 +59,10 @@ func init() { Name: "remove-actor", Usage: "Remove all instances of the actor from the level. Value is their filename or UUID.", }, + &cli.BoolFlag{ + Name: "touch", + Usage: "simply load and re-save the level, to migrate it to a zipfile", + }, }, Action: func(c *cli.Context) error { if c.NArg() < 1 { @@ -94,6 +98,11 @@ func editLevel(c *cli.Context, filename string) error { * Update level properties * ***************************/ + if c.Bool("touch") { + log.Info("Just touching and resaving the file") + modified = true + } + if c.String("title") != "" { lvl.Title = c.String("title") log.Info("Set title: %s", lvl.Title) diff --git a/cmd/doodad/commands/show.go b/cmd/doodad/commands/show.go index 6ed99a8..e4bfd48 100644 --- a/cmd/doodad/commands/show.go +++ b/cmd/doodad/commands/show.go @@ -92,9 +92,16 @@ func showLevel(c *cli.Context, filename string) error { } } + // Is it a new zipfile format? + var fileType = "json or gzip" + if lvl.Zipfile != nil { + fileType = "zipfile" + } + fmt.Printf("===== Level: %s =====\n", filename) fmt.Println("Headers:") + fmt.Printf(" File format: %s\n", fileType) fmt.Printf(" File version: %d\n", lvl.Version) fmt.Printf(" Game version: %s\n", lvl.GameVersion) fmt.Printf(" Level title: %s\n", lvl.Title) @@ -170,9 +177,16 @@ func showDoodad(c *cli.Context, filename string) error { return nil } + // Is it a new zipfile format? + var fileType = "json or gzip" + if dd.Zipfile != nil { + fileType = "zipfile" + } + fmt.Printf("===== Doodad: %s =====\n", filename) fmt.Println("Headers:") + fmt.Printf(" File format: %s\n", fileType) fmt.Printf(" File version: %d\n", dd.Version) fmt.Printf(" Game version: %s\n", dd.GameVersion) fmt.Printf(" Doodad title: %s\n", dd.Title) diff --git a/pkg/balance/boolprops.go b/pkg/balance/boolprops.go index fa0552f..dd997e7 100644 --- a/pkg/balance/boolprops.go +++ b/pkg/balance/boolprops.go @@ -41,10 +41,6 @@ var Boolprops = map[string]Boolprop{ Get: func() bool { return usercfg.Current.HorizontalToolbars }, Set: func(v bool) { usercfg.Current.HorizontalToolbars = v }, }, - "compress-drawings": { - Get: func() bool { return CompressDrawings }, - Set: func(v bool) { CompressDrawings = v }, - }, "eager-render": { Get: func() bool { return EagerRenderLevelChunks }, Set: func(v bool) { EagerRenderLevelChunks = v }, diff --git a/pkg/balance/numbers.go b/pkg/balance/numbers.go index aa4f288..0b6eac4 100644 --- a/pkg/balance/numbers.go +++ b/pkg/balance/numbers.go @@ -6,6 +6,15 @@ import ( "git.kirsle.net/go/render" ) +// Format for level and doodad files. +type Format int + +const ( + FormatJSON Format = iota // v0: plain json files + FormatGZip // v1: gzip compressed json files + FormatZipfile // v2: zip archive with external chunks +) + // Numbers. var ( // Window dimensions. @@ -87,7 +96,14 @@ var ( EmbeddedWallpaperBasePath = "assets/wallpapers/" // File formats: save new levels and doodads gzip compressed - CompressDrawings = true + DrawingFormat = FormatZipfile + + // Zipfile drawings: max size of the LRU cache for loading chunks from + // a zip file. Normally the chunker discards chunks not loaded in a + // recent tick, but when iterating the full level this limits the max + // size of loaded chunks before some will be freed to make room. + // 0 = do not cap the cache. + ChunkerLRUCacheMax = 0 // Play Mode Touchscreen controls. PlayModeIdleTimeout = 2200 * time.Millisecond diff --git a/pkg/doodads/doodad.go b/pkg/doodads/doodad.go index 6e3cb72..6a216dc 100644 --- a/pkg/doodads/doodad.go +++ b/pkg/doodads/doodad.go @@ -52,6 +52,25 @@ func New(size int) *Doodad { } } +// AddLayer adds a new layer to the doodad. Call this rather than appending +// your own layer so it points the Zipfile and layer number in. The chunker +// is optional - pass nil and a new blank chunker is created. +func (d *Doodad) AddLayer(name string, chunker *level.Chunker) Layer { + if chunker == nil { + chunker = level.NewChunker(d.ChunkSize()) + } + + layer := Layer{ + Name: name, + Chunker: chunker, + } + layer.Chunker.Layer = len(d.Layers) + d.Layers = append(d.Layers, layer) + d.Inflate() + + return layer +} + // Teardown cleans up texture cache memory when the doodad is no longer needed by the game. func (d *Doodad) Teardown() { var ( @@ -101,7 +120,8 @@ func (d *Doodad) Rect() render.Rect { // Inflate attaches the pixels to their swatches after loading from disk. func (d *Doodad) Inflate() { d.Palette.Inflate() - for _, layer := range d.Layers { + for i, layer := range d.Layers { + layer.Chunker.Layer = i layer.Chunker.Inflate(d.Palette) } } diff --git a/pkg/doodads/fmt_zipfile.go b/pkg/doodads/fmt_zipfile.go new file mode 100644 index 0000000..1baeb2e --- /dev/null +++ b/pkg/doodads/fmt_zipfile.go @@ -0,0 +1,110 @@ +package doodads + +import ( + "archive/zip" + "bytes" + "encoding/json" + "fmt" + + "git.kirsle.net/apps/doodle/pkg/balance" + "git.kirsle.net/apps/doodle/pkg/log" +) + +// ToZipfile serializes the doodad into zipfile format. +func (d *Doodad) ToZipfile() ([]byte, error) { + fh := bytes.NewBuffer([]byte{}) + zipper := zip.NewWriter(fh) + defer zipper.Close() + + // Migrate the Chunker caches into the zipfile. + for _, layer := range d.Layers { + if err := layer.Chunker.MigrateZipfile(zipper); err != nil { + return nil, fmt.Errorf("MigrateZipfile: %s", err) + } + } + + // Write the header json. + { + header, err := d.AsJSON() + if err != nil { + return nil, err + } + + writer, err := zipper.Create("doodad.json") + if err != nil { + return nil, err + } + + if n, err := writer.Write(header); err != nil { + return nil, err + } else { + log.Debug("Written doodad.json to zipfile: %d bytes", n) + } + } + + zipper.Close() + + // Refresh our Zipfile reader from the zipper we just wrote. + bin := fh.Bytes() + if err := d.ReloadZipfile(bin); err != nil { + log.Error("ReloadZipfile: %s", err) + } + + return fh.Bytes(), nil +} + +// FromZipfile reads a doodad from zipfile format. +func FromZipfile(data []byte) (*Doodad, error) { + var ( + doodad = New(balance.DoodadSize) + err = doodad.populateFromZipfile(data) + ) + return doodad, err +} + +// ReloadZipfile re-reads the level's zipfile after a write. +func (d *Doodad) ReloadZipfile(data []byte) error { + return d.populateFromZipfile(data) +} + +// Common function between FromZipfile and ReloadZipFile. +func (d *Doodad) populateFromZipfile(data []byte) error { + var ( + buf = bytes.NewReader(data) + zf *zip.Reader + decoder *json.Decoder + ) + + zf, err := zip.NewReader(buf, buf.Size()) + if err != nil { + return err + } + + // Read the doodad.json. + file, err := zf.Open("doodad.json") + if err != nil { + return err + } + + decoder = json.NewDecoder(file) + err = decoder.Decode(d) + + // Keep the zipfile reader handy. + d.Zipfile = zf + for i, layer := range d.Layers { + layer.Chunker.Layer = i + layer.Chunker.Zipfile = zf + } + + return err +} + +// Loop may be called each loop to allow the level to maintain its +// memory usage, e.g., for chunks not requested recently from a zipfile +// level to free those from RAM. +func (d *Doodad) Loop() error { + for _, layer := range d.Layers { + layer.Chunker.FreeCaches() + } + return nil +} diff --git a/pkg/doodads/json.go b/pkg/doodads/json.go index 6a50e98..3a32117 100644 --- a/pkg/doodads/json.go +++ b/pkg/doodads/json.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "io/ioutil" + "net/http" "path/filepath" "git.kirsle.net/apps/doodle/pkg/balance" @@ -17,10 +18,20 @@ import ( // and the return value is gz bytes and not the raw JSON. func (d *Doodad) ToJSON() ([]byte, error) { // Gzip compressing? - if balance.CompressDrawings { + if balance.DrawingFormat == balance.FormatGZip { return d.ToGzip() } + // Zipfile? + if balance.DrawingFormat == balance.FormatZipfile { + return d.ToZipfile() + } + + return d.AsJSON() +} + +// AsJSON returns it just as JSON without any fancy gzip/zip magic. +func (d *Doodad) AsJSON() ([]byte, error) { out := bytes.NewBuffer([]byte{}) encoder := json.NewEncoder(out) if usercfg.Current.JSONIndent { @@ -48,6 +59,12 @@ func FromJSON(filename string, data []byte) (*Doodad, error) { } else { doodad = gzd } + } else if http.DetectContentType(data) == "application/zip" { + if zipdoodad, err := FromZipfile(data); err != nil { + return nil, err + } else { + doodad = zipdoodad + } } // Inflate the chunk metadata to map the pixels to their palette indexes. diff --git a/pkg/editor_scene.go b/pkg/editor_scene.go index 283cfb6..b2c6382 100644 --- a/pkg/editor_scene.go +++ b/pkg/editor_scene.go @@ -289,7 +289,7 @@ func (s *EditorScene) Loop(d *Doodle, ev *event.State) error { } if s.UI.Canvas != nil { inside, outside := s.UI.Canvas.LoadUnloadMetrics() - *s.debLoadingViewport = fmt.Sprintf("%d in %d out", inside, outside) + *s.debLoadingViewport = fmt.Sprintf("%d in %d out %d cached", inside, outside, s.UI.Canvas.Chunker().CacheSize()) } // Has the window been resized? diff --git a/pkg/editor_ui_popups.go b/pkg/editor_ui_popups.go index 39a8b70..7e00c63 100644 --- a/pkg/editor_ui_popups.go +++ b/pkg/editor_ui_popups.go @@ -354,11 +354,10 @@ func (u *EditorUI) SetupPopups(d *Doodle) { u.layersWindow.Show() }, OnAddLayer: func() { - layer := doodads.Layer{ - Name: fmt.Sprintf("layer %d", len(scene.Doodad.Layers)), - Chunker: level.NewChunker(scene.DoodadSize), - } - scene.Doodad.Layers = append(scene.Doodad.Layers, layer) + layer := scene.Doodad.AddLayer( + fmt.Sprintf("layer %d", len(scene.Doodad.Layers)), + nil, + ) log.Info("Added new layer: %d %s", len(scene.Doodad.Layers), layer.Name) diff --git a/pkg/level/chunk.go b/pkg/level/chunk.go index ab4d234..f74c6b5 100644 --- a/pkg/level/chunk.go +++ b/pkg/level/chunk.go @@ -226,7 +226,7 @@ func (c *Chunk) Teardown() int { if c.texture != nil { c.texture.Free() - c.texture = nil + c.texture = nil // NPE <- here freed++ } diff --git a/pkg/level/chunk_map.go b/pkg/level/chunk_map.go index 60a3f09..2a391e4 100644 --- a/pkg/level/chunk_map.go +++ b/pkg/level/chunk_map.go @@ -4,22 +4,28 @@ import ( "encoding/json" "errors" "fmt" + "sync" "git.kirsle.net/go/render" ) // MapAccessor implements a chunk accessor by using a map of points to their // palette indexes. This is the simplest accessor and is best for sparse chunks. -type MapAccessor map[render.Point]*Swatch +type MapAccessor struct { + grid map[render.Point]*Swatch + mu sync.RWMutex +} // NewMapAccessor initializes a MapAccessor. -func NewMapAccessor() MapAccessor { - return MapAccessor{} +func NewMapAccessor() *MapAccessor { + return &MapAccessor{ + grid: map[render.Point]*Swatch{}, + } } // Inflate the sparse swatches from their palette indexes. -func (a MapAccessor) Inflate(pal *Palette) error { - for point, swatch := range a { +func (a *MapAccessor) Inflate(pal *Palette) error { + for point, swatch := range a.grid { if swatch.IsSparse() { // Replace this with the correct swatch from the palette. if swatch.paletteIndex >= len(pal.Swatches) { @@ -29,19 +35,24 @@ func (a MapAccessor) Inflate(pal *Palette) error { len(pal.Swatches), ) } - a[point] = pal.Swatches[swatch.paletteIndex] + + a.mu.Lock() + a.grid[point] = pal.Swatches[swatch.paletteIndex] // <- concurrent write + a.mu.Unlock() } } return nil } // Len returns the current size of the map, or number of pixels registered. -func (a MapAccessor) Len() int { - return len(a) +func (a *MapAccessor) Len() int { + a.mu.RLock() + defer a.mu.RUnlock() + return len(a.grid) } // IterViewport returns a channel to loop over pixels in the viewport. -func (a MapAccessor) IterViewport(viewport render.Rect) <-chan Pixel { +func (a *MapAccessor) IterViewport(viewport render.Rect) <-chan Pixel { pipe := make(chan Pixel) go func() { for px := range a.Iter() { @@ -55,24 +66,29 @@ func (a MapAccessor) IterViewport(viewport render.Rect) <-chan Pixel { } // Iter returns a channel to loop over all points in this chunk. -func (a MapAccessor) Iter() <-chan Pixel { +func (a *MapAccessor) Iter() <-chan Pixel { pipe := make(chan Pixel) go func() { - for point, swatch := range a { + a.mu.Lock() + for point, swatch := range a.grid { pipe <- Pixel{ X: point.X, Y: point.Y, Swatch: swatch, } } + a.mu.Unlock() close(pipe) }() return pipe } // Get a pixel from the map. -func (a MapAccessor) Get(p render.Point) (*Swatch, error) { - pixel, ok := a[p] +func (a *MapAccessor) Get(p render.Point) (*Swatch, error) { + a.mu.Lock() + defer a.mu.Unlock() + + pixel, ok := a.grid[p] // <- concurrent read and write if !ok { return nil, errors.New("no pixel") } @@ -80,17 +96,23 @@ func (a MapAccessor) Get(p render.Point) (*Swatch, error) { } // Set a pixel on the map. -func (a MapAccessor) Set(p render.Point, sw *Swatch) error { - a[p] = sw +func (a *MapAccessor) Set(p render.Point, sw *Swatch) error { + a.mu.Lock() + defer a.mu.Unlock() + a.grid[p] = sw return nil } // Delete a pixel from the map. -func (a MapAccessor) Delete(p render.Point) error { - if _, ok := a[p]; ok { - delete(a, p) +func (a *MapAccessor) Delete(p render.Point) error { + a.mu.Lock() + defer a.mu.Unlock() + + if _, ok := a.grid[p]; ok { + delete(a.grid, p) return nil } + return errors.New("pixel was not there") } @@ -99,9 +121,12 @@ func (a MapAccessor) Delete(p render.Point) error { // When serialized, the key is the "X,Y" coordinate and the value is the // swatch index of the Palette, rather than redundantly serializing out the // Swatch object for every pixel. -func (a MapAccessor) MarshalJSON() ([]byte, error) { +func (a *MapAccessor) MarshalJSON() ([]byte, error) { + a.mu.Lock() + defer a.mu.Unlock() + dict := map[string]int{} - for point, sw := range a { + for point, sw := range a.grid { dict[point.String()] = sw.Index() } @@ -110,7 +135,10 @@ func (a MapAccessor) MarshalJSON() ([]byte, error) { } // UnmarshalJSON to convert the chunk map back from JSON. -func (a MapAccessor) UnmarshalJSON(b []byte) error { +func (a *MapAccessor) UnmarshalJSON(b []byte) error { + a.mu.Lock() + defer a.mu.Unlock() + var dict map[string]int err := json.Unmarshal(b, &dict) if err != nil { @@ -122,7 +150,7 @@ func (a MapAccessor) UnmarshalJSON(b []byte) error { if err != nil { return fmt.Errorf("MapAccessor.UnmarshalJSON: %s", err) } - a[point] = NewSparseSwatch(index) + a.grid[point] = NewSparseSwatch(index) } return nil diff --git a/pkg/level/chunker.go b/pkg/level/chunker.go index b4a8ec6..24b9f37 100644 --- a/pkg/level/chunker.go +++ b/pkg/level/chunker.go @@ -1,11 +1,15 @@ package level import ( + "archive/zip" "encoding/json" "fmt" "math" + "sync" + "git.kirsle.net/apps/doodle/pkg/balance" "git.kirsle.net/apps/doodle/pkg/log" + "git.kirsle.net/apps/doodle/pkg/shmem" "git.kirsle.net/go/render" ) @@ -13,8 +17,35 @@ import ( // provides the API to interact with the pixels using their absolute coordinates // while abstracting away the underlying details. type Chunker struct { - Size int `json:"size"` - Chunks ChunkMap `json:"chunks"` + // Layer is optional for the caller, levels use only 0 and + // doodads use them for frames. When chunks are exported to + // zipfile the Layer keeps them from overlapping. + Layer int + Size int `json:"size"` + + // A Zipfile reference for new-style levels and doodads which + // keep their chunks in external parts of a zip file. + Zipfile *zip.Reader `json:"-"` + + // Chunks, oh boy. + // The v1 drawing format had all the chunks in the JSON file. + // New drawings write them to zips. Legacy drawings can be converted + // simply by loading and resaving: their Chunks loads from JSON and + // is committed to zipfile on save. This makes Chunks also a good + // cache even when we have a zipfile to fall back on. + Chunks ChunkMap `json:"chunks"` + chunkMu sync.RWMutex + + // If we have a zipfile, only keep chunks warm in memory if they + // are actively wanted by the game. + lastTick uint64 // NOTE: tracks from shmem.Tick + chunkRequestsThisTick map[render.Point]interface{} + requestsN1 map[render.Point]interface{} + requestsN2 map[render.Point]interface{} + requestMu sync.Mutex + + // The palette reference from first call to Inflate() + pal *Palette } // NewChunker creates a new chunk manager with a given chunk size. @@ -22,6 +53,10 @@ func NewChunker(size int) *Chunker { return &Chunker{ Size: size, Chunks: ChunkMap{}, + + chunkRequestsThisTick: map[render.Point]interface{}{}, + requestsN1: map[render.Point]interface{}{}, + requestsN2: map[render.Point]interface{}{}, } } @@ -29,6 +64,10 @@ func NewChunker(size int) *Chunker { // Sparse Swatches (which have only their palette index, from the file format // on disk) to connect references to the swatches in the palette. func (c *Chunker) Inflate(pal *Palette) error { + c.pal = pal + + c.chunkMu.RLock() + defer c.chunkMu.RUnlock() for coord, chunk := range c.Chunks { chunk.Point = coord chunk.Size = c.Size @@ -70,11 +109,70 @@ func (c *Chunker) IterViewport(viewport render.Rect) <-chan Pixel { // IterChunks returns a channel to iterate over all chunks in the drawing. func (c *Chunker) IterChunks() <-chan render.Point { - pipe := make(chan render.Point) + var ( + pipe = make(chan render.Point) + sent = map[render.Point]interface{}{} + ) + go func() { + c.chunkMu.RLock() + + // Send the chunk coords we have in working memory. + // v1 levels: had all their chunks there in their JSON data + // v2 levels: chunks are in zipfile, cached ones are here for point := range c.Chunks { + sent[point] = nil pipe <- point } + + c.chunkMu.RUnlock() + + // If we have a zipfile, send any remaining chunks that are + // in colder storage. + if c.Zipfile != nil { + for _, point := range ChunksInZipfile(c.Zipfile, c.Layer) { + if _, ok := sent[point]; ok { + continue // Already sent from active memory + } + pipe <- point + } + } + + close(pipe) + }() + return pipe +} + +/* +IterChunksThemselves iterates all chunks in the drawing rather than coords. + +Note: this will mark every chunk as "touched" this frame, so in a zipfile +level will load ALL chunks into memory. +*/ +func (c *Chunker) IterChunksThemselves() <-chan *Chunk { + pipe := make(chan *Chunk) + go func() { + for coord := range c.IterChunks() { + if chunk, ok := c.GetChunk(coord); ok { + pipe <- chunk + } + } + close(pipe) + }() + return pipe +} + +// IterCachedChunks iterates ONLY over the chunks currently cached in memory, +// e.g. so they can be torn down without loading extra chunks by looping normally. +func (c *Chunker) IterCachedChunks() <-chan *Chunk { + pipe := make(chan *Chunk) + go func() { + c.chunkMu.RLock() + defer c.chunkMu.RUnlock() + + for _, chunk := range c.Chunks { + pipe <- chunk + } close(pipe) }() return pipe @@ -128,7 +226,7 @@ func (c *Chunker) IterViewportChunks(viewport render.Rect) <-chan render.Point { func (c *Chunker) IterPixels() <-chan Pixel { pipe := make(chan Pixel) go func() { - for _, chunk := range c.Chunks { + for chunk := range c.IterChunksThemselves() { for px := range chunk.Iter() { pipe <- px } @@ -166,7 +264,7 @@ func (c *Chunker) WorldSizePositive() render.Rect { // Bounds returns the boundary points of the lowest and highest chunk which // have any data in them. func (c *Chunker) Bounds() (low, high render.Point) { - for coord := range c.Chunks { + for coord := range c.IterChunks() { if coord.X < low.X { low.X = coord.X } @@ -185,16 +283,168 @@ func (c *Chunker) Bounds() (low, high render.Point) { return low, high } -// GetChunk gets a chunk at a certain position. Returns false if not found. +/* +GetChunk gets a chunk at a certain position. Returns false if not found. + +This should be the centralized function to request a Chunk from the Chunker +(or IterChunksThemselves). On old-style levels all of the chunks were just +in memory as part of the JSON struct, in Zip files we can load/unload them +at will from external files. +*/ func (c *Chunker) GetChunk(p render.Point) (*Chunk, bool) { + // It's currently cached in memory? + c.chunkMu.RLock() chunk, ok := c.Chunks[p] - return chunk, ok + c.chunkMu.RUnlock() + + if ok { + // An empty chunk? We hang onto these until save time to commit + // the empty chunk to ZIP. + if chunk.Len() == 0 { + return nil, false + } + + c.logChunkAccess(p, chunk) // for the LRU cache + return chunk, ok + } + + // Hit the zipfile for it. + if c.Zipfile != nil { + if chunk, err := ChunkFromZipfile(c.Zipfile, c.Layer, p); err == nil { + log.Debug("GetChunk(%s) cache miss, read from zip", p) + c.SetChunk(p, chunk) // cache it + c.logChunkAccess(p, chunk) // for the LRU cache + if c.pal != nil { + chunk.Point = p + chunk.Size = c.Size + chunk.Inflate(c.pal) + } + return chunk, true + } + } + + // Is our chunk cache getting too full? e.g. on full level + // sweeps where a whole zip file's worth of chunks are scanned. + if balance.ChunkerLRUCacheMax > 0 && len(c.Chunks) > balance.ChunkerLRUCacheMax { + log.Error("Chunks in memory (%d) exceeds LRU cache cap of %d, freeing random chunks") + c.chunkMu.Lock() + defer c.chunkMu.Unlock() + + var ( + i = 0 + limit = len(c.Chunks) - balance.ChunkerLRUCacheMax + ) + for coord := range c.Chunks { + if i < limit { + delete(c.Chunks, coord) + } + i++ + } + } + + return nil, false +} + +// LRU cache for chunks from zipfiles: log which chunks were accessed +// this tick, so they can be compared to the tick prior, and then freed +// up after that. +func (c *Chunker) logChunkAccess(p render.Point, chunk *Chunk) { + // Record this point. + c.requestMu.Lock() + if c.chunkRequestsThisTick == nil { + c.chunkRequestsThisTick = map[render.Point]interface{}{} + } + c.chunkRequestsThisTick[p] = nil + c.requestMu.Unlock() +} + +// FreeCaches unloads chunks that have not been requested in 2 frames. +// +// Only on chunkers that have zipfiles, old-style levels without zips +// wouldn't be able to restore their chunks otherwise! Returns -1 if +// no Zipfile, otherwise number of chunks freed. +func (c *Chunker) FreeCaches() int { + if c.Zipfile == nil { + return -1 + } + + var thisTick = shmem.Tick + + // Very first tick this chunker has seen? + if c.lastTick == 0 { + c.lastTick = thisTick + } + + // A new tick? + if (thisTick-c.lastTick)%4 == 0 { + c.requestMu.Lock() + c.chunkMu.Lock() + defer c.requestMu.Unlock() + defer c.chunkMu.Unlock() + + var ( + requestsThisTick = c.chunkRequestsThisTick + requestsN2 = c.requestsN2 + delete_coords = []render.Point{} + ) + + // Chunks not requested this last tick, unload from the cache. + for coord := range requestsN2 { + // Old point not requested recently? + if _, ok := requestsThisTick[coord]; !ok { + delete_coords = append(delete_coords, coord) + } + } + + for _, coord := range delete_coords { + c.FreeChunk(coord) + } + + // Rotate the cached ticks and clean the slate. + c.requestsN2 = c.requestsN1 + c.requestsN1 = requestsThisTick + c.chunkRequestsThisTick = map[render.Point]interface{}{} + + c.lastTick = thisTick + + return len(delete_coords) + } + + return 0 +} + +// SetChunk writes the chunk into the cache dict and nothing more. +// +// This function should be the singular writer to the chunk cache. +func (c *Chunker) SetChunk(p render.Point, chunk *Chunk) { + c.chunkMu.Lock() + c.Chunks[p] = chunk + c.chunkMu.Unlock() + + c.logChunkAccess(p, chunk) +} + +// FreeChunk unloads a chunk from active memory for zipfile-backed levels. +// +// Not thread safe: it is assumed the caller has the lock on c.Chunks. +func (c *Chunker) FreeChunk(p render.Point) bool { + if c.Zipfile == nil { + return false + } + + // Don't delete empty chunks, hang on until next zipfile save. + if chunk, ok := c.Chunks[p]; ok && chunk.Len() == 0 { + return false + } + + delete(c.Chunks, p) + return true } // Redraw marks every chunk as dirty and invalidates all their texture caches, // forcing the drawing to re-generate from scratch. func (c *Chunker) Redraw() { - for _, chunk := range c.Chunks { + for chunk := range c.IterChunksThemselves() { chunk.SetDirty() } } @@ -203,7 +453,7 @@ func (c *Chunker) Redraw() { // the whole drawing for smooth gameplay rather than chunks lazy rendering as // they enter the screen. func (c *Chunker) Prerender() { - for _, chunk := range c.Chunks { + for chunk := range c.IterChunksThemselves() { _ = chunk.CachedBitmap(render.Invisible) } } @@ -218,7 +468,7 @@ func (c *Chunker) PrerenderN(n int) (remaining int) { modified int // number modified this call ) - for _, chunk := range c.Chunks { + for chunk := range c.IterChunksThemselves() { total++ if chunk.bitmap != nil { totalRendered++ @@ -241,7 +491,7 @@ func (c *Chunker) PrerenderN(n int) (remaining int) { func (c *Chunker) Get(p render.Point) (*Swatch, error) { // Compute the chunk coordinate. coord := c.ChunkCoordinate(p) - if chunk, ok := c.Chunks[coord]; ok { + if chunk, ok := c.GetChunk(coord); ok { return chunk.Get(p) } return nil, fmt.Errorf("no chunk %s exists for point %s", coord, p) @@ -250,12 +500,12 @@ func (c *Chunker) Get(p render.Point) (*Swatch, error) { // Set a pixel at the given coordinate. func (c *Chunker) Set(p render.Point, sw *Swatch) error { coord := c.ChunkCoordinate(p) - chunk, ok := c.Chunks[coord] + chunk, ok := c.GetChunk(coord) if !ok { chunk = NewChunk() - c.Chunks[coord] = chunk chunk.Point = coord chunk.Size = c.Size + c.SetChunk(coord, chunk) } return chunk.Set(p, sw) @@ -281,9 +531,8 @@ func (c *Chunker) SetRect(r render.Rect, sw *Swatch) error { // Delete a pixel at the given coordinate. func (c *Chunker) Delete(p render.Point) error { coord := c.ChunkCoordinate(p) - defer c.pruneChunk(coord) - if chunk, ok := c.Chunks[coord]; ok { + if chunk, ok := c.GetChunk(coord); ok { return chunk.Delete(p) } return fmt.Errorf("no chunk %s exists for point %s", coord, p) @@ -308,17 +557,6 @@ func (c *Chunker) DeleteRect(r render.Rect) error { return nil } -// pruneChunk will remove an empty chunk from the chunk map, called after -// delete operations. -func (c *Chunker) pruneChunk(coord render.Point) { - if chunk, ok := c.Chunks[coord]; ok { - if chunk.Len() == 0 { - log.Info("Chunker.pruneChunk: %s has become empty", coord) - delete(c.Chunks, coord) - } - } -} - // ChunkCoordinate computes a chunk coordinate from an absolute coordinate. func (c *Chunker) ChunkCoordinate(abs render.Point) render.Point { if c.Size == 0 { diff --git a/pkg/level/chunker_zipfile.go b/pkg/level/chunker_zipfile.go new file mode 100644 index 0000000..47764d3 --- /dev/null +++ b/pkg/level/chunker_zipfile.go @@ -0,0 +1,196 @@ +package level + +import ( + "archive/zip" + "fmt" + "io/ioutil" + "regexp" + "strconv" + + "git.kirsle.net/apps/doodle/pkg/log" + "git.kirsle.net/go/render" +) + +// Zipfile interactions for the Chunker to cache and manage which +// chunks of large levels need be in active memory. + +var ( + zipChunkfileRegexp = regexp.MustCompile(`^chunks/(\d+)/(.+?)\.json$`) +) + +// MigrateZipfile is called on save to migrate old-style ChunkMap +// chunks into external zipfile members and free up space in the +// master Level or Doodad struct. +func (c *Chunker) MigrateZipfile(zf *zip.Writer) error { + // Identify if any chunks in active memory had been completely erased. + var ( + erasedChunks = map[render.Point]interface{}{} + chunksZipped = map[render.Point]interface{}{} + ) + for coord, chunk := range c.Chunks { + if chunk.Len() == 0 { + log.Info("Chunker.MigrateZipfile: %s has become empty, remove from zip", coord) + erasedChunks[coord] = nil + } + } + + // Copy all COLD STORED chunks from our original zipfile into the new one. + // These are chunks that are NOT actively loaded (those are written next), + // and erasedChunks are not written to the zipfile at all. + if c.Zipfile != nil { + log.Info("MigrateZipfile: Copying chunk files from old zip to new zip") + for _, file := range c.Zipfile.File { + m := zipChunkfileRegexp.FindStringSubmatch(file.Name) + if len(m) > 0 { + mLayer, _ := strconv.Atoi(m[1]) + coord := m[2] + + // Not our layer, not our problem. + if mLayer != c.Layer { + continue + } + + point, err := render.ParsePoint(coord) + if err != nil { + return err + } + + // Don't create zip files for empty (0 pixel) chunks. + if _, ok := erasedChunks[point]; ok { + log.Debug("Skip copying %s: chunk is empty", coord) + continue + } + + // Don't ever write duplicate files. + if _, ok := chunksZipped[point]; ok { + log.Debug("Skip copying duplicate chunk %s", coord) + continue + } + chunksZipped[point] = nil + + // Don't copy the chunks we have currently in memory: those + // are written next. Apparently zip files are allowed to + // have duplicate named members! + if _, ok := c.Chunks[point]; ok { + log.Debug("Skip chunk %s (in memory)", coord) + continue + } + + log.Info("Copy existing chunk %s", file.Name) + if err := zf.Copy(file); err != nil { + return err + } + } + } + } else { + log.Warn("Chunker.MigrateZipfile: the drawing did not give me a zipfile!") + } + + if len(c.Chunks) == 0 { + return nil + } + + log.Info("MigrateZipfile: chunker has %d in memory, exporting to zipfile", len(c.Chunks)) + + // Flush in-memory chunks out to zipfile. + for coord, chunk := range c.Chunks { + filename := fmt.Sprintf("chunks/%d/%s.json", c.Layer, coord.String()) + log.Info("Flush in-memory chunks to %s", filename) + chunk.ToZipfile(zf, filename) + } + + // Flush the chunkmap out. + // TODO: do similar to move old attached files (wallpapers) too + c.Chunks = ChunkMap{} + + return nil +} + +// ClearChunkCache completely flushes the ChunkMap from memory. BE CAREFUL. +// If the level is a Zipfile the chunks will reload as needed, but old style +// levels this will nuke the whole drawing! +func (c *Chunker) ClearChunkCache() { + c.chunkMu.Lock() + c.Chunks = ChunkMap{} + c.chunkMu.Unlock() +} + +// CacheSize returns the number of chunks in memory. +func (c *Chunker) CacheSize() int { + return len(c.Chunks) +} + +// ToZipfile writes just a chunk's data into a zipfile. +func (c *Chunk) ToZipfile(zf *zip.Writer, filename string) error { + writer, err := zf.Create(filename) + if err != nil { + return err + } + + json, err := c.MarshalJSON() + if err != nil { + return err + } + + n, err := writer.Write(json) + if err != nil { + return err + } + + log.Debug("Written chunk to zipfile: %s (%d bytes)", filename, n) + return nil +} + +// ChunkFromZipfile loads a chunk from a zipfile. +func ChunkFromZipfile(zf *zip.Reader, layer int, coord render.Point) (*Chunk, error) { + filename := fmt.Sprintf("chunks/%d/%s.json", layer, coord) + + file, err := zf.Open(filename) + if err != nil { + return nil, err + } + + bin, err := ioutil.ReadAll(file) + if err != nil { + return nil, err + } + + var chunk = NewChunk() + err = chunk.UnmarshalJSON(bin) + if err != nil { + return nil, err + } + + return chunk, nil +} + +// ChunksInZipfile returns the list of chunk coordinates in a zipfile. +func ChunksInZipfile(zf *zip.Reader, layer int) []render.Point { + var ( + result = []render.Point{} + sLayer = fmt.Sprintf("%d", layer) + ) + + for _, file := range zf.File { + m := zipChunkfileRegexp.FindStringSubmatch(file.Name) + if len(m) > 0 { + var ( + mLayer = m[1] + mPoint = m[2] + ) + + // Not our layer? + if mLayer != sLayer { + continue + } + + if point, err := render.ParsePoint(mPoint); err == nil { + result = append(result, point) + } else { + log.Error("ChunksInZipfile: file '%s' didn't parse as a point: %s", file.Name, err) + } + } + } + + return result +} diff --git a/pkg/level/fmt_json.go b/pkg/level/fmt_json.go index ca64118..8ef002a 100644 --- a/pkg/level/fmt_json.go +++ b/pkg/level/fmt_json.go @@ -1,12 +1,13 @@ package level import ( + "archive/zip" "bytes" "compress/gzip" "encoding/json" - "errors" "fmt" "io/ioutil" + "net/http" "git.kirsle.net/apps/doodle/pkg/balance" "git.kirsle.net/apps/doodle/pkg/log" @@ -32,8 +33,14 @@ func FromJSON(filename string, data []byte) (*Level, error) { } else { m = gzmap } + } else if http.DetectContentType(data) == "application/zip" { + if zipmap, err := FromZipfile(data); err != nil { + return nil, err + } else { + m = zipmap + } } else { - return nil, errors.New("invalid file format") + return nil, fmt.Errorf("invalid file format") } // Fill in defaults. @@ -55,10 +62,20 @@ func FromJSON(filename string, data []byte) (*Level, error) { // "gzip supported" in their descriptions. func (m *Level) ToJSON() ([]byte, error) { // Gzip compressing? - if balance.CompressDrawings { + if balance.DrawingFormat == balance.FormatGZip { return m.ToGzip() } + // Zipfile? + if balance.DrawingFormat == balance.FormatZipfile { + return m.ToZipfile() + } + + return m.AsJSON() +} + +// AsJSON returns it just as JSON without any fancy gzip/zip magic. +func (m *Level) AsJSON() ([]byte, error) { out := bytes.NewBuffer([]byte{}) encoder := json.NewEncoder(out) if usercfg.Current.JSONIndent { @@ -83,6 +100,50 @@ func (m *Level) ToGzip() ([]byte, error) { return handle.Bytes(), err } +// ToZipfile serializes the level as a ZIP archive and also migrates +// data loaded from an older save into the new zip format. +func (m *Level) ToZipfile() ([]byte, error) { + // If we do not have a Zipfile yet, migrate legacy data into one. + // if m.Zipfile == nil { + fh := bytes.NewBuffer([]byte{}) + zipper := zip.NewWriter(fh) + defer zipper.Close() + + // Migrate any legacy Chunker data into external files in the zip. + if err := m.Chunker.MigrateZipfile(zipper); err != nil { + return nil, fmt.Errorf("MigrateZipfile: %s", err) + } + + // Write the header json. + { + header, err := m.AsJSON() + if err != nil { + return nil, err + } + + writer, err := zipper.Create("level.json") + if err != nil { + return nil, fmt.Errorf("zipping index.js: %s", err) + } + + if n, err := writer.Write(header); err != nil { + return nil, err + } else { + log.Debug("Written level.json to zipfile: %s bytes", n) + } + } + + zipper.Close() + + // Refresh our Zipfile reader from the zipper we just wrote. + bin := fh.Bytes() + if err := m.ReloadZipfile(bin); err != nil { + log.Error("ReloadZipfile: %s", err) + } + + return fh.Bytes(), nil +} + // FromGzip deserializes a gzip compressed level JSON. func FromGzip(data []byte) (*Level, error) { // This function works, do not touch. @@ -104,6 +165,49 @@ func FromGzip(data []byte) (*Level, error) { return level, nil } +// FromZipfile reads a level in zipfile format. +func FromZipfile(data []byte) (*Level, error) { + var ( + level = New() + err = level.populateFromZipfile(data) + ) + return level, err +} + +// ReloadZipfile re-reads the level's zipfile after a write. +func (m *Level) ReloadZipfile(data []byte) error { + return m.populateFromZipfile(data) +} + +// Common function between FromZipfile and ReloadZipFile. +func (m *Level) populateFromZipfile(data []byte) error { + var ( + buf = bytes.NewReader(data) + zf *zip.Reader + decoder *json.Decoder + ) + + zf, err := zip.NewReader(buf, buf.Size()) + if err != nil { + return err + } + + // Read the level.json. + file, err := zf.Open("level.json") + if err != nil { + return err + } + + decoder = json.NewDecoder(file) + err = decoder.Decode(m) + + // Keep the zipfile reader handy. + m.Zipfile = zf + m.Chunker.Zipfile = zf + + return err +} + // LoadJSON loads a map from JSON file (gzip supported). func LoadJSON(filename string) (*Level, error) { data, err := ioutil.ReadFile(filename) @@ -128,3 +232,11 @@ func (m *Level) WriteJSON(filename string) error { return nil } + +// Loop may be called each loop to allow the level to maintain its +// memory usage, e.g., for chunks not requested recently from a zipfile +// level to free those from RAM. +func (m *Level) Loop() error { + m.Chunker.FreeCaches() + return nil +} diff --git a/pkg/level/fmt_maintenance.go b/pkg/level/fmt_maintenance.go index 076a8d5..de51671 100644 --- a/pkg/level/fmt_maintenance.go +++ b/pkg/level/fmt_maintenance.go @@ -4,19 +4,6 @@ import "git.kirsle.net/apps/doodle/pkg/log" // Maintenance functions for the file format on disk. -// PruneChunks cleans up any level chunks that have no pixel data. -func (m *Level) PruneChunks() int { - var count int - for coord, chunk := range m.Chunker.Chunks { - if chunk.Len() == 0 { - log.Info("PruneChunks: %d has no pixels", coord) - delete(m.Chunker.Chunks, coord) - count++ - } - } - return count -} - // PruneLinks cleans up any Actor Links that can not be resolved in the // level data. For example, if actors were linked in Edit Mode and one // actor is deleted leaving a broken link. diff --git a/pkg/level/fmt_readwrite.go b/pkg/level/fmt_readwrite.go index 92e662e..d8abcf7 100644 --- a/pkg/level/fmt_readwrite.go +++ b/pkg/level/fmt_readwrite.go @@ -1,7 +1,6 @@ package level import ( - "errors" "fmt" "io/ioutil" "runtime" @@ -82,9 +81,8 @@ func LoadFile(filename string) (*Level, error) { return level, nil } else { log.Warn(err.Error()) + return nil, err } - - return nil, errors.New("invalid file type") } // WriteFile saves a level to disk in the user's config directory. @@ -98,7 +96,6 @@ func (m *Level) WriteFile(filename string) error { m.GameVersion = branding.Version // Maintenance functions, clean up cruft before save. - m.PruneChunks() m.PruneLinks() bin, err := m.ToJSON() diff --git a/pkg/level/types.go b/pkg/level/types.go index 87a64c4..55b0f11 100644 --- a/pkg/level/types.go +++ b/pkg/level/types.go @@ -1,6 +1,7 @@ package level import ( + "archive/zip" "encoding/json" "fmt" "os" @@ -26,6 +27,13 @@ type Base struct { Author string `json:"author"` Locked bool `json:"locked"` + // v2 level format: zip files with external chunks. + // (v0 was json text, v1 was gzip compressed json text). + // The game must load levels created using the previous + // formats, they will not have a Zipfile and will have + // Chunkers in memory from their (gz) json. + Zipfile *zip.Reader `json:"-"` + // Every drawing type is able to embed other files inside of itself. Files FileSystem `json:"files"` } @@ -95,12 +103,11 @@ func (m *Level) Teardown() { textures int ) - for coord := range m.Chunker.IterChunks() { - if chunk, ok := m.Chunker.GetChunk(coord); ok { - freed := chunk.Teardown() - chunks++ - textures += freed - } + // Free any CACHED chunks' memory. + for chunk := range m.Chunker.IterCachedChunks() { + freed := chunk.Teardown() + chunks++ + textures += freed } log.Debug("Teardown level (%s): Freed %d textures across %d level chunks", m.Title, textures, chunks) diff --git a/pkg/play_scene.go b/pkg/play_scene.go index 17629ec..91d53a3 100644 --- a/pkg/play_scene.go +++ b/pkg/play_scene.go @@ -660,7 +660,7 @@ func (s *PlayScene) Loop(d *Doodle, ev *event.State) error { *s.debViewport = s.drawing.Viewport().String() *s.debScroll = s.drawing.Scroll.String() inside, outside := s.drawing.LoadUnloadMetrics() - *s.debLoadUnload = fmt.Sprintf("%d in %d out", inside, outside) + *s.debLoadUnload = fmt.Sprintf("%d in %d out %d cached", inside, outside, s.drawing.Chunker().CacheSize()) // Update the timer. s.timerLabel.Text = savegame.FormatDuration(time.Since(s.startTime)) diff --git a/pkg/uix/canvas.go b/pkg/uix/canvas.go index 93bee64..122d679 100644 --- a/pkg/uix/canvas.go +++ b/pkg/uix/canvas.go @@ -272,7 +272,14 @@ func (w *Canvas) Loop(ev *event.State) error { _ = w.loopConstrainScroll() // Every so often, eager-load/unload chunk bitmaps to save on memory. - w.LoadUnloadChunks() + if w.level != nil { + // Unloads bitmaps and textures every N frames... + w.LoadUnloadChunks() + + // Unloads chunks themselves (from zipfile levels) that aren't + // recently accessed. + w.chunks.FreeCaches() + } // Remove any actors that were destroyed the previous tick. var newActors []*Actor diff --git a/pkg/uix/canvas_memory.go b/pkg/uix/canvas_memory.go index 5a27ade..f261b5a 100644 --- a/pkg/uix/canvas_memory.go +++ b/pkg/uix/canvas_memory.go @@ -45,19 +45,18 @@ func (w *Canvas) LoadUnloadChunks() { wg.Add(1) go func(i int) { for coord := range chunks { - if chunk, ok := w.level.Chunker.GetChunk(coord); ok { - chunk := chunk - - if _, ok := chunksInside[coord]; ok { - // Preload its bitmap image. + if _, ok := chunksInside[coord]; ok { + // This chunk is INSIDE our viewport, preload its bitmap. + if chunk, ok := w.level.Chunker.GetChunk(coord); ok { _ = chunk.CachedBitmap(render.Invisible) resultInside++ - } else { - // Unload its bitmap and texture. - chunksTeardown = append(chunksTeardown, chunk) - resultOutside++ + continue } } + + // Chunks outside the viewport, we won't load them and + // the Chunker will flush them out to (zip) file. + resultOutside++ } wg.Done() }(i)