LoadUnloadChunk for Memory Optimization

Instead of the loadscreen eager-loading ALL level chunks to Go Images, only
load the chunks within the "LoadingViewport" - which is the on-screen
Viewport plus a margin of chunks off the screen edges.

During gameplay, every few ticks, reevaluate which chunks are inside or
outside the LoadingViewport; for chunks outside, free their SDL2 textures
and free their cached bitmaps to keep overall memory usage down. The
AzulianTag-Forest level now stays under 200 Textures at any given time
and the loadscreen goes faster as it doesn't have to load every chunk's
images up front.

The LoadUnloadChunk feature can be turned on/off with feature flags. If
disabled the old behavior is restored: loadscreen loads all images and
the LoadUnloadChunks function is not run.

Other changes:

* loadscreen: do not free textures in the Hide() function as this runs on
  a different goroutine and may break. The 4 wallpaper textures are OK
  to keep in memory anyway, the loadscreen is reused often!
* Free more leaked textures: on the Inventory frame and when an actor
  calls Self.Destroy()
* Stop leaking goroutines in the PubSub feature of the doodad script
  engine; scripting.Supervisor.Teardown() sends a stop signal to all
  scripts to clean up neatly. Canvas.Destroy() tears down its scripting
  supervisor automatically.

go.mod

@@ -5,7 +5,7 @@ go 1.16
 require (
 	git.kirsle.net/go/audio v0.0.0-20201121073642-65068820cbc0
 	git.kirsle.net/go/log v0.0.0-20200902035305-70ac2848949b
-	git.kirsle.net/go/render v0.0.0-20220409212556-df69b8e52508
+	git.kirsle.net/go/render v0.0.0-20220410192720-c0c2d05619bc
 	git.kirsle.net/go/ui v0.0.0-20220409211920-3b653e503c5a
 	github.com/aichaos/rivescript-go v0.3.1
 	github.com/dgrijalva/jwt-go v3.2.0+incompatible
@@ -14,7 +14,7 @@ require (
 	github.com/fsnotify/fsnotify v1.4.9
 	github.com/gen2brain/dlgs v0.0.0-20211108104213-bade24837f0b
 	github.com/google/uuid v1.3.0
-	github.com/gopherjs/gopherjs v0.0.0-20220221023154-0b2280d3ff96 // indirect
+	github.com/gopherjs/gopherjs v0.0.0-20220410123724-9e86199038b0 // indirect
 	github.com/kirsle/configdir v0.0.0-20170128060238-e45d2f54772f
 	github.com/robertkrimen/otto v0.0.0-20211024170158-b87d35c0b86f // indirect
 	github.com/tomnomnom/xtermcolor v0.0.0-20160428124646-b78803f00a7e // indirect

go.sum

@@ -42,8 +42,8 @@ git.kirsle.net/go/audio v0.0.0-20201121073642-65068820cbc0/go.mod h1:RD2Kyiy2lga
 git.kirsle.net/go/log v0.0.0-20200902035305-70ac2848949b h1:TDxEEWOJqMzsu9JW8/QgmT1lgQ9WD2KWlb2lKN/Ql2o=
 git.kirsle.net/go/log v0.0.0-20200902035305-70ac2848949b/go.mod h1:jl+Qr58W3Op7OCxIYIT+b42jq8xFncJXzPufhrvza7Y=
 git.kirsle.net/go/render v0.0.0-20211231003948-9e640ab5c3da/go.mod h1:ss7pvZbGWrMaDuZwyUTjV9+T0AJwAkxZZHwMFsvHrkk=
-git.kirsle.net/go/render v0.0.0-20220409212556-df69b8e52508 h1:cL5XdHJgU81XFfsyXUO+6NxLcZ11IMaV/ZhiC5799cA=
-git.kirsle.net/go/render v0.0.0-20220409212556-df69b8e52508/go.mod h1:ss7pvZbGWrMaDuZwyUTjV9+T0AJwAkxZZHwMFsvHrkk=
+git.kirsle.net/go/render v0.0.0-20220410192720-c0c2d05619bc h1:tYbgVPFoADy9yQaH7+MeHXDATqTkSHoFXvZLcr9lwyc=
+git.kirsle.net/go/render v0.0.0-20220410192720-c0c2d05619bc/go.mod h1:ss7pvZbGWrMaDuZwyUTjV9+T0AJwAkxZZHwMFsvHrkk=
 git.kirsle.net/go/ui v0.0.0-20220409211920-3b653e503c5a h1:7pmqBcIfxOxCMCvnCS0XWM9NeXbzKTSQfiJc2IKJW+k=
 git.kirsle.net/go/ui v0.0.0-20220409211920-3b653e503c5a/go.mod h1:cWD/tl2OUj0jWUQxW7BcapxiOFmnCPkrfzJt2tOiD1E=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
@@ -169,8 +169,8 @@ github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
-github.com/gopherjs/gopherjs v0.0.0-20220221023154-0b2280d3ff96 h1:QJq7UBOuoynsywLk+aC75rC2Cbi2+lQRDaLaizhA+fA=
-github.com/gopherjs/gopherjs v0.0.0-20220221023154-0b2280d3ff96/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k=
+github.com/gopherjs/gopherjs v0.0.0-20220410123724-9e86199038b0 h1:fWY+zXdWhvWndXqnMj4SyC/vi8sK508OjhGCtMzsA9M=
+github.com/gopherjs/gopherjs v0.0.0-20220410123724-9e86199038b0/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k=
 github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
 github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
 github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=

pkg/balance/feature_flags.go

@@ -20,6 +20,12 @@ var Feature = feature{
 
 	// Reassign an existing level's palette to a different builtin.
 	ChangePalette: true,
+
+	// LoadUnloadChunk feature to better optimize memory. Set it to false and the
+	// loadscreen will eager load all chunk bitmaps (stable, but uses a lot of
+	// memory), set true and the Canvas will load/unload bitmaps + free SDL textures
+	// for chunks falling outside the LoadingViewport (new, maybe unstable).
+	LoadUnloadChunk: true,
 }
 
 // FeaturesOn turns on all feature flags, from CLI --experimental option.
@@ -33,4 +39,5 @@ type feature struct {
 	ChangePalette     bool
 	EmbeddableDoodads bool
 	ViewportWindow    bool
+	LoadUnloadChunk   bool
 }

pkg/balance/numbers.go

@@ -113,6 +113,10 @@ var (
 	// `boolProp eager-render false` and the loadscreen will go quicker cuz it won't
 	// load the whole entire level. Maybe useful to explore memory issues.
 	EagerRenderLevelChunks = true
+
+	// Number of chunks margin outside the Canvas Viewport for the LoadingViewport.
+	LoadingViewportMarginChunks        = 2
+	CanvasLoadUnloadModuloTicks uint64 = 4
 )
 
 // Edit Mode Values

pkg/editor_scene.go

@@ -47,6 +47,7 @@ type EditorScene struct {
 	debTool            *string
 	debSwatch          *string
 	debWorldIndex      *string
+	debLoadingViewport *string
 
 	// Last saved filename by the user.
 	filename string
@@ -65,10 +66,12 @@ func (s *EditorScene) Setup(d *Doodle) error {
 	s.debTool = new(string)
 	s.debSwatch = new(string)
 	s.debWorldIndex = new(string)
+	s.debLoadingViewport = new(string)
 	customDebugLabels = []debugLabel{
 		{"Pixel:", s.debWorldIndex},
 		{"Tool:", s.debTool},
 		{"Swatch:", s.debSwatch},
+		{"Chunks:", s.debLoadingViewport},
 	}
 
 	// Initialize autosave time.
@@ -278,11 +281,16 @@ func (s *EditorScene) Loop(d *Doodle, ev *event.State) error {
 	*s.debTool = s.UI.Canvas.Tool.String()
 	*s.debSwatch = "???"
 	*s.debWorldIndex = s.UI.Canvas.WorldIndexAt(s.UI.cursor).String()
+	*s.debLoadingViewport = "???"
 
 	// Safely...
 	if s.UI.Canvas.Palette != nil && s.UI.Canvas.Palette.ActiveSwatch != nil {
 		*s.debSwatch = s.UI.Canvas.Palette.ActiveSwatch.Name
 	}
+	if s.UI.Canvas != nil {
+		inside, outside := s.UI.Canvas.LoadUnloadMetrics()
+		*s.debLoadingViewport = fmt.Sprintf("%d in %d out", inside, outside)
+	}
 
 	// Has the window been resized?
 	if ev.WindowResized {

pkg/fps.go

@@ -61,8 +61,7 @@ func (d *Doodle) DrawDebugOverlay() {
 	// Get the size of cached SDL2 textures at the render engine level.
 	var texCount = "n/a"
 	if sdl, ok := d.Engine.(*sdl.Renderer); ok {
-		gotex, sdltex := sdl.CountTextures()
-		texCount = fmt.Sprintf("%d img, %d sdl", gotex, sdltex)
+		texCount = fmt.Sprintf("%d", sdl.CountTextures())
 	}
 
 	var (
@@ -73,7 +72,7 @@ func (d *Doodle) DrawDebugOverlay() {
 			"FPS:",
 			"Scene:",
 			"Mouse:",
-			"Textures:",
+			"Tex:",
 		}
 		values = []string{
 			fmt.Sprintf("%d   %s", fpsCurrent, framesSkipped),

pkg/level/chunk.go

@@ -220,6 +220,10 @@ func (c *Chunk) ToBitmap(mask render.Color) image.Image {
 func (c *Chunk) Teardown() int {
 	var freed int
 
+	if c.bitmap != nil {
+		c.bitmap = nil
+	}
+
 	if c.texture != nil {
 		c.texture.Free()
 		c.texture = nil

pkg/modal/loadscreen/loadscreen.go

@@ -87,9 +87,15 @@ func Resized() {
 	}
 }
 
-// Hide the loading screen.
+/*
+Hide the loading screen.
+
+NOTICE: the loadscreen is hidden on an async goroutine and it is NOT SAFE to clean up
+textures used by the wallpaper images, but this is OK because the loadscreen uses the
+same wallpaper every time and is called many times during gameplay, it can hold its
+textures.
+*/
 func Hide() {
-	canvas.Destroy() // cleanup wallpaper textures
 	visible = false
 }
 
@@ -225,6 +231,11 @@ func Loop(windowSize render.Rect, e render.Engine) {
 // loading screen and will set the Progress percent based on the total number
 // of chunks vs. chunks remaining to pre-cache bitmaps from.
 func PreloadAllChunkBitmaps(chunker *level.Chunker) {
+	// If we're using the smarter (experimental) chunk loader, return.
+	if balance.Feature.LoadUnloadChunk {
+		return
+	}
+
 	loadChunksTarget := len(chunker.Chunks)
 
 	// Skipping the eager rendering of chunks?

pkg/play_scene.go

@@ -62,6 +62,7 @@ type PlayScene struct {
 	debViewport   *string
 	debScroll     *string
 	debWorldIndex *string
+	debLoadUnload *string
 
 	// Player character
 	Player              *uix.Actor
@@ -141,11 +142,13 @@ func (s *PlayScene) setupAsync(d *Doodle) error {
 	s.debViewport = new(string)
 	s.debScroll = new(string)
 	s.debWorldIndex = new(string)
+	s.debLoadUnload = new(string)
 	customDebugLabels = []debugLabel{
 		{"Pixel:", s.debWorldIndex},
 		{"Player:", s.debPosition},
 		{"Viewport:", s.debViewport},
 		{"Scroll:", s.debScroll},
+		{"Chunks:", s.debLoadUnload},
 	}
 
 	// Initialize the "Edit Map" button.
@@ -656,6 +659,8 @@ func (s *PlayScene) Loop(d *Doodle, ev *event.State) error {
 	*s.debPosition = s.Player.Position().String() + " vel " + s.Player.Velocity().String()
 	*s.debViewport = s.drawing.Viewport().String()
 	*s.debScroll = s.drawing.Scroll.String()
+	inside, outside := s.drawing.LoadUnloadMetrics()
+	*s.debLoadUnload = fmt.Sprintf("%d in %d out", inside, outside)
 
 	// Update the timer.
 	s.timerLabel.Text = savegame.FormatDuration(time.Since(s.startTime))
@@ -912,5 +917,11 @@ func (s *PlayScene) Destroy() error {
 	// their bitmaps cached and will regen the textures as needed.
 	s.drawing.Destroy()
 
+	// Free inventory doodad textures.
+	for _, can := range s.invenDoodads {
+		log.Info("Destroy inventory doodad: %s", can)
+		can.Destroy()
+	}
+
 	return nil
 }

pkg/scripting/pubsub.go

@@ -31,7 +31,13 @@ func RegisterPublishHooks(s *Supervisor, vm *VM) {
 			}
 		}()
 
-		for msg := range vm.Inbound {
+		// Watch the Inbound channel for PubSub messages and the stop channel for Teardown.
+		for {
+			select {
+			case <-vm.stop:
+				log.Info("JavaScript VM %s stopping PubSub goroutine", vm.Name)
+				return
+			case msg := <-vm.Inbound:
 				vm.muSubscribe.Lock()
 
 				if _, ok := vm.subscribe[msg.Name]; ok {
@@ -45,6 +51,7 @@ func RegisterPublishHooks(s *Supervisor, vm *VM) {
 
 				vm.muSubscribe.Unlock()
 			}
+		}
 	}()
 
 	// Register the Message.Subscribe and Message.Publish functions.

pkg/scripting/scripting.go

@@ -29,6 +29,14 @@ func NewSupervisor() *Supervisor {
 	}
 }
 
+// Teardown the supervisor to clean up goroutines.
+func (s *Supervisor) Teardown() {
+	log.Info("scripting.Teardown(): stop all (%d) scripts", len(s.scripts))
+	for _, vm := range s.scripts {
+		vm.stop <- true
+	}
+}
+
 // Loop the supervisor to invoke timer events in any running scripts.
 func (s *Supervisor) Loop() error {
 	now := time.Now()

pkg/scripting/vm.go

@@ -25,6 +25,7 @@ type VM struct {
 	//     messages.
 	Inbound     chan Message
 	Outbound    []chan Message
+	stop        chan bool
 	subscribe   map[string][]goja.Value // Subscribed message handlers by name.
 	muSubscribe sync.RWMutex
 
@@ -45,6 +46,7 @@ func NewVM(name string) *VM {
 		// Pub/sub structs.
 		Inbound:   make(chan Message),
 		Outbound:  []chan Message{},
+		stop:      make(chan bool, 1),
 		subscribe: map[string][]goja.Value{},
 	}
 	vm.Events = NewEvents(vm.vm)

pkg/uix/canvas.go

@@ -113,6 +113,10 @@ type Canvas struct {
 	scrollStartAt   render.Point // Cursor point at beginning of pan
 	scrollWasAt     render.Point // copy of Scroll at beginning of pan
 	scrollLastDelta render.Point // multitouch spam
+
+	// LoadUnloadChunks metrics for the debug overlay.
+	loadUnloadInside  int
+	loadUnloadOutside int
 }
 
 // NewCanvas initializes a Canvas widget.
@@ -177,6 +181,10 @@ func (w *Canvas) Destroy() {
 			log.Debug("%s.Destroy(): freed %d wallpaper textures", w, freed)
 		}
 	}
+
+	if w.scripting != nil {
+		w.scripting.Teardown()
+	}
 }
 
 // Load initializes the Canvas using an existing Palette and Grid.
@@ -263,10 +271,14 @@ func (w *Canvas) Loop(ev *event.State) error {
 	}
 	_ = w.loopConstrainScroll()
 
+	// Every so often, eager-load/unload chunk bitmaps to save on memory.
+	w.LoadUnloadChunks()
+
 	// Remove any actors that were destroyed the previous tick.
 	var newActors []*Actor
 	for _, a := range w.actors {
 		if a.flagDestroy {
+			a.Canvas.Destroy()
 			continue
 		}
 		newActors = append(newActors, a)
@@ -289,6 +301,7 @@ func (w *Canvas) Loop(ev *event.State) error {
 			return w.loopEditable(ev)
 		}
 	}
+
 	return nil
 }
 
@@ -330,6 +343,40 @@ func (w *Canvas) ViewportRelative() render.Rect {
 	}
 }
 
+// LoadingViewport is the viewport of chunks that ought to be preloaded and
+// ready to display soon. It is the Viewport of chunks on screen + a margin
+// of neighboring chunks outside the screen.
+//
+// For memory optimization, chunks falling inside this viewport have their
+// Go image.Image rendered and cached ready to convert to an SDL2 Texture
+// when they come on screen. Chunks outside of the LoadingViewport can be
+// unloaded (textures and images freed) to keep memory consumption on large
+// levels under control.
+func (w *Canvas) LoadingViewport() render.Rect {
+	var (
+		chunkSize int
+		vp        = w.Viewport()
+		margin    = balance.LoadingViewportMarginChunks
+	)
+
+	// This function is meant for levels only, but..
+	if w.level != nil {
+		chunkSize = w.level.Chunker.Size
+	} else if w.doodad != nil {
+		chunkSize = w.doodad.ChunkSize()
+	} else {
+		chunkSize = balance.ChunkSize
+		log.Error("Canvas.LoadingViewport: no drawing to get chunk size from, default to %d", chunkSize)
+	}
+
+	return render.Rect{
+		X: vp.X - chunkSize*margin,
+		Y: vp.Y - chunkSize*margin,
+		W: vp.W + chunkSize*margin,
+		H: vp.H + chunkSize*margin,
+	}
+}
+
 // WorldIndexAt returns the World Index that corresponds to a Screen Pixel
 // on the screen. If the screen pixel is the mouse coordinate (relative to
 // the application window) this will return the World Index of the pixel below

pkg/uix/canvas_memory.go

@@ -0,0 +1,91 @@
+package uix
+
+import (
+	"runtime"
+	"sync"
+
+	"git.kirsle.net/apps/doodle/pkg/balance"
+	"git.kirsle.net/apps/doodle/pkg/level"
+	"git.kirsle.net/apps/doodle/pkg/log"
+	"git.kirsle.net/apps/doodle/pkg/shmem"
+	"git.kirsle.net/go/render"
+)
+
+// Memory optimization features of the Canvas.
+
+/*
+LoadUnloadChunks optimizes memory for (level) canvases by warming up chunk images
+that fall within the LoadingViewport and freeing chunks that are outside of it.
+*/
+func (w *Canvas) LoadUnloadChunks() {
+	if w.level == nil || shmem.Tick%balance.CanvasLoadUnloadModuloTicks != 0 || !balance.Feature.LoadUnloadChunk {
+		return
+	}
+
+	var (
+		vp             = w.LoadingViewport()
+		chunks         = make(chan render.Point)
+		chunksInside   = map[render.Point]interface{}{}
+		chunksTeardown = []*level.Chunk{}
+		cores          = runtime.NumCPU()
+		wg             sync.WaitGroup
+
+		// Collect metrics for the debug overlay.
+		resultInside  int
+		resultOutside int
+	)
+
+	// Collect the chunks that are inside the viewport so we know which ones are not.
+	for chunk := range w.level.Chunker.IterViewportChunks(vp) {
+		chunksInside[chunk] = nil
+	}
+
+	// Spawn background goroutines to process the chunks quickly.
+	for i := 0; i < cores; i++ {
+		wg.Add(1)
+		go func(i int) {
+			for coord := range chunks {
+				if chunk, ok := w.level.Chunker.GetChunk(coord); ok {
+					chunk := chunk
+
+					if _, ok := chunksInside[coord]; ok {
+						// Preload its bitmap image.
+						_ = chunk.CachedBitmap(render.Invisible)
+						resultInside++
+					} else {
+						// Unload its bitmap and texture.
+						chunksTeardown = append(chunksTeardown, chunk)
+						resultOutside++
+					}
+				}
+			}
+			wg.Done()
+		}(i)
+	}
+
+	for chunk := range w.level.Chunker.IterChunks() {
+		chunks <- chunk
+	}
+	close(chunks)
+	wg.Wait()
+
+	// Tear down the SDL2 textures of chunks to free.
+	for i, chunk := range chunksTeardown {
+		if chunk == nil {
+			log.Error("LoadUnloadChunks: chunksTeardown#%d was nil??", i)
+			continue
+		}
+
+		chunk.Teardown()
+	}
+
+	// Export the metrics for the debug overlay.
+	w.loadUnloadInside = resultInside
+	w.loadUnloadOutside = resultOutside
+}
+
+// LoadUnloadMetrics returns the canvas's stored metrics from the LoadUnloadChunks
+// function, for the debug overlay.
+func (w *Canvas) LoadUnloadMetrics() (inside, outside int) {
+	return w.loadUnloadInside, w.loadUnloadOutside
+}