Merge pull request #56 from jwforres/fix_compilation_mem_bloat

Fix extreme memory bloat during compile time. Also, when strings are valid UTF-8 inline them directly as raw strings when in uncompressed mode.

release.go

@@ -5,10 +5,13 @@
 package bindata
 
 import (
+	"bytes"
 	"compress/gzip"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"os"
+	"unicode/utf8"
 )
 
 // writeRelease writes the release code file.
@@ -72,6 +75,19 @@ func writeReleaseAsset(w io.Writer, c *Config, asset *Asset) error {
 	}
 }
 
+// sanitize prepares a valid UTF-8 string as a raw string constant.
+// Based on https://code.google.com/p/go/source/browse/godoc/static/makestatic.go?repo=tools
+func sanitize(b []byte) []byte {
+	// Replace ` with `+"`"+`
+	b = bytes.Replace(b, []byte("`"), []byte("`+\"`\"+`"), -1)
+
+	// Replace BOM with `+"\xEF\xBB\xBF"+`
+	// (A BOM is valid UTF-8 but not permitted in Go source files.
+	// I wouldn't bother handling this, but for some insane reason
+	// jquery.js has a BOM somewhere in the middle.)
+	return bytes.Replace(b, []byte("\xEF\xBB\xBF"), []byte("`+\"\\xEF\\xBB\\xBF\"+`"), -1)
+}
+
 func header_compressed_nomemcopy(w io.Writer) error {
 	_, err := fmt.Fprintf(w, `import (
 	"bytes"
@@ -202,14 +218,12 @@ func %s() ([]byte, error) {
 }
 
 func compressed_memcopy(w io.Writer, asset *Asset, r io.Reader) error {
-	_, err := fmt.Fprintf(w, `func %s() ([]byte, error) {
-	return bindata_read([]byte{`, asset.Func)
-
+	_, err := fmt.Fprintf(w, `var _%s = []byte("`, asset.Func)
 	if err != nil {
-		return nil
+		return err
 	}
 
-	gz := gzip.NewWriter(&ByteWriter{Writer: w})
+	gz := gzip.NewWriter(&StringWriter{Writer: w})
 	_, err = io.Copy(gz, r)
 	gz.Close()
 
@@ -217,13 +231,16 @@ func compressed_memcopy(w io.Writer, asset *Asset, r io.Reader) error {
 		return err
 	}
 
-	_, err = fmt.Fprintf(w, `
-	},
+	_, err = fmt.Fprintf(w, `")
+
+func %s() ([]byte, error) {
+	return bindata_read(
+		_%s,
 		%q,
 	)
 }
 
-`, asset.Name)
+`, asset.Func, asset.Func, asset.Name)
 	return err
 }
 
@@ -252,21 +269,27 @@ func %s() ([]byte, error) {
 }
 
 func uncompressed_memcopy(w io.Writer, asset *Asset, r io.Reader) error {
-	_, err := fmt.Fprintf(w, `func %s() ([]byte, error) {
-	return []byte{`, asset.Func)
+	_, err := fmt.Fprintf(w, `var _%s = []byte(`, asset.Func)
 	if err != nil {
 		return err
 	}
 
-	_, err = io.Copy(&ByteWriter{Writer: w}, r)
+	b, err := ioutil.ReadAll(r)
 	if err != nil {
 		return err
 	}
-
-	_, err = fmt.Fprintf(w, `
-	}, nil
+	if utf8.Valid(b) {
+		fmt.Fprintf(w, "`%s`", sanitize(b))
+	} else {
+		fmt.Fprintf(w, "%q", b)
 	}
 
-`)
+	_, err = fmt.Fprintf(w, `)
+
+func %s() ([]byte, error) {
+	return _%s, nil
+}
+
+`, asset.Func, asset.Func)
 	return err
 }