From 2c5bf09f1bed590cfa8c2f8e1e9f0b2f564a2e80 Mon Sep 17 00:00:00 2001 From: Jessica Forrester Date: Thu, 20 Nov 2014 10:10:11 -0500 Subject: [PATCH] Fix extreme memory bloat during compile time. When using uncompressed mode show raw UTF-8 strings for better diffs. --- release.go | 51 +++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 37 insertions(+), 14 deletions(-) diff --git a/release.go b/release.go index 2ffd29a..527d928 100644 --- a/release.go +++ b/release.go @@ -5,10 +5,13 @@ package bindata import ( + "bytes" "compress/gzip" "fmt" "io" + "io/ioutil" "os" + "unicode/utf8" ) // writeRelease writes the release code file. @@ -72,6 +75,19 @@ func writeReleaseAsset(w io.Writer, c *Config, asset *Asset) error { } } +// sanitize prepares a valid UTF-8 string as a raw string constant. +// Based on https://code.google.com/p/go/source/browse/godoc/static/makestatic.go?repo=tools +func sanitize(b []byte) []byte { + // Replace ` with `+"`"+` + b = bytes.Replace(b, []byte("`"), []byte("`+\"`\"+`"), -1) + + // Replace BOM with `+"\xEF\xBB\xBF"+` + // (A BOM is valid UTF-8 but not permitted in Go source files. + // I wouldn't bother handling this, but for some insane reason + // jquery.js has a BOM somewhere in the middle.) + return bytes.Replace(b, []byte("\xEF\xBB\xBF"), []byte("`+\"\\xEF\\xBB\\xBF\"+`"), -1) +} + func header_compressed_nomemcopy(w io.Writer) error { _, err := fmt.Fprintf(w, `import ( "bytes" @@ -202,14 +218,12 @@ func %s() ([]byte, error) { } func compressed_memcopy(w io.Writer, asset *Asset, r io.Reader) error { - _, err := fmt.Fprintf(w, `func %s() ([]byte, error) { - return bindata_read([]byte{`, asset.Func) - + _, err := fmt.Fprintf(w, `var _%s = []byte("`, asset.Func) if err != nil { - return nil + return err } - gz := gzip.NewWriter(&ByteWriter{Writer: w}) + gz := gzip.NewWriter(&StringWriter{Writer: w}) _, err = io.Copy(gz, r) gz.Close() @@ -217,13 +231,16 @@ func compressed_memcopy(w io.Writer, asset *Asset, r io.Reader) error { return err } - _, err = fmt.Fprintf(w, ` - }, + _, err = fmt.Fprintf(w, `") + +func %s() ([]byte, error) { + return bindata_read( + _%s, %q, ) } -`, asset.Name) +`, asset.Func, asset.Func, asset.Name) return err } @@ -252,21 +269,27 @@ func %s() ([]byte, error) { } func uncompressed_memcopy(w io.Writer, asset *Asset, r io.Reader) error { - _, err := fmt.Fprintf(w, `func %s() ([]byte, error) { - return []byte{`, asset.Func) + _, err := fmt.Fprintf(w, `var _%s = []byte(`, asset.Func) if err != nil { return err } - _, err = io.Copy(&ByteWriter{Writer: w}, r) + b, err := ioutil.ReadAll(r) if err != nil { return err } + if utf8.Valid(b) { + fmt.Fprintf(w, "`%s`", sanitize(b)) + } else { + fmt.Fprintf(w, "%q", b) + } - _, err = fmt.Fprintf(w, ` - }, nil + _, err = fmt.Fprintf(w, `) + +func %s() ([]byte, error) { + return _%s, nil } -`) +`, asset.Func, asset.Func) return err }