Combine the old and new code generation methods. We can select the .rodata hack described in issue #1 by supplying the -m commandline flag. The default code generation mode is the old one. While it uses more memory, it is a safer version and offers no problems when used on platforms that restrict usage of the unsafe and reflect packages. Additionally I did some cleanup and refactoring of the code. Bumped version to 1.0.0

README.md

@@ -24,7 +24,7 @@ string and one function named `gophercolor_png` with the following signature:
 You can now simply include the new .go file in your program and call
 `gophercolor_png()` to get the uncompressed image data. The function panics
 if something went wrong during decompression. See the testdata directory for
-example input and output.
+example input and output files for various modes.
 
 Aternatively, you can pipe the input file data into stdin. bindata will then
 spit out the generated Go code to stdout. This does require explicitly naming
@@ -36,6 +36,53 @@ The package name will still default to 'main'.
 Invoke the program with the -h flag for more options.
 
 
+### Lower memory footprint
+
+Using the `-m` flag, will alter the way the output file is generated.
+It will employ a hack that allows us to read the file data directly from
+the compiled program's `.rodata` section. This ensures that when we call
+call our generate function, we omit unnecessary memcopies.
+
+The downside of this, is that it requires dependencies on the `reflect` and
+`unsafe` packages. These may be restricted on platforms like AppEngine and
+thus prevent you from using this mode.
+
+Another disadvantage is that the byte slice we create, is strictly read-only.
+For most use-cases this is not a problem, but if you ever try to alter the
+returned byte slice, a runtime panic is thrown. Use this mode only on target
+platforms where memory constraints are an issue.
+
+The default behaviour is to use the old code generation method. This
+prevents the two previously mentioned issues, but will employ at least one
+extra memcopy and thus increase memory requirements.
+
+For instance, consider the following two examples...
+
+This would be the default mode, using an extra memcopy but gives a safe
+implementation without dependencies on `reflect` and `unsafe`:
+
+    func myfile() []byte {
+        return []byte{0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a}
+    }
+
+Here is the same functionality, but uses the .rodata hack.
+The byte slice returned from this example can not be written to without
+generating a runtime error.
+
+    var _myfile = "\x89\x50\x4e\x47\x0d\x0a\x1a"
+    
+    func myfile() []byte {
+        var empty [0]byte
+	    sx := (*reflect.StringHeader)(unsafe.Pointer(&_myfile))
+	    b := empty[:]
+	    bx := (*reflect.SliceHeader)(unsafe.Pointer(&b))
+	    bx.Data = sx.Data
+	    bx.Len = len(_myfile)
+	    bx.Cap = bx.Len
+	    return b
+    }
+
+
 ### Optional compression
 
 When the `-u` flag is given, the supplied resource is *not* GZIP compressed

bytewriter.go

@@ -0,0 +1,35 @@
+// This work is subject to the CC0 1.0 Universal (CC0 1.0) Public Domain Dedication
+// license. Its contents can be found at:
+// http://creativecommons.org/publicdomain/zero/1.0/
+
+package main
+
+import (
+	"fmt"
+	"io"
+)
+
+var newline = []byte{'\n'}
+
+type ByteWriter struct {
+	io.Writer
+	c int
+}
+
+func (w *ByteWriter) Write(p []byte) (n int, err error) {
+	if len(p) == 0 {
+		return
+	}
+
+	for n = range p {
+		if w.c%12 == 0 {
+			w.Writer.Write(newline)
+			w.c = 0
+		}
+
+		fmt.Fprintf(w.Writer, "0x%02x,", p[n])
+		w.c++
+	}
+
+	return
+}

main.go

@@ -16,26 +16,59 @@ import (
 
 const (
 	AppName    = "bindata"
-	AppVersion = "0.8"
+	AppVersion = "1.0.0"
+)
+
+var (
+	pipe         = false
+	in           = flag.String("i", "", "Path to the input file. Alternatively, pipe the file data into stdin.")
+	out          = flag.String("o", "", "Optional path to the output file.")
+	pkgname      = flag.String("p", "", "Optional name of the package to generate.")
+	funcname     = flag.String("f", "", "Optional name of the function/variable to generate.")
+	uncompressed = flag.Bool("u", false, "The specified resource will /not/ be GZIP compressed when this flag is specified. This alters the generated output code.")
+	nomemcopy    = flag.Bool("m", false, "Use the memcopy hack to get rid of unnecessary memcopies. Refer to the documentation to see what implications this carries.")
+	version      = flag.Bool("v", false, "Display version information.")
 )
 
 func main() {
-	in := flag.String("i", "", "Path to the input file. Alternatively, pipe the file data into stdin.")
+	parseArgs()
-	out := flag.String("o", "", "Optional path to the output file.")
-	pkgname := flag.String("p", "", "Optional name of the package to generate.")
-	funcname := flag.String("f", "", "Optional name of the function/variable to generate.")
-	uncompressed := flag.Bool("u", false, "The specified resource will /not/ be GZIP compressed when this flag is specified. This alters the generated output code.")
-	version := flag.Bool("v", false, "Display version information.")
 
+	if pipe {
+		translate(os.Stdin, os.Stdout, *pkgname, *funcname, *uncompressed, *nomemcopy)
+	} else {
+		fs, err := os.Open(*in)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "[e] %s\n", err)
+			return
+		}
+
+		defer fs.Close()
+
+		fd, err := os.Create(*out)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "[e] %s\n", err)
+			return
+		}
+
+		defer fd.Close()
+
+		translate(fs, fd, *pkgname, *funcname, *uncompressed, *nomemcopy)
+	}
+
+	fmt.Fprintln(os.Stdout, "[i] Done.")
+}
+
+// parseArgs processes and verifies commandline arguments.
+func parseArgs() {
 	flag.Parse()
 
 	if *version {
 		fmt.Fprintf(os.Stdout, "%s v%s (Go runtime %s)\n",
 			AppName, AppVersion, runtime.Version())
-		return
+		os.Exit(0)
 	}
 
-	pipe := len(*in) == 0
+	pipe = len(*in) == 0
 
 	if !pipe && len(*out) == 0 {
 		// Ensure we create our own output filename that does not already exist.
@@ -74,7 +107,7 @@ func main() {
 		if pipe {
 			// Can't infer from input file name in this mode.
 			fmt.Fprintln(os.Stderr, "[e] No function name specified.")
-			return
+			os.Exit(1)
 		}
 
 		_, file := path.Split(*in)
@@ -91,32 +124,4 @@ func main() {
 		fmt.Fprintf(os.Stderr, "[w] No function name specified. Using '%s'.\n", file)
 		*funcname = file
 	}
-
-	// Read the input file, transform it into a gzip compressed data stream and
-	// write it out as a go source file.
-	if pipe {
-		translate(os.Stdin, os.Stdout, *pkgname, *funcname, *uncompressed)
-		fmt.Fprintln(os.Stdout, "[i] Done.")
-		return
-	}
-
-	var fs, fd *os.File
-	var err error
-
-	if fs, err = os.Open(*in); err != nil {
-		fmt.Fprintf(os.Stderr, "[e] %s\n", err)
-		return
-	}
-
-	defer fs.Close()
-
-	if fd, err = os.Create(*out); err != nil {
-		fmt.Fprintf(os.Stderr, "[e] %s\n", err)
-		return
-	}
-
-	defer fd.Close()
-
-	translate(fs, fd, *pkgname, *funcname, *uncompressed)
-	fmt.Fprintln(os.Stdout, "[i] Done.")
 }

stringwriter.go

@@ -11,12 +11,12 @@ import (
 
 var line = []byte("\"+\n\"")
 
-type GoWriter struct {
+type StringWriter struct {
 	io.Writer
 	c int
 }
 
-func (w *GoWriter) Write(p []byte) (n int, err error) {
+func (w *StringWriter) Write(p []byte) (n int, err error) {
 	if len(p) == 0 {
 		return
 	}

testdata/nomemcpy-compressed.go

@@ -1379,11 +1379,8 @@ var _gophercolor_png = "" +
 	"\x3a\x92\xd3\x63\x31\x00\x00\x00\x00\x49\x45\x4e\x44\xae\x42\x60" +
 	"\x82\x01\x00\x00\xff\xff\x25\x50\x56\x5e\x8b\x55\x00\x00"
 
-// gophercolor_png returns the binary data for a given file.
+// gophercolor_png returns the raw, uncompressed file data data.
 func gophercolor_png() []byte {
-	// This bit of black magic ensures we do not get
-	// unneccesary memcpy's and can read directly from
-	// the .rodata section.
 	var empty [0]byte
 	sx := (*reflect.StringHeader)(unsafe.Pointer(&_gophercolor_png))
 	b := empty[:]

testdata/nomemcpy-uncompressed.go

@@ -1376,7 +1376,10 @@ var _gophercolor_png = "" +
 	"\x17\x7e\x60\x6e\xcb\x7f\x00\x4a\x3f\xff\x3a\x92\xd3\x63\x31\x00" +
 	"\x00\x00\x00\x49\x45\x4e\x44\xae\x42\x60\x82"
 
-// gophercolor_png returns the binary data for a given file.
+// gophercolor_png returns the raw file data data.
+//
+// WARNING: The returned byte slice is READ-ONLY.
+// Attempting to alter the slice contents will yield a runtime panic.
 func gophercolor_png() []byte {
 	var empty [0]byte
 	sx := (*reflect.StringHeader)(unsafe.Pointer(&_gophercolor_png))

translate.go

@@ -4,88 +4,21 @@
 
 package main
 
-import (
+import "io"
-	"compress/gzip"
-	"fmt"
-	"io"
-)
-
-// Translate the input file with optional GZIP compression.
-// input [-> gzip] -> gowriter -> output.
-func translate(input io.Reader, output io.Writer, pkgname, funcname string, uncompressed bool) {
-	fmt.Fprintf(output, `package %s
-
-import (`, pkgname)
 
+// translate translates the input file to go source code.
+func translate(input io.Reader, output io.Writer, pkgname, funcname string, uncompressed, nomemcpy bool) {
+	if nomemcpy {
 		if uncompressed {
-		fmt.Fprint(output, `
+			translate_nomemcpy_uncomp(input, output, pkgname, funcname)
-		"reflect"
-		"unsafe"`)
 		} else {
-		fmt.Fprint(output, `
+			translate_nomemcpy_comp(input, output, pkgname, funcname)
-		"bytes"
-		"compress/gzip"
-		"io"
-		"reflect"
-		"unsafe"`)
 		}
-
+	} else {
-	fmt.Fprintf(output, `
-)
-
-var _%s = "`, funcname)
-
 		if uncompressed {
-		io.Copy(&GoWriter{Writer: output}, input)
+			translate_memcpy_uncomp(input, output, pkgname, funcname)
 		} else {
-		gz := gzip.NewWriter(&GoWriter{Writer: output})
+			translate_memcpy_comp(input, output, pkgname, funcname)
-		io.Copy(gz, input)
-		gz.Close()
 		}
-
-	fmt.Fprintf(output, `"
-
-// %s returns the binary data for a given file.
-func %s() []byte {`, funcname, funcname)
-
-	if uncompressed {
-		fmt.Fprintf(output, `
-		// This bit of black magic ensures we do not get
-		// unneccesary memcpy's and can read directly from
-		// the .rodata section.
-		var empty [0]byte
-		sx := (*reflect.StringHeader)(unsafe.Pointer(&_%s))
-		b := empty[:]
-		bx := (*reflect.SliceHeader)(unsafe.Pointer(&b))
-		bx.Data = sx.Data
-		bx.Len = len(_%s)
-		bx.Cap = bx.Len
-		return b`, funcname, funcname)
-	} else {
-		fmt.Fprintf(output, `
-		// This bit of black magic ensures we do not get
-		// unneccesary memcpy's and can read directly from
-		// the .rodata section.
-		var empty [0]byte
-		sx := (*reflect.StringHeader)(unsafe.Pointer(&_%s))
-		b := empty[:]
-		bx := (*reflect.SliceHeader)(unsafe.Pointer(&b))
-		bx.Data = sx.Data
-		bx.Len = len(_%s)
-		bx.Cap = bx.Len
-		
-		gz, err := gzip.NewReader(bytes.NewBuffer(b))
-
-		if err != nil {
-			panic("Decompression failed: " + err.Error())
 	}
-
-		var buf bytes.Buffer
-		io.Copy(&buf, gz)
-		gz.Close()
-
-		return buf.Bytes()`, funcname, funcname)
-	}
-
-	fmt.Fprintf(output, "\n}")
 }

translate_memcpy_comp.go

@@ -0,0 +1,44 @@
+// This work is subject to the CC0 1.0 Universal (CC0 1.0) Public Domain Dedication
+// license. Its contents can be found at:
+// http://creativecommons.org/publicdomain/zero/1.0/
+
+package main
+
+import (
+	"compress/gzip"
+	"fmt"
+	"io"
+)
+
+// input -> gzip -> gowriter -> output.
+func translate_memcpy_comp(input io.Reader, output io.Writer, pkgname, funcname string) {
+	fmt.Fprintf(output, `package %s
+
+import (
+	"bytes"
+	"compress/gzip"
+	"io"
+)
+
+// %s returns the raw, uncompressed file data data.
+func %s() []byte {
+	gz, err := gzip.NewReader(bytes.NewBuffer([]byte{`, pkgname, funcname, funcname)
+
+	gz := gzip.NewWriter(&ByteWriter{Writer: output})
+	io.Copy(gz, input)
+	gz.Close()
+
+	fmt.Fprint(output, `
+	}))
+
+	if err != nil {
+		panic("Decompression failed: " + err.Error())
+	}
+
+	var b bytes.Buffer
+	io.Copy(&b, gz)
+	gz.Close()
+
+	return b.Bytes()
+}`)
+}

translate_memcpy_uncomp.go

@@ -0,0 +1,25 @@
+// This work is subject to the CC0 1.0 Universal (CC0 1.0) Public Domain Dedication
+// license. Its contents can be found at:
+// http://creativecommons.org/publicdomain/zero/1.0/
+
+package main
+
+import (
+	"fmt"
+	"io"
+)
+
+// input -> gzip -> gowriter -> output.
+func translate_memcpy_uncomp(input io.Reader, output io.Writer, pkgname, funcname string) {
+	fmt.Fprintf(output, `package %s
+
+// %s returns the raw file data data.
+func %s() []byte {
+	return []byte{`, pkgname, funcname, funcname)
+
+	io.Copy(&ByteWriter{Writer: output}, input)
+
+	fmt.Fprint(output, `
+	}
+}`)
+}

translate_nomemcpy_comp.go

@@ -0,0 +1,56 @@
+// This work is subject to the CC0 1.0 Universal (CC0 1.0) Public Domain Dedication
+// license. Its contents can be found at:
+// http://creativecommons.org/publicdomain/zero/1.0/
+
+package main
+
+import (
+	"compress/gzip"
+	"fmt"
+	"io"
+)
+
+// input -> gzip -> gowriter -> output.
+func translate_nomemcpy_comp(input io.Reader, output io.Writer, pkgname, funcname string) {
+	fmt.Fprintf(output, `package %s
+
+import (
+	"bytes"
+	"compress/gzip"
+	"io"
+	"reflect"
+	"unsafe"
+)
+
+var _%s = "`, pkgname, funcname)
+
+	gz := gzip.NewWriter(&StringWriter{Writer: output})
+	io.Copy(gz, input)
+	gz.Close()
+
+	fmt.Fprintf(output, `"
+
+// %s returns the raw, uncompressed file data data.
+func %s() []byte {
+	var empty [0]byte
+	sx := (*reflect.StringHeader)(unsafe.Pointer(&_%s))
+	b := empty[:]
+	bx := (*reflect.SliceHeader)(unsafe.Pointer(&b))
+	bx.Data = sx.Data
+	bx.Len = len(_%s)
+	bx.Cap = bx.Len
+
+	gz, err := gzip.NewReader(bytes.NewBuffer(b))
+
+	if err != nil {
+		panic("Decompression failed: " + err.Error())
+	}
+
+	var buf bytes.Buffer
+	io.Copy(&buf, gz)
+	gz.Close()
+
+	return buf.Bytes()
+}
+`, funcname, funcname, funcname, funcname)
+}

translate_nomemcpy_uncomp.go

@@ -0,0 +1,42 @@
+// This work is subject to the CC0 1.0 Universal (CC0 1.0) Public Domain Dedication
+// license. Its contents can be found at:
+// http://creativecommons.org/publicdomain/zero/1.0/
+
+package main
+
+import (
+	"fmt"
+	"io"
+)
+
+// input -> gowriter -> output.
+func translate_nomemcpy_uncomp(input io.Reader, output io.Writer, pkgname, funcname string) {
+	fmt.Fprintf(output, `package %s
+
+import (
+	"reflect"
+	"unsafe"
+)
+
+var _%s = "`, pkgname, funcname)
+
+	io.Copy(&StringWriter{Writer: output}, input)
+
+	fmt.Fprintf(output, `"
+
+// %s returns the raw file data data.
+//
+// WARNING: The returned byte slice is READ-ONLY.
+// Attempting to alter the slice contents will yield a runtime panic.
+func %s() []byte {
+	var empty [0]byte
+	sx := (*reflect.StringHeader)(unsafe.Pointer(&_%s))
+	b := empty[:]
+	bx := (*reflect.SliceHeader)(unsafe.Pointer(&b))
+	bx.Data = sx.Data
+	bx.Len = len(_%s)
+	bx.Cap = bx.Len
+	return b
+}
+`, funcname, funcname, funcname, funcname)
+}