Combine the old and new code generation methods. We can select the .rodata hack described in issue #1 by supplying the -m commandline flag. The default code generation mode is the old one. While it uses more memory, it is a safer version and offers no problems when used on platforms that restrict usage of the unsafe
and reflect
packages. Additionally I did some cleanup and refactoring of the code. Bumped version to 1.0.0
This commit is contained in:
parent
c2d800d607
commit
98c1704190
49
README.md
49
README.md
|
@ -24,7 +24,7 @@ string and one function named `gophercolor_png` with the following signature:
|
|||
You can now simply include the new .go file in your program and call
|
||||
`gophercolor_png()` to get the uncompressed image data. The function panics
|
||||
if something went wrong during decompression. See the testdata directory for
|
||||
example input and output.
|
||||
example input and output files for various modes.
|
||||
|
||||
Aternatively, you can pipe the input file data into stdin. bindata will then
|
||||
spit out the generated Go code to stdout. This does require explicitly naming
|
||||
|
@ -36,6 +36,53 @@ The package name will still default to 'main'.
|
|||
Invoke the program with the -h flag for more options.
|
||||
|
||||
|
||||
### Lower memory footprint
|
||||
|
||||
Using the `-m` flag, will alter the way the output file is generated.
|
||||
It will employ a hack that allows us to read the file data directly from
|
||||
the compiled program's `.rodata` section. This ensures that when we call
|
||||
call our generate function, we omit unnecessary memcopies.
|
||||
|
||||
The downside of this, is that it requires dependencies on the `reflect` and
|
||||
`unsafe` packages. These may be restricted on platforms like AppEngine and
|
||||
thus prevent you from using this mode.
|
||||
|
||||
Another disadvantage is that the byte slice we create, is strictly read-only.
|
||||
For most use-cases this is not a problem, but if you ever try to alter the
|
||||
returned byte slice, a runtime panic is thrown. Use this mode only on target
|
||||
platforms where memory constraints are an issue.
|
||||
|
||||
The default behaviour is to use the old code generation method. This
|
||||
prevents the two previously mentioned issues, but will employ at least one
|
||||
extra memcopy and thus increase memory requirements.
|
||||
|
||||
For instance, consider the following two examples...
|
||||
|
||||
This would be the default mode, using an extra memcopy but gives a safe
|
||||
implementation without dependencies on `reflect` and `unsafe`:
|
||||
|
||||
func myfile() []byte {
|
||||
return []byte{0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a}
|
||||
}
|
||||
|
||||
Here is the same functionality, but uses the .rodata hack.
|
||||
The byte slice returned from this example can not be written to without
|
||||
generating a runtime error.
|
||||
|
||||
var _myfile = "\x89\x50\x4e\x47\x0d\x0a\x1a"
|
||||
|
||||
func myfile() []byte {
|
||||
var empty [0]byte
|
||||
sx := (*reflect.StringHeader)(unsafe.Pointer(&_myfile))
|
||||
b := empty[:]
|
||||
bx := (*reflect.SliceHeader)(unsafe.Pointer(&b))
|
||||
bx.Data = sx.Data
|
||||
bx.Len = len(_myfile)
|
||||
bx.Cap = bx.Len
|
||||
return b
|
||||
}
|
||||
|
||||
|
||||
### Optional compression
|
||||
|
||||
When the `-u` flag is given, the supplied resource is *not* GZIP compressed
|
||||
|
|
35
bytewriter.go
Normal file
35
bytewriter.go
Normal file
|
@ -0,0 +1,35 @@
|
|||
// This work is subject to the CC0 1.0 Universal (CC0 1.0) Public Domain Dedication
|
||||
// license. Its contents can be found at:
|
||||
// http://creativecommons.org/publicdomain/zero/1.0/
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
)
|
||||
|
||||
var newline = []byte{'\n'}
|
||||
|
||||
type ByteWriter struct {
|
||||
io.Writer
|
||||
c int
|
||||
}
|
||||
|
||||
func (w *ByteWriter) Write(p []byte) (n int, err error) {
|
||||
if len(p) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
for n = range p {
|
||||
if w.c%12 == 0 {
|
||||
w.Writer.Write(newline)
|
||||
w.c = 0
|
||||
}
|
||||
|
||||
fmt.Fprintf(w.Writer, "0x%02x,", p[n])
|
||||
w.c++
|
||||
}
|
||||
|
||||
return
|
||||
}
|
81
main.go
81
main.go
|
@ -16,26 +16,59 @@ import (
|
|||
|
||||
const (
|
||||
AppName = "bindata"
|
||||
AppVersion = "0.8"
|
||||
AppVersion = "1.0.0"
|
||||
)
|
||||
|
||||
var (
|
||||
pipe = false
|
||||
in = flag.String("i", "", "Path to the input file. Alternatively, pipe the file data into stdin.")
|
||||
out = flag.String("o", "", "Optional path to the output file.")
|
||||
pkgname = flag.String("p", "", "Optional name of the package to generate.")
|
||||
funcname = flag.String("f", "", "Optional name of the function/variable to generate.")
|
||||
uncompressed = flag.Bool("u", false, "The specified resource will /not/ be GZIP compressed when this flag is specified. This alters the generated output code.")
|
||||
nomemcopy = flag.Bool("m", false, "Use the memcopy hack to get rid of unnecessary memcopies. Refer to the documentation to see what implications this carries.")
|
||||
version = flag.Bool("v", false, "Display version information.")
|
||||
)
|
||||
|
||||
func main() {
|
||||
in := flag.String("i", "", "Path to the input file. Alternatively, pipe the file data into stdin.")
|
||||
out := flag.String("o", "", "Optional path to the output file.")
|
||||
pkgname := flag.String("p", "", "Optional name of the package to generate.")
|
||||
funcname := flag.String("f", "", "Optional name of the function/variable to generate.")
|
||||
uncompressed := flag.Bool("u", false, "The specified resource will /not/ be GZIP compressed when this flag is specified. This alters the generated output code.")
|
||||
version := flag.Bool("v", false, "Display version information.")
|
||||
parseArgs()
|
||||
|
||||
if pipe {
|
||||
translate(os.Stdin, os.Stdout, *pkgname, *funcname, *uncompressed, *nomemcopy)
|
||||
} else {
|
||||
fs, err := os.Open(*in)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "[e] %s\n", err)
|
||||
return
|
||||
}
|
||||
|
||||
defer fs.Close()
|
||||
|
||||
fd, err := os.Create(*out)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "[e] %s\n", err)
|
||||
return
|
||||
}
|
||||
|
||||
defer fd.Close()
|
||||
|
||||
translate(fs, fd, *pkgname, *funcname, *uncompressed, *nomemcopy)
|
||||
}
|
||||
|
||||
fmt.Fprintln(os.Stdout, "[i] Done.")
|
||||
}
|
||||
|
||||
// parseArgs processes and verifies commandline arguments.
|
||||
func parseArgs() {
|
||||
flag.Parse()
|
||||
|
||||
if *version {
|
||||
fmt.Fprintf(os.Stdout, "%s v%s (Go runtime %s)\n",
|
||||
AppName, AppVersion, runtime.Version())
|
||||
return
|
||||
os.Exit(0)
|
||||
}
|
||||
|
||||
pipe := len(*in) == 0
|
||||
pipe = len(*in) == 0
|
||||
|
||||
if !pipe && len(*out) == 0 {
|
||||
// Ensure we create our own output filename that does not already exist.
|
||||
|
@ -74,7 +107,7 @@ func main() {
|
|||
if pipe {
|
||||
// Can't infer from input file name in this mode.
|
||||
fmt.Fprintln(os.Stderr, "[e] No function name specified.")
|
||||
return
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
_, file := path.Split(*in)
|
||||
|
@ -91,32 +124,4 @@ func main() {
|
|||
fmt.Fprintf(os.Stderr, "[w] No function name specified. Using '%s'.\n", file)
|
||||
*funcname = file
|
||||
}
|
||||
|
||||
// Read the input file, transform it into a gzip compressed data stream and
|
||||
// write it out as a go source file.
|
||||
if pipe {
|
||||
translate(os.Stdin, os.Stdout, *pkgname, *funcname, *uncompressed)
|
||||
fmt.Fprintln(os.Stdout, "[i] Done.")
|
||||
return
|
||||
}
|
||||
|
||||
var fs, fd *os.File
|
||||
var err error
|
||||
|
||||
if fs, err = os.Open(*in); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "[e] %s\n", err)
|
||||
return
|
||||
}
|
||||
|
||||
defer fs.Close()
|
||||
|
||||
if fd, err = os.Create(*out); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "[e] %s\n", err)
|
||||
return
|
||||
}
|
||||
|
||||
defer fd.Close()
|
||||
|
||||
translate(fs, fd, *pkgname, *funcname, *uncompressed)
|
||||
fmt.Fprintln(os.Stdout, "[i] Done.")
|
||||
}
|
||||
|
|
|
@ -11,12 +11,12 @@ import (
|
|||
|
||||
var line = []byte("\"+\n\"")
|
||||
|
||||
type GoWriter struct {
|
||||
type StringWriter struct {
|
||||
io.Writer
|
||||
c int
|
||||
}
|
||||
|
||||
func (w *GoWriter) Write(p []byte) (n int, err error) {
|
||||
func (w *StringWriter) Write(p []byte) (n int, err error) {
|
||||
if len(p) == 0 {
|
||||
return
|
||||
}
|
1849
testdata/memcpy-compressed.go
vendored
Normal file
1849
testdata/memcpy-compressed.go
vendored
Normal file
File diff suppressed because it is too large
Load Diff
1832
testdata/memcpy-uncompressed.go
vendored
Normal file
1832
testdata/memcpy-uncompressed.go
vendored
Normal file
File diff suppressed because it is too large
Load Diff
|
@ -1379,11 +1379,8 @@ var _gophercolor_png = "" +
|
|||
"\x3a\x92\xd3\x63\x31\x00\x00\x00\x00\x49\x45\x4e\x44\xae\x42\x60" +
|
||||
"\x82\x01\x00\x00\xff\xff\x25\x50\x56\x5e\x8b\x55\x00\x00"
|
||||
|
||||
// gophercolor_png returns the binary data for a given file.
|
||||
// gophercolor_png returns the raw, uncompressed file data data.
|
||||
func gophercolor_png() []byte {
|
||||
// This bit of black magic ensures we do not get
|
||||
// unneccesary memcpy's and can read directly from
|
||||
// the .rodata section.
|
||||
var empty [0]byte
|
||||
sx := (*reflect.StringHeader)(unsafe.Pointer(&_gophercolor_png))
|
||||
b := empty[:]
|
|
@ -1376,7 +1376,10 @@ var _gophercolor_png = "" +
|
|||
"\x17\x7e\x60\x6e\xcb\x7f\x00\x4a\x3f\xff\x3a\x92\xd3\x63\x31\x00" +
|
||||
"\x00\x00\x00\x49\x45\x4e\x44\xae\x42\x60\x82"
|
||||
|
||||
// gophercolor_png returns the binary data for a given file.
|
||||
// gophercolor_png returns the raw file data data.
|
||||
//
|
||||
// WARNING: The returned byte slice is READ-ONLY.
|
||||
// Attempting to alter the slice contents will yield a runtime panic.
|
||||
func gophercolor_png() []byte {
|
||||
var empty [0]byte
|
||||
sx := (*reflect.StringHeader)(unsafe.Pointer(&_gophercolor_png))
|
95
translate.go
95
translate.go
|
@ -4,88 +4,21 @@
|
|||
|
||||
package main
|
||||
|
||||
import (
|
||||
"compress/gzip"
|
||||
"fmt"
|
||||
"io"
|
||||
)
|
||||
import "io"
|
||||
|
||||
// Translate the input file with optional GZIP compression.
|
||||
// input [-> gzip] -> gowriter -> output.
|
||||
func translate(input io.Reader, output io.Writer, pkgname, funcname string, uncompressed bool) {
|
||||
fmt.Fprintf(output, `package %s
|
||||
|
||||
import (`, pkgname)
|
||||
|
||||
if uncompressed {
|
||||
fmt.Fprint(output, `
|
||||
"reflect"
|
||||
"unsafe"`)
|
||||
} else {
|
||||
fmt.Fprint(output, `
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"io"
|
||||
"reflect"
|
||||
"unsafe"`)
|
||||
}
|
||||
|
||||
fmt.Fprintf(output, `
|
||||
)
|
||||
|
||||
var _%s = "`, funcname)
|
||||
|
||||
if uncompressed {
|
||||
io.Copy(&GoWriter{Writer: output}, input)
|
||||
} else {
|
||||
gz := gzip.NewWriter(&GoWriter{Writer: output})
|
||||
io.Copy(gz, input)
|
||||
gz.Close()
|
||||
}
|
||||
|
||||
fmt.Fprintf(output, `"
|
||||
|
||||
// %s returns the binary data for a given file.
|
||||
func %s() []byte {`, funcname, funcname)
|
||||
|
||||
if uncompressed {
|
||||
fmt.Fprintf(output, `
|
||||
// This bit of black magic ensures we do not get
|
||||
// unneccesary memcpy's and can read directly from
|
||||
// the .rodata section.
|
||||
var empty [0]byte
|
||||
sx := (*reflect.StringHeader)(unsafe.Pointer(&_%s))
|
||||
b := empty[:]
|
||||
bx := (*reflect.SliceHeader)(unsafe.Pointer(&b))
|
||||
bx.Data = sx.Data
|
||||
bx.Len = len(_%s)
|
||||
bx.Cap = bx.Len
|
||||
return b`, funcname, funcname)
|
||||
} else {
|
||||
fmt.Fprintf(output, `
|
||||
// This bit of black magic ensures we do not get
|
||||
// unneccesary memcpy's and can read directly from
|
||||
// the .rodata section.
|
||||
var empty [0]byte
|
||||
sx := (*reflect.StringHeader)(unsafe.Pointer(&_%s))
|
||||
b := empty[:]
|
||||
bx := (*reflect.SliceHeader)(unsafe.Pointer(&b))
|
||||
bx.Data = sx.Data
|
||||
bx.Len = len(_%s)
|
||||
bx.Cap = bx.Len
|
||||
|
||||
gz, err := gzip.NewReader(bytes.NewBuffer(b))
|
||||
|
||||
if err != nil {
|
||||
panic("Decompression failed: " + err.Error())
|
||||
// translate translates the input file to go source code.
|
||||
func translate(input io.Reader, output io.Writer, pkgname, funcname string, uncompressed, nomemcpy bool) {
|
||||
if nomemcpy {
|
||||
if uncompressed {
|
||||
translate_nomemcpy_uncomp(input, output, pkgname, funcname)
|
||||
} else {
|
||||
translate_nomemcpy_comp(input, output, pkgname, funcname)
|
||||
}
|
||||
} else {
|
||||
if uncompressed {
|
||||
translate_memcpy_uncomp(input, output, pkgname, funcname)
|
||||
} else {
|
||||
translate_memcpy_comp(input, output, pkgname, funcname)
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
io.Copy(&buf, gz)
|
||||
gz.Close()
|
||||
|
||||
return buf.Bytes()`, funcname, funcname)
|
||||
}
|
||||
|
||||
fmt.Fprintf(output, "\n}")
|
||||
}
|
||||
|
|
44
translate_memcpy_comp.go
Normal file
44
translate_memcpy_comp.go
Normal file
|
@ -0,0 +1,44 @@
|
|||
// This work is subject to the CC0 1.0 Universal (CC0 1.0) Public Domain Dedication
|
||||
// license. Its contents can be found at:
|
||||
// http://creativecommons.org/publicdomain/zero/1.0/
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"compress/gzip"
|
||||
"fmt"
|
||||
"io"
|
||||
)
|
||||
|
||||
// input -> gzip -> gowriter -> output.
|
||||
func translate_memcpy_comp(input io.Reader, output io.Writer, pkgname, funcname string) {
|
||||
fmt.Fprintf(output, `package %s
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"io"
|
||||
)
|
||||
|
||||
// %s returns the raw, uncompressed file data data.
|
||||
func %s() []byte {
|
||||
gz, err := gzip.NewReader(bytes.NewBuffer([]byte{`, pkgname, funcname, funcname)
|
||||
|
||||
gz := gzip.NewWriter(&ByteWriter{Writer: output})
|
||||
io.Copy(gz, input)
|
||||
gz.Close()
|
||||
|
||||
fmt.Fprint(output, `
|
||||
}))
|
||||
|
||||
if err != nil {
|
||||
panic("Decompression failed: " + err.Error())
|
||||
}
|
||||
|
||||
var b bytes.Buffer
|
||||
io.Copy(&b, gz)
|
||||
gz.Close()
|
||||
|
||||
return b.Bytes()
|
||||
}`)
|
||||
}
|
25
translate_memcpy_uncomp.go
Normal file
25
translate_memcpy_uncomp.go
Normal file
|
@ -0,0 +1,25 @@
|
|||
// This work is subject to the CC0 1.0 Universal (CC0 1.0) Public Domain Dedication
|
||||
// license. Its contents can be found at:
|
||||
// http://creativecommons.org/publicdomain/zero/1.0/
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
)
|
||||
|
||||
// input -> gzip -> gowriter -> output.
|
||||
func translate_memcpy_uncomp(input io.Reader, output io.Writer, pkgname, funcname string) {
|
||||
fmt.Fprintf(output, `package %s
|
||||
|
||||
// %s returns the raw file data data.
|
||||
func %s() []byte {
|
||||
return []byte{`, pkgname, funcname, funcname)
|
||||
|
||||
io.Copy(&ByteWriter{Writer: output}, input)
|
||||
|
||||
fmt.Fprint(output, `
|
||||
}
|
||||
}`)
|
||||
}
|
56
translate_nomemcpy_comp.go
Normal file
56
translate_nomemcpy_comp.go
Normal file
|
@ -0,0 +1,56 @@
|
|||
// This work is subject to the CC0 1.0 Universal (CC0 1.0) Public Domain Dedication
|
||||
// license. Its contents can be found at:
|
||||
// http://creativecommons.org/publicdomain/zero/1.0/
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"compress/gzip"
|
||||
"fmt"
|
||||
"io"
|
||||
)
|
||||
|
||||
// input -> gzip -> gowriter -> output.
|
||||
func translate_nomemcpy_comp(input io.Reader, output io.Writer, pkgname, funcname string) {
|
||||
fmt.Fprintf(output, `package %s
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"io"
|
||||
"reflect"
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
var _%s = "`, pkgname, funcname)
|
||||
|
||||
gz := gzip.NewWriter(&StringWriter{Writer: output})
|
||||
io.Copy(gz, input)
|
||||
gz.Close()
|
||||
|
||||
fmt.Fprintf(output, `"
|
||||
|
||||
// %s returns the raw, uncompressed file data data.
|
||||
func %s() []byte {
|
||||
var empty [0]byte
|
||||
sx := (*reflect.StringHeader)(unsafe.Pointer(&_%s))
|
||||
b := empty[:]
|
||||
bx := (*reflect.SliceHeader)(unsafe.Pointer(&b))
|
||||
bx.Data = sx.Data
|
||||
bx.Len = len(_%s)
|
||||
bx.Cap = bx.Len
|
||||
|
||||
gz, err := gzip.NewReader(bytes.NewBuffer(b))
|
||||
|
||||
if err != nil {
|
||||
panic("Decompression failed: " + err.Error())
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
io.Copy(&buf, gz)
|
||||
gz.Close()
|
||||
|
||||
return buf.Bytes()
|
||||
}
|
||||
`, funcname, funcname, funcname, funcname)
|
||||
}
|
42
translate_nomemcpy_uncomp.go
Normal file
42
translate_nomemcpy_uncomp.go
Normal file
|
@ -0,0 +1,42 @@
|
|||
// This work is subject to the CC0 1.0 Universal (CC0 1.0) Public Domain Dedication
|
||||
// license. Its contents can be found at:
|
||||
// http://creativecommons.org/publicdomain/zero/1.0/
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
)
|
||||
|
||||
// input -> gowriter -> output.
|
||||
func translate_nomemcpy_uncomp(input io.Reader, output io.Writer, pkgname, funcname string) {
|
||||
fmt.Fprintf(output, `package %s
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
var _%s = "`, pkgname, funcname)
|
||||
|
||||
io.Copy(&StringWriter{Writer: output}, input)
|
||||
|
||||
fmt.Fprintf(output, `"
|
||||
|
||||
// %s returns the raw file data data.
|
||||
//
|
||||
// WARNING: The returned byte slice is READ-ONLY.
|
||||
// Attempting to alter the slice contents will yield a runtime panic.
|
||||
func %s() []byte {
|
||||
var empty [0]byte
|
||||
sx := (*reflect.StringHeader)(unsafe.Pointer(&_%s))
|
||||
b := empty[:]
|
||||
bx := (*reflect.SliceHeader)(unsafe.Pointer(&b))
|
||||
bx.Data = sx.Data
|
||||
bx.Len = len(_%s)
|
||||
bx.Cap = bx.Len
|
||||
return b
|
||||
}
|
||||
`, funcname, funcname, funcname, funcname)
|
||||
}
|
Loading…
Reference in New Issue
Block a user