new implementation

.github/workflows/test.yaml

@@ -21,7 +21,4 @@ jobs:
 
       - name: Run tests
         run: |
-          # fuck this
-          sed -i tests/mymod_test.out -e 's/846284/83d954/g'
-          sed -i tests/mymod_test_bare_map.out -e 's/845da4/83d4b4/g'
           v -stats test .

Makefile

@@ -1,20 +0,0 @@
-SRC_DIR ?= src
-DOC_DIR ?= doc
-TESTS_DIR ?= .
-
-all: test
-
-test:
-	v test $(TESTS_DIR)
-
-doc:
-	v doc -f html -m ./$(SRC_DIR) -o $(DOC_DIR)
-
-serve: clean doc
-	v -e "import net.http.file; file.serve(folder: '$(DOC_DIR)')"
-
-clean:
-	rm -r $(DOC_DIR) || true
-
-cli:
-	v cmd/mkembedfs -o mkembedfs

README.md

@@ -1,16 +1,17 @@
-## Code generator for embedding directories with files into executables
+# Embedding Directories into Executables
 
-The `$embed_file` statement in V embeds only a single file. This module makes
-it easy to embed entire directories into the final executable.
+The `$embed_file()` call in V embeds only a single file. This module makes it
+easy to embed entire directories into the final executable.
 
-File embedding in V is done at compile time, and unfortunately there is no way
-to dynamically embed arbitrary files into an application. The `embedfs` module
-is a simple code generator that creates a separate `.v` file with the code for
-embedding files. That is, the embedfs call must be made before the code is
-compiled. So embedfs is a build dependency.
+File embedding in V is done at compile time, and there is no way to dynamically
+embed arbitrary files into an application. The `embedfs` module is a simple
+code generator that creates a separate `.v` file with the code for embedding
+files.
+
+## Installation
 
 ```
-v install --git https://github.com/gechandesu/embedfs
+v install https://github.com/gechandesu/embedfs
 ```
 
 ## Usage
@@ -18,28 +19,30 @@ v install --git https://github.com/gechandesu/embedfs
 For example you have following file structure:
 
 ```
-v.mod
-src/
-    main.v
-    assets/
-        css/style.css
-        js/app.js
+./
+├── src/
+│   ├── assets/
+│   │   ├── css/
+│   │   │   └── style.css
+│   │   └── js/
+│   │       └── app.js
+│   └── main.v
+└── v.mod
 ```
 
 Lets embed the `assets` directory.
 
-Create `embed_assets.vsh` next to your v.mod:
+Create `embed_assets.vsh` next to v.mod:
 
 ```v
-#!/usr/bin/env -S v
+#!/usr/bin/env v
 
+import os
 import embedfs
 
-chdir('src')!
-assets := embedfs.CodeGenerator{
-    path: 'assets'
-}
-write_file('assets_generated.v', assets.generate())!
+os.chdir('src')!
+assets := embedfs.generate('assets')!
+os.write_file('assets_generated.v', assets)!
 ```
 
 Run it:
@@ -48,46 +51,40 @@ Run it:
 v run embed_assets.vsh
 ```
 
-Now you have `src/assets_generated.v`. Take a look inside it. So you can use
-`embedfs` const in `src/main.v` in this way:
+Now you have `src/assets_generated.v`. Take a look inside it:
+
+```v
+module main
+
+const embed_files = {
+        'assets/css/style.css': $embed_file('assets/css/style.css')
+        'assets/js/app.js': $embed_file('assets/js/app.js')
+}
+```
+
+You can use it in `src/main.v` in this way:
 
 ```v
 module main
 
 fn main() {
-    style := embedfs.files['assets/css/style.css']!
-    // If `bare_map` parameter is set to `true` use:
-    // style := embedfs['assets/css/style.css']!
-    println(style.data.to_string())
+    style := unsafe { embed_files['assets/css/style.css'].to_string() }
+    println(style)
 }
 ```
 
-The generated `embedfs` const value example (from `tests/`):
+The map type is `map[string]embed_file.EmbedFileData`, see the
+[v.embed_file](https://modules.vlang.io/v.embed_file.html#EmbedFileData)
+module docs for details.
 
-```v okfmt
-EmbedFileSystem{
-    files: {'assets/example.json': EmbedFile{
-        data: embed_file.EmbedFileData{ len: 22, path: "assets/example.json", apath: "", uncompressed: 846284 }
-        meta: EmbedFileMetadata{
-            key: 'assets/example.json'
-            name: 'example.json'
-            ext: 'json'
-            mimetype: 'application/json'
-        }
-    }}
-}
+## `bin2v` tool
+
+Also there is `v bin2v` utility that generates the V modules with embedded
+files. See:
+
+```
+v help bin2v
 ```
 
-The generated const value if `bare_map` parameter is `true`:
-
-```v okfmt
-{'assets/example.json': EmbedFile{
-    data: embed_file.EmbedFileData{ len: 22, path: "assets/example.json", apath: "", uncompressed: 845da4 }
-    meta: EmbedFileMetadata{
-        key: 'assets/example.json'
-        name: 'example.json'
-        ext: 'json'
-        mimetype: 'application/json'
-    }
-}}
-```
+In contrast with embedfs, bin2v generates constants into which it writes
+files as fixed-length byte arrays.

cmd/mkembedfs/help.txt

@@ -1,11 +1,13 @@
-mkembedfs - generate V code for embed directories with files into executable.
+mkembedfs - generate V code to embed directories into executables.
 usage: mkembedfs [flags] [<path>]
 flags:
-  -help                 print this help message and exit
-  -chdir <string>       change working directory before codegen
-  -prefix <string>      path prefix for file keys, none by default
+  -help                 print this help message and exit.
+  -chdir <string>       change working directory before codegen.
   -ignore <string>      path globs to ignore (allowed multiple times)
-  -module-name <string> generated module name, 'main' by default
-  -const-name <string>  generated constant name with data, 'embedfs' by default
-  -no-pub               do not make symbols in generated module public
-  -force-mimetype       set 'application/octet-stream' mime type for unknown files
+  -prefix <string>      path prefix for file keys, none by default.
+  -path-prefix <string> path prefix for files in $embed_file() calls.
+  -compression          apply zlib compression to $embed_file().
+  -module-name <string> generated module name, 'main' by default.
+  -const-name <string>  generated const name, 'embed_files' by default.
+  -no-pub               do not make symbols in generated module public.
+  -notice <string>      notice comment text, set empty to disable.

cmd/mkembedfs/mkembedfs.v

@@ -5,9 +5,9 @@ import flag
 import embedfs
 
 fn main() {
-	mut path := os.getwd()
-	mut flags, no_matches := flag.to_struct[FlagConfig](os.args, skip: 1, style: .v) or {
-		eprintln('cmdline parsing error, see -help for info')
+	mut path := '.'
+	mut flags, no_matches := flag.to_struct[FlagConfig](os.args, skip: 1, style: .go_flag) or {
+		println('cmdline parsing error, see -help for info')
 		exit(2)
 	}
 	if no_matches.len > 1 {
@@ -26,25 +26,32 @@ fn main() {
 			exit(1)
 		}
 	}
-	generator := embedfs.CodeGenerator{
-		path:            path
-		prefix:          flags.prefix
-		ignore_patterns: flags.ignore
-		module_name:     flags.module_name
-		const_name:      flags.const_name
-		make_pub:        if flags.no_pub { false } else { true }
-		force_mimetype:  flags.force_mimetype
-	}
-	println(generator.generate())
+	dump(flags)
+	dump(path)
+	code := embedfs.generate(path,
+		key_path_prefix:  flags.prefix
+		file_path_prefix: flags.path_prefix
+		ignore:           flags.ignore
+		module_name:      flags.module_name
+		const_name:       flags.const_name
+		const_public:     !flags.no_pub
+		notice:           flags.notice
+		compression:      flags.compression
+	)!
+	print(code)
+	flush_stdout()
 }
 
 struct FlagConfig {
-	help           bool
-	chdir          string
-	prefix         string
-	ignore         []string
-	module_name    string = 'main'
-	const_name     string = 'embedfs'
-	no_pub         bool
-	force_mimetype bool
+mut:
+	help        bool
+	chdir       string
+	ignore      []string
+	prefix      string
+	path_prefix string
+	module_name string = 'main'
+	const_name  string = 'embed_files'
+	no_pub      bool
+	notice      string = 'This file is generated by embedfs module, DO NOT EDIT!'
+	compression bool
 }

embedfs.template

@@ -0,0 +1,21 @@
+module ${module_name}
+
+@if !skip_notice
+/*
+	${notice}
+*/
+
+@endif
+@if const_public
+pub const ${const_name} = {
+@else
+const ${const_name} = {
+@endif
+@for key, value in files
+@if compression
+	'${key}': $embed_file('${value}', .zlib)
+@else
+	'${key}': $embed_file('${value}')
+@endif
+@endfor
+}

embedfs.v

@@ -0,0 +1,62 @@
+module embedfs
+
+import os
+
+@[params]
+pub struct EmbedFsParams {
+pub:
+	module_name      string = 'main'        // name of the generated V module.
+	const_name       string = 'embed_files' // name of the constant that will store the embedded data.
+	const_public     bool   // if true make the const public.
+	key_path_prefix  string // path prefix for all embedded files, will be added to data map key.
+	file_path_prefix string // path prefix for files used in $embed_file() call in generated file.
+	compression      bool   // add zlib compression flag to $embed_file() call in generated file.
+	notice           string = 'This file is generated by embedfs module, DO NOT EDIT!'
+}
+
+@[params]
+pub struct GenerateParams {
+	EmbedFsParams
+pub:
+	ignore []string // glob expressions for files that should not be embedded.
+}
+
+// generate generates the V code that embeds the contents of directory `dir`.
+pub fn generate(dir string, params GenerateParams) !string {
+	files := collect_files(dir, params.ignore)
+	return generate_with(files, params.EmbedFsParams)!
+}
+
+// generate_with generates the V code that embeds the files listed in `paths`.
+pub fn generate_with(paths []string, params EmbedFsParams) !string {
+	module_name := params.module_name
+	const_name := params.const_name
+	const_public := params.const_public
+	compression := params.compression
+	notice := params.notice
+	skip_notice := params.notice == ''
+	mut files := map[string]string{}
+	for file in paths {
+		file_path := if params.file_path_prefix == '' {
+			file
+		} else {
+			os.join_path_single(params.file_path_prefix, file)
+		}
+		files[os.join_path_single(params.key_path_prefix, file)] = file_path
+	}
+	result := $tmpl('embedfs.template')
+	return result
+}
+
+fn collect_files(path string, ignore []string) []string {
+	mut file_list := &[]string{}
+	os.walk(path, fn [mut file_list, ignore] (file string) {
+		for globexpr in ignore {
+			if file.match_glob(globexpr) {
+				return
+			}
+		}
+		file_list << file
+	})
+	return *file_list
+}

src/embedfs.v

@@ -1,122 +0,0 @@
-module embedfs
-
-import os
-import strings
-import net.http.mime
-
-pub struct CodeGenerator {
-pub:
-	// Path to file or directory to embed
-	path string @[required]
-	// Path prefix if you want to add extra prefix for file paths
-	prefix string
-	// Glob patterns to match files the must be ignored when generating the code
-	ignore_patterns []string
-	// If true set the default MIME-type for files if no MIME-type detected
-	force_mimetype bool
-	// Default MIME-type for files. See https://www.iana.org/assignments/media-types/media-types.xhtml
-	default_mimetype string = 'application/octet-stream'
-	// Name of generated module
-	module_name string = 'main'
-	// Name of constant which will contain embedded files
-	const_name string = 'embedfs'
-	// If true make constant public
-	make_const_pub bool
-	// If true make all symbols in generated module public
-	make_pub bool
-	// Generate map[string]EmbedFile instead of EmbedFileSystem instance
-	bare_map bool
-}
-
-struct EmbedFileSpec {
-	key      string
-	path     string
-	name     string
-	ext      string
-	mimetype string
-}
-
-pub fn (g CodeGenerator) generate() string {
-	visible := if g.make_pub == true { 'pub ' } else { '' }
-	const_visible := if g.make_pub == true || g.make_const_pub == true { 'pub ' } else { '' }
-	mut b := strings.new_builder(1024 * 4)
-	b.writeln('// !WARNING! This file is generated by embedfs module, do not edit it.')
-	b.writeln('')
-	b.writeln('module ${g.module_name}')
-	b.writeln('')
-	b.writeln('import v.embed_file { EmbedFileData }')
-	b.writeln('')
-	b.writeln('${visible}struct EmbedFileMetadata {')
-	b.writeln('\tkey      string')
-	b.writeln('\tname     string')
-	b.writeln('\text      string')
-	b.writeln('\tmimetype string')
-	b.writeln('}')
-	b.writeln('')
-	b.writeln('${visible}struct EmbedFile {')
-	b.writeln('\tdata EmbedFileData')
-	b.writeln('\tmeta EmbedFileMetadata')
-	b.writeln('}')
-	b.writeln('')
-	b.writeln('${visible}struct EmbedFileSystem {')
-	b.writeln('\tfiles map[string]EmbedFile')
-	b.writeln('}')
-	b.writeln('')
-	if g.bare_map {
-		b.writeln('${const_visible}const ${g.const_name} = {')
-		g.write_embed_file_map_item(mut b)
-		b.writeln('}')
-	} else {
-		b.writeln('${const_visible}const ${g.const_name} = EmbedFileSystem{')
-		b.writeln('\tfiles: {')
-		g.write_embed_file_map_item(mut b)
-		b.writeln('\t}')
-		b.writeln('}')
-	}
-	return b.str()
-}
-
-fn (g CodeGenerator) get_files() []EmbedFileSpec {
-	mut file_list := &[]string{}
-	os.walk(g.path, fn [mut file_list] (file string) {
-		file_list << file
-	})
-	mut files := []EmbedFileSpec{}
-	outer: for file in file_list {
-		for glob in g.ignore_patterns {
-			if file.match_glob(glob) {
-				continue outer
-			}
-		}
-		file_key := os.join_path_single(g.prefix, file)
-		file_path := file // the actual path used in $embed_file() statement
-		file_name := os.file_name(file_path)
-		file_ext := os.file_ext(file_name).replace_once('.', '')
-		mut mimetype := mime.get_mime_type(file_ext)
-		if g.force_mimetype && mimetype == '' {
-			mimetype = g.default_mimetype
-		}
-		files << EmbedFileSpec{
-			key:      file_key
-			path:     file_path
-			name:     file_name
-			ext:      file_ext
-			mimetype: mimetype
-		}
-	}
-	return files
-}
-
-fn (g CodeGenerator) write_embed_file_map_item(mut b strings.Builder) {
-	for filespec in g.get_files() {
-		b.writeln("\t\t'${filespec.key}': EmbedFile{")
-		b.writeln("\t\t\tdata: \$embed_file('${filespec.path}')")
-		b.writeln('\t\t\tmeta: EmbedFileMetadata{')
-		b.writeln("\t\t\t\tkey: '${filespec.key}'")
-		b.writeln("\t\t\t\tname: '${filespec.name}'")
-		b.writeln("\t\t\t\text: '${filespec.ext}'")
-		b.writeln("\t\t\t\tmimetype: '${filespec.mimetype}'")
-		b.writeln('\t\t\t}')
-		b.writeln('\t\t},')
-	}
-}

tests/generate_test.v

@@ -0,0 +1,17 @@
+import embedfs
+
+fn test_generate() {
+	expected := r"module main
+
+/*
+	This file is generated by embedfs module, DO NOT EDIT!
+*/
+
+const embed_files = {
+	'cmd/mkembedfs/mkembedfs.v': $embed_file('cmd/mkembedfs/mkembedfs.v')
+	'cmd/mkembedfs/help.txt': $embed_file('cmd/mkembedfs/help.txt')
+}
+"
+	data := embedfs.generate('cmd')!
+	assert data == expected
+}

tests/mymod/main.v

@@ -1,12 +1,6 @@
 module main
 
 fn main() {
-	println(embedfs)
-	$if bare_map ? {
-		json_file := embedfs['assets/example.json'] or { EmbedFile{} }
-		println(json_file.data.to_string().trim_space())
-	} $else {
-		json_file := embedfs.files['assets/example.json'] or { EmbedFile{} }
-		println(json_file.data.to_string().trim_space())
-	}
+	json_file := unsafe { embed_files['assets/example.json'] }
+	println(json_file.to_string().trim_space())
 }

tests/mymod_test.out

@@ -1,12 +1 @@
-EmbedFileSystem{
-    files: {'assets/example.json': EmbedFile{
-        data: embed_file.EmbedFileData{ len: 22, path: "assets/example.json", apath: "", uncompressed: 846284 }
-        meta: EmbedFileMetadata{
-            key: 'assets/example.json'
-            name: 'example.json'
-            ext: 'json'
-            mimetype: 'application/json'
-        }
-    }}
-}
 {"some": "JSON data"}

tests/mymod_test.v

@@ -8,27 +8,10 @@ fn test_mymod() {
 	oldpwd := os.getwd()
 	expected_out := os.read_file('tests/mymod_test.out')!
 	os.chdir('tests/mymod')!
-	gen := embedfs.CodeGenerator{
-		path: 'assets'
-	}
-	os.write_file('assets_generated.v', gen.generate())!
+	gen := embedfs.generate('assets')!
+	os.write_file('assets_generated.v', gen)!
 	ret := os.execute('${os.quoted_path(@VEXE)} run .')
 	dump(diff.compare_text(ret.output, expected_out)!)
 	assert ret.output == expected_out
 	os.chdir(oldpwd)!
 }
-
-fn test_mymod_bare_map() {
-	oldpwd := os.getwd()
-	expected_out := os.read_file('tests/mymod_test_bare_map.out')!
-	os.chdir('tests/mymod')!
-	gen := embedfs.CodeGenerator{
-		path:     'assets'
-		bare_map: true
-	}
-	os.write_file('assets_generated.v', gen.generate())!
-	ret := os.execute('${os.quoted_path(@VEXE)} -d bare_map run .')
-	dump(diff.compare_text(ret.output, expected_out)!)
-	assert ret.output == expected_out
-	os.chdir(oldpwd)!
-}

tests/mymod_test_bare_map.out

@@ -1,10 +0,0 @@
-{'assets/example.json': EmbedFile{
-    data: embed_file.EmbedFileData{ len: 22, path: "assets/example.json", apath: "", uncompressed: 845da4 }
-    meta: EmbedFileMetadata{
-        key: 'assets/example.json'
-        name: 'example.json'
-        ext: 'json'
-        mimetype: 'application/json'
-    }
-}}
-{"some": "JSON data"}

v.mod

@@ -1,7 +1,7 @@
 Module {
 	name: 'embedfs'
-	description: 'Code generator for embedding directories with files into executables'
-	version: '0.0.2'
+	description: 'Embed directories into executables'
+	version: '1.0.0'
 	license: 'Unlicense'
 	dependencies: []
 }