From 95ceb5aeac35ba128c3b6d51370b8b7f9a0bb917 Mon Sep 17 00:00:00 2001 From: ge Date: Tue, 5 May 2026 23:02:06 +0300 Subject: [PATCH] new implementation --- README.md | 105 +++++++++++++------------- cmd/mkembedfs/help.txt | 18 +++-- cmd/mkembedfs/mkembedfs.v | 49 ++++++------ embedfs.template | 21 ++++++ embedfs.v | 152 ++++++++++++-------------------------- tests/generate_test.v | 17 +++++ tests/mymod/main.v | 9 +-- tests/mymod_test.v | 21 +----- v.mod | 4 +- 9 files changed, 179 insertions(+), 217 deletions(-) create mode 100644 embedfs.template create mode 100644 tests/generate_test.v diff --git a/README.md b/README.md index 2d9d519..046b3de 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,17 @@ -## Code generator for embedding directories with files into executables +# Embedding Directories into Executables -The `$embed_file` statement in V embeds only a single file. This module makes -it easy to embed entire directories into the final executable. +The `$embed_file()` call in V embeds only a single file. This module makes it +easy to embed entire directories into the final executable. -File embedding in V is done at compile time, and unfortunately there is no way -to dynamically embed arbitrary files into an application. The `embedfs` module -is a simple code generator that creates a separate `.v` file with the code for -embedding files. That is, the embedfs call must be made before the code is -compiled. So embedfs is a build dependency. +File embedding in V is done at compile time, and there is no way to dynamically +embed arbitrary files into an application. The `embedfs` module is a simple +code generator that creates a separate `.v` file with the code for embedding +files. + +## Installation ``` -v install --git https://github.com/gechandesu/embedfs +v install https://github.com/gechandesu/embedfs ``` ## Usage @@ -18,28 +19,30 @@ v install --git https://github.com/gechandesu/embedfs For example you have following file structure: ``` -v.mod -src/ - main.v - assets/ - css/style.css - js/app.js +./ +├── src/ +│   ├── assets/ +│   │   ├── css/ +│   │   │   └── style.css +│   │   └── js/ +│   │   └── app.js +│   └── main.v +└── v.mod ``` Lets embed the `assets` directory. -Create `embed_assets.vsh` next to your v.mod: +Create `embed_assets.vsh` next to v.mod: ```v -#!/usr/bin/env -S v +#!/usr/bin/env v +import os import embedfs -chdir('src')! -assets := embedfs.CodeGenerator{ - path: 'assets' -} -write_file('assets_generated.v', assets.generate())! +os.chdir('src')! +assets := embedfs.generate('assets')! +os.write_file('assets_generated.v', assets)! ``` Run it: @@ -48,46 +51,40 @@ Run it: v run embed_assets.vsh ``` -Now you have `src/assets_generated.v`. Take a look inside it. So you can use -`embedfs` const in `src/main.v` in this way: +Now you have `src/assets_generated.v`. Take a look inside it: + +```v +module main + +const embed_files = { + 'assets/css/style.css': $embed_file('assets/css/style.css') + 'assets/js/app.js': $embed_file('assets/js/app.js') +} +``` + +You can use it in `src/main.v` in this way: ```v module main fn main() { - style := embedfs.files['assets/css/style.css']! - // If `bare_map` parameter is set to `true` use: - // style := embedfs['assets/css/style.css']! - println(style.data.to_string()) + style := unsafe { embed_files['assets/css/style.css'].to_string() } + println(style) } ``` -The generated `embedfs` const value example (from `tests/`): +The map type is `map[string]embed_file.EmbedFileData`, see the +[v.embed_file](https://modules.vlang.io/v.embed_file.html#EmbedFileData) +module docs for details. -```v okfmt -EmbedFileSystem{ - files: {'assets/example.json': EmbedFile{ - data: embed_file.EmbedFileData{ len: 22, path: "assets/example.json", apath: "", uncompressed: 846284 } - meta: EmbedFileMetadata{ - key: 'assets/example.json' - name: 'example.json' - ext: 'json' - mimetype: 'application/json' - } - }} -} +## `bin2v` tool + +Also there is `v bin2v` utility that generates the V modules with embedded +files. See: + +``` +v help bin2v ``` -The generated const value if `bare_map` parameter is `true`: - -```v okfmt -{'assets/example.json': EmbedFile{ - data: embed_file.EmbedFileData{ len: 22, path: "assets/example.json", apath: "", uncompressed: 845da4 } - meta: EmbedFileMetadata{ - key: 'assets/example.json' - name: 'example.json' - ext: 'json' - mimetype: 'application/json' - } -}} -``` +In contrast with embedfs, bin2v generates constants into which it writes +files as fixed-length byte arrays. diff --git a/cmd/mkembedfs/help.txt b/cmd/mkembedfs/help.txt index 1f0f87b..318f667 100644 --- a/cmd/mkembedfs/help.txt +++ b/cmd/mkembedfs/help.txt @@ -1,11 +1,13 @@ -mkembedfs - generate V code for embed directories with files into executable. +mkembedfs - generate V code to embed directories into executables. usage: mkembedfs [flags] [] flags: - -help print this help message and exit - -chdir change working directory before codegen - -prefix path prefix for file keys, none by default + -help print this help message and exit. + -chdir change working directory before codegen. -ignore path globs to ignore (allowed multiple times) - -module-name generated module name, 'main' by default - -const-name generated constant name with data, 'embedfs' by default - -no-pub do not make symbols in generated module public - -force-mimetype set 'application/octet-stream' mime type for unknown files + -prefix path prefix for file keys, none by default. + -path-prefix path prefix for files in $embed_file() calls. + -compression apply zlib compression to $embed_file(). + -module-name generated module name, 'main' by default. + -const-name generated const name, 'embed_files' by default. + -no-pub do not make symbols in generated module public. + -notice notice comment text, set empty to disable. diff --git a/cmd/mkembedfs/mkembedfs.v b/cmd/mkembedfs/mkembedfs.v index d579345..f593b3a 100644 --- a/cmd/mkembedfs/mkembedfs.v +++ b/cmd/mkembedfs/mkembedfs.v @@ -5,9 +5,9 @@ import flag import embedfs fn main() { - mut path := os.getwd() - mut flags, no_matches := flag.to_struct[FlagConfig](os.args, skip: 1, style: .v) or { - eprintln('cmdline parsing error, see -help for info') + mut path := '.' + mut flags, no_matches := flag.to_struct[FlagConfig](os.args, skip: 1, style: .go_flag) or { + println('cmdline parsing error, see -help for info') exit(2) } if no_matches.len > 1 { @@ -26,25 +26,32 @@ fn main() { exit(1) } } - generator := embedfs.CodeGenerator{ - path: path - prefix: flags.prefix - ignore_patterns: flags.ignore - module_name: flags.module_name - const_name: flags.const_name - make_pub: if flags.no_pub { false } else { true } - force_mimetype: flags.force_mimetype - } - println(generator.generate()) + dump(flags) + dump(path) + code := embedfs.generate(path, + key_path_prefix: flags.prefix + file_path_prefix: flags.path_prefix + ignore: flags.ignore + module_name: flags.module_name + const_name: flags.const_name + const_public: !flags.no_pub + notice: flags.notice + compression: flags.compression + )! + print(code) + flush_stdout() } struct FlagConfig { - help bool - chdir string - prefix string - ignore []string - module_name string = 'main' - const_name string = 'embedfs' - no_pub bool - force_mimetype bool +mut: + help bool + chdir string + ignore []string + prefix string + path_prefix string + module_name string = 'main' + const_name string = 'embed_files' + no_pub bool + notice string = 'This file is generated by embedfs module, DO NOT EDIT!' + compression bool } diff --git a/embedfs.template b/embedfs.template new file mode 100644 index 0000000..5fb5859 --- /dev/null +++ b/embedfs.template @@ -0,0 +1,21 @@ +module ${module_name} + +@if !skip_notice +/* + ${notice} +*/ + +@endif +@if const_public +pub const ${const_name} = { +@else +const ${const_name} = { +@endif +@for key, value in files +@if compression + '${key}': $embed_file('${value}', .zlib) +@else + '${key}': $embed_file('${value}') +@endif +@endfor +} diff --git a/embedfs.v b/embedfs.v index 46eea6a..b94df7c 100644 --- a/embedfs.v +++ b/embedfs.v @@ -1,122 +1,62 @@ module embedfs import os -import strings -import net.http.mime -pub struct CodeGenerator { +@[params] +pub struct EmbedFsParams { pub: - // Path to file or directory to embed - path string @[required] - // Path prefix if you want to add extra prefix for file paths - prefix string - // Glob patterns to match files the must be ignored when generating the code - ignore_patterns []string - // If true set the default MIME-type for files if no MIME-type detected - force_mimetype bool - // Default MIME-type for files. See https://www.iana.org/assignments/media-types/media-types.xhtml - default_mimetype string = 'application/octet-stream' - // Name of generated module - module_name string = 'main' - // Name of constant which will contain embedded files - const_name string = 'embedfs' - // If true make constant public - make_const_pub bool - // If true make all symbols in generated module public - make_pub bool - // Generate map[string]EmbedFile instead of EmbedFileSystem instance - bare_map bool + module_name string = 'main' // name of the generated V module. + const_name string = 'embed_files' // name of the constant that will store the embedded data. + const_public bool // if true make the const public. + key_path_prefix string // path prefix for all embedded files, will be added to data map key. + file_path_prefix string // path prefix for files used in $embed_file() call in generated file. + compression bool // add zlib compression flag to $embed_file() call in generated file. + notice string = 'This file is generated by embedfs module, DO NOT EDIT!' } -struct EmbedFileSpec { - key string - path string - name string - ext string - mimetype string +@[params] +pub struct GenerateParams { + EmbedFsParams +pub: + ignore []string // glob expressions for files that should not be embedded. } -pub fn (g CodeGenerator) generate() string { - visible := if g.make_pub == true { 'pub ' } else { '' } - const_visible := if g.make_pub == true || g.make_const_pub == true { 'pub ' } else { '' } - mut b := strings.new_builder(1024 * 4) - b.writeln('// !WARNING! This file is generated by embedfs module, do not edit it.') - b.writeln('') - b.writeln('module ${g.module_name}') - b.writeln('') - b.writeln('import v.embed_file { EmbedFileData }') - b.writeln('') - b.writeln('${visible}struct EmbedFileMetadata {') - b.writeln('\tkey string') - b.writeln('\tname string') - b.writeln('\text string') - b.writeln('\tmimetype string') - b.writeln('}') - b.writeln('') - b.writeln('${visible}struct EmbedFile {') - b.writeln('\tdata EmbedFileData') - b.writeln('\tmeta EmbedFileMetadata') - b.writeln('}') - b.writeln('') - b.writeln('${visible}struct EmbedFileSystem {') - b.writeln('\tfiles map[string]EmbedFile') - b.writeln('}') - b.writeln('') - if g.bare_map { - b.writeln('${const_visible}const ${g.const_name} = {') - g.write_embed_file_map_item(mut b) - b.writeln('}') - } else { - b.writeln('${const_visible}const ${g.const_name} = EmbedFileSystem{') - b.writeln('\tfiles: {') - g.write_embed_file_map_item(mut b) - b.writeln('\t}') - b.writeln('}') +// generate generates the V code that embeds the contents of directory `dir`. +pub fn generate(dir string, params GenerateParams) !string { + files := collect_files(dir, params.ignore) + return generate_with(files, params.EmbedFsParams)! +} + +// generate_with generates the V code that embeds the files listed in `paths`. +pub fn generate_with(paths []string, params EmbedFsParams) !string { + module_name := params.module_name + const_name := params.const_name + const_public := params.const_public + compression := params.compression + notice := params.notice + skip_notice := params.notice == '' + mut files := map[string]string{} + for file in paths { + file_path := if params.file_path_prefix == '' { + file + } else { + os.join_path_single(params.file_path_prefix, file) + } + files[os.join_path_single(params.key_path_prefix, file)] = file_path } - return b.str() + result := $tmpl('embedfs.template') + return result } -fn (g CodeGenerator) get_files() []EmbedFileSpec { +fn collect_files(path string, ignore []string) []string { mut file_list := &[]string{} - os.walk(g.path, fn [mut file_list] (file string) { - file_list << file - }) - mut files := []EmbedFileSpec{} - outer: for file in file_list { - for glob in g.ignore_patterns { - if file.match_glob(glob) { - continue outer + os.walk(path, fn [mut file_list, ignore] (file string) { + for globexpr in ignore { + if file.match_glob(globexpr) { + return } } - file_key := os.join_path_single(g.prefix, file) - file_path := file // the actual path used in $embed_file() statement - file_name := os.file_name(file_path) - file_ext := os.file_ext(file_name).replace_once('.', '') - mut mimetype := mime.get_mime_type(file_ext) - if g.force_mimetype && mimetype == '' { - mimetype = g.default_mimetype - } - files << EmbedFileSpec{ - key: file_key - path: file_path - name: file_name - ext: file_ext - mimetype: mimetype - } - } - return files -} - -fn (g CodeGenerator) write_embed_file_map_item(mut b strings.Builder) { - for filespec in g.get_files() { - b.writeln("\t\t'${filespec.key}': EmbedFile{") - b.writeln("\t\t\tdata: \$embed_file('${filespec.path}')") - b.writeln('\t\t\tmeta: EmbedFileMetadata{') - b.writeln("\t\t\t\tkey: '${filespec.key}'") - b.writeln("\t\t\t\tname: '${filespec.name}'") - b.writeln("\t\t\t\text: '${filespec.ext}'") - b.writeln("\t\t\t\tmimetype: '${filespec.mimetype}'") - b.writeln('\t\t\t}') - b.writeln('\t\t},') - } + file_list << file + }) + return *file_list } diff --git a/tests/generate_test.v b/tests/generate_test.v new file mode 100644 index 0000000..2b16b48 --- /dev/null +++ b/tests/generate_test.v @@ -0,0 +1,17 @@ +import embedfs + +fn test_generate() { + expected := r"module main + +/* + This file is generated by embedfs module, DO NOT EDIT! +*/ + +const embed_files = { + 'cmd/mkembedfs/mkembedfs.v': $embed_file('cmd/mkembedfs/mkembedfs.v') + 'cmd/mkembedfs/help.txt': $embed_file('cmd/mkembedfs/help.txt') +} +" + data := embedfs.generate('cmd')! + assert data == expected +} diff --git a/tests/mymod/main.v b/tests/mymod/main.v index f2f56a2..226be6c 100644 --- a/tests/mymod/main.v +++ b/tests/mymod/main.v @@ -1,11 +1,6 @@ module main fn main() { - $if bare_map ? { - json_file := embedfs['assets/example.json'] or { EmbedFile{} } - println(json_file.data.to_string().trim_space()) - } $else { - json_file := embedfs.files['assets/example.json'] or { EmbedFile{} } - println(json_file.data.to_string().trim_space()) - } + json_file := unsafe { embed_files['assets/example.json'] } + println(json_file.to_string().trim_space()) } diff --git a/tests/mymod_test.v b/tests/mymod_test.v index 271e35a..387fab5 100644 --- a/tests/mymod_test.v +++ b/tests/mymod_test.v @@ -8,27 +8,10 @@ fn test_mymod() { oldpwd := os.getwd() expected_out := os.read_file('tests/mymod_test.out')! os.chdir('tests/mymod')! - gen := embedfs.CodeGenerator{ - path: 'assets' - } - os.write_file('assets_generated.v', gen.generate())! + gen := embedfs.generate('assets')! + os.write_file('assets_generated.v', gen)! ret := os.execute('${os.quoted_path(@VEXE)} run .') dump(diff.compare_text(ret.output, expected_out)!) assert ret.output == expected_out os.chdir(oldpwd)! } - -fn test_mymod_bare_map() { - oldpwd := os.getwd() - expected_out := os.read_file('tests/mymod_test.out')! - os.chdir('tests/mymod')! - gen := embedfs.CodeGenerator{ - path: 'assets' - bare_map: true - } - os.write_file('assets_generated.v', gen.generate())! - ret := os.execute('${os.quoted_path(@VEXE)} -d bare_map run .') - dump(diff.compare_text(ret.output, expected_out)!) - assert ret.output == expected_out - os.chdir(oldpwd)! -} diff --git a/v.mod b/v.mod index 334d7dd..038f73b 100644 --- a/v.mod +++ b/v.mod @@ -1,7 +1,7 @@ Module { name: 'embedfs' - description: 'Code generator for embedding directories with files into executables' - version: '0.0.2' + description: 'Embed directories into executables' + version: '1.0.0' license: 'Unlicense' dependencies: [] }