fix(compiler): support goscript deploy embed fs

compiler/build-flags.go

@@ -1,6 +1,9 @@
 package compiler
 
-import "strings"
+import (
+	"slices"
+	"strings"
+)
 
 const goScriptBuildTag = "goscript"
 
@@ -23,10 +26,8 @@ func appendBuildTag(value string, tag string) string {
 	tags := strings.FieldsFunc(value, func(r rune) bool {
 		return r == ',' || r == ' ' || r == '\t' || r == '\n'
 	})
-	for _, existing := range tags {
-		if existing == tag {
-			return strings.Join(tags, " ")
-		}
+	if slices.Contains(tags, tag) {
+		return strings.Join(tags, " ")
 	}
 	tags = append(tags, tag)
 	if len(tags) == 0 {

compiler/lowering.go

@@ -2013,35 +2013,218 @@ func (o *LoweringOwner) lowerGoEmbedValue(
 	typ types.Type,
 	patterns []string,
 ) (string, []Diagnostic) {
+	if isEmbedFSType(typ) {
+		return o.lowerGoEmbedFSValue(ctx, patterns)
+	}
 	if len(patterns) != 1 {
 		return "", []Diagnostic{loweringUnsupported("declaration", ctx.semPkg.pkgPath, "unsupported go:embed pattern list")}
 	}
-	pattern := strings.Trim(patterns[0], "`\"")
+	cleanPattern, diagnostics := cleanGoEmbedFilePattern(ctx, patterns[0])
+	if len(diagnostics) != 0 {
+		return "", diagnostics
+	}
+	data, diagnostics := readGoEmbedFile(ctx, cleanPattern)
+	if len(diagnostics) != 0 {
+		return "", diagnostics
+	}
+	if isStringType(typ) {
+		return strconv.Quote(string(data)), nil
+	}
+	if slice, ok := types.Unalias(typ).Underlying().(*types.Slice); ok && isByteType(slice.Elem()) {
+		return byteSliceLiteral(data), nil
+	}
+	diag := loweringUnsupported("declaration", ctx.semPkg.pkgPath, "unsupported go:embed target type")
+	diag.Detail = "target type: " + types.TypeString(typ, func(pkg *types.Package) string {
+		if pkg == nil {
+			return ""
+		}
+		return pkg.Path()
+	})
+	return "", []Diagnostic{diag}
+}
+
+func (o *LoweringOwner) lowerGoEmbedFSValue(ctx lowerFileContext, patterns []string) (string, []Diagnostic) {
+	embedAlias := ctx.importPaths["embed"]
+	if embedAlias == "" {
+		return "", []Diagnostic{loweringUnsupported("declaration", ctx.semPkg.pkgPath, "unsupported go:embed FS import")}
+	}
+	if len(patterns) == 0 {
+		return "", []Diagnostic{loweringUnsupported("declaration", ctx.semPkg.pkgPath, "unsupported go:embed pattern list")}
+	}
+
+	filesByPath := make(map[string][]byte)
+	for _, pattern := range patterns {
+		files, diagnostics := expandGoEmbedPattern(ctx, pattern)
+		if len(diagnostics) != 0 {
+			return "", diagnostics
+		}
+		for _, file := range files {
+			filesByPath[file.path] = file.data
+		}
+	}
+	paths := make([]string, 0, len(filesByPath))
+	for path := range filesByPath {
+		paths = append(paths, path)
+	}
+	slices.Sort(paths)
+	entries := make([]string, 0, len(paths))
+	for _, path := range paths {
+		entries = append(entries, "["+strconv.Quote(path)+", "+byteSliceLiteral(filesByPath[path])+"]")
+	}
+	builtinAlias := o.runtimeOwner.BuiltinImport().Alias
+	return builtinAlias + ".markAsStructValue(new " + embedAlias + ".FS(new Map<string, Uint8Array>([" + strings.Join(entries, ", ") + "])))", nil
+}
+
+type goEmbedFile struct {
+	path string
+	data []byte
+}
+
+func cleanGoEmbedFilePattern(ctx lowerFileContext, pattern string) (string, []Diagnostic) {
+	cleanPattern, _, diagnostics := cleanGoEmbedPattern(ctx, pattern)
+	if len(diagnostics) != 0 {
+		return "", diagnostics
+	}
+	if strings.Contains(cleanPattern, "*") {
+		return "", []Diagnostic{loweringUnsupported("declaration", ctx.semPkg.pkgPath, "unsupported go:embed pattern")}
+	}
+	info, err := os.Stat(filepath.Join(filepath.Dir(ctx.sourcePath), filepath.FromSlash(cleanPattern)))
+	if err != nil {
+		return "", []Diagnostic{goEmbedReadDiagnostic(ctx, err)}
+	}
+	if info.IsDir() {
+		return "", []Diagnostic{loweringUnsupported("declaration", ctx.semPkg.pkgPath, "unsupported go:embed directory target")}
+	}
+	return cleanPattern, nil
+}
+
+func cleanGoEmbedPattern(ctx lowerFileContext, pattern string) (string, bool, []Diagnostic) {
+	pattern = strings.Trim(pattern, "`\"")
+	all := false
+	if strings.HasPrefix(pattern, "all:") {
+		all = true
+		pattern = strings.TrimPrefix(pattern, "all:")
+	}
 	cleanPattern := path.Clean(pattern)
 	if pattern == "" ||
-		strings.Contains(pattern, "*") ||
 		path.IsAbs(pattern) ||
 		cleanPattern == "." ||
 		cleanPattern == ".." ||
 		strings.HasPrefix(cleanPattern, "../") {
-		return "", []Diagnostic{loweringUnsupported("declaration", ctx.semPkg.pkgPath, "unsupported go:embed pattern")}
+		return "", false, []Diagnostic{loweringUnsupported("declaration", ctx.semPkg.pkgPath, "unsupported go:embed pattern")}
+	}
+	return cleanPattern, all, nil
+}
+
+func expandGoEmbedPattern(ctx lowerFileContext, pattern string) ([]goEmbedFile, []Diagnostic) {
+	cleanPattern, all, diagnostics := cleanGoEmbedPattern(ctx, pattern)
+	if len(diagnostics) != 0 {
+		return nil, diagnostics
+	}
+	pkgDir := filepath.Dir(ctx.sourcePath)
+	paths := []string{filepath.Join(pkgDir, filepath.FromSlash(cleanPattern))}
+	if strings.Contains(cleanPattern, "*") {
+		matches, err := filepath.Glob(filepath.Join(pkgDir, filepath.FromSlash(cleanPattern)))
+		if err != nil {
+			return nil, []Diagnostic{loweringUnsupported("declaration", ctx.semPkg.pkgPath, "unsupported go:embed pattern")}
+		}
+		if len(matches) == 0 {
+			return nil, []Diagnostic{loweringUnsupported("declaration", ctx.semPkg.pkgPath, "go:embed pattern matched no files")}
+		}
+		paths = matches
 	}
-	data, err := os.ReadFile(filepath.Join(filepath.Dir(ctx.sourcePath), filepath.FromSlash(cleanPattern)))
+
+	var files []goEmbedFile
+	for _, path := range paths {
+		collected, diagnostics := collectGoEmbedPath(ctx, pkgDir, path, all)
+		if len(diagnostics) != 0 {
+			return nil, diagnostics
+		}
+		files = append(files, collected...)
+	}
+	slices.SortFunc(files, func(a, b goEmbedFile) int {
+		return cmp.Compare(a.path, b.path)
+	})
+	return files, nil
+}
+
+func collectGoEmbedPath(ctx lowerFileContext, pkgDir, absPath string, all bool) ([]goEmbedFile, []Diagnostic) {
+	info, err := os.Stat(absPath)
 	if err != nil {
-		return "", []Diagnostic{{
-			Severity: DiagnosticSeverityError,
-			Code:     "goscript/lowering:embed",
-			Message:  "failed to read go:embed file",
-			Detail:   ctx.semPkg.pkgPath + ": " + err.Error(),
-		}}
+		return nil, []Diagnostic{goEmbedReadDiagnostic(ctx, err)}
 	}
-	if isStringType(typ) {
-		return strconv.Quote(string(data)), nil
+	if !info.IsDir() {
+		file, diagnostics := readGoEmbedAbsFile(ctx, pkgDir, absPath)
+		if len(diagnostics) != 0 {
+			return nil, diagnostics
+		}
+		return []goEmbedFile{file}, nil
 	}
-	if slice, ok := types.Unalias(typ).Underlying().(*types.Slice); ok && isByteType(slice.Elem()) {
-		return byteSliceLiteral(data), nil
+
+	var files []goEmbedFile
+	if err := filepath.WalkDir(absPath, func(path string, entry os.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if path != absPath && !all && (strings.HasPrefix(entry.Name(), ".") || strings.HasPrefix(entry.Name(), "_")) {
+			if entry.IsDir() {
+				return filepath.SkipDir
+			}
+			return nil
+		}
+		if entry.IsDir() {
+			return nil
+		}
+		file, diagnostics := readGoEmbedAbsFile(ctx, pkgDir, path)
+		if len(diagnostics) != 0 {
+			return fmt.Errorf("%s", diagnostics[0].Detail)
+		}
+		files = append(files, file)
+		return nil
+	}); err != nil {
+		return nil, []Diagnostic{goEmbedReadDiagnostic(ctx, err)}
+	}
+	if len(files) == 0 {
+		return nil, []Diagnostic{loweringUnsupported("declaration", ctx.semPkg.pkgPath, "go:embed directory matched no files")}
+	}
+	return files, nil
+}
+
+func readGoEmbedFile(ctx lowerFileContext, cleanPattern string) ([]byte, []Diagnostic) {
+	file, diagnostics := readGoEmbedAbsFile(ctx, filepath.Dir(ctx.sourcePath), filepath.Join(filepath.Dir(ctx.sourcePath), filepath.FromSlash(cleanPattern)))
+	if len(diagnostics) != 0 {
+		return nil, diagnostics
+	}
+	return file.data, nil
+}
+
+func readGoEmbedAbsFile(ctx lowerFileContext, pkgDir, absPath string) (goEmbedFile, []Diagnostic) {
+	relPath, err := filepath.Rel(pkgDir, absPath)
+	if err != nil {
+		return goEmbedFile{}, []Diagnostic{goEmbedReadDiagnostic(ctx, err)}
+	}
+	data, err := os.ReadFile(absPath)
+	if err != nil {
+		return goEmbedFile{}, []Diagnostic{goEmbedReadDiagnostic(ctx, err)}
+	}
+	return goEmbedFile{path: filepath.ToSlash(relPath), data: data}, nil
+}
+
+func goEmbedReadDiagnostic(ctx lowerFileContext, err error) Diagnostic {
+	return Diagnostic{
+		Severity: DiagnosticSeverityError,
+		Code:     "goscript/lowering:embed",
+		Message:  "failed to read go:embed file",
+		Detail:   ctx.semPkg.pkgPath + ": " + err.Error(),
+	}
+}
+
+func isEmbedFSType(typ types.Type) bool {
+	named, _ := types.Unalias(typ).(*types.Named)
+	if named == nil || named.Obj() == nil || named.Obj().Pkg() == nil {
+		return false
 	}
-	return "", []Diagnostic{loweringUnsupported("declaration", ctx.semPkg.pkgPath, "unsupported go:embed target type")}
+	return named.Obj().Pkg().Path() == "embed" && named.Obj().Name() == "FS"
 }
 
 func byteSliceLiteral(data []byte) string {

compiler/skeleton_test.go

@@ -1062,6 +1062,47 @@ func TestCompilePackagesUsesEmbedOverride(t *testing.T) {
 	}
 }
 
+func TestCompilePackagesEmbedsFS(t *testing.T) {
+	moduleDir := writePackageGraphFixture(t, map[string]string{
+		"go.mod":             "module example.test/embedfs\n\ngo 1.25.3\n",
+		"assets/config.json": `{"ok":true}`,
+		"assets/nested.txt":  "nested",
+		"extra.txt":          "extra",
+		"main.go": strings.Join([]string{
+			"package embedfs",
+			"import \"embed\"",
+			"//go:embed assets *.txt",
+			"var StaticFS embed.FS",
+			"",
+		}, "\n"),
+	})
+	outputDir := filepath.Join(t.TempDir(), "output")
+	comp, err := NewCompiler(&Config{Dir: moduleDir, OutputPath: outputDir, AllDependencies: true}, nil, nil)
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+
+	if _, err := comp.CompilePackages(context.Background(), "."); err != nil {
+		t.Fatal(err.Error())
+	}
+	content, err := os.ReadFile(filepath.Join(outputDir, "@goscript", "example.test", "embedfs", "main.gs.ts"))
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+	if !strings.Contains(string(content), `export let StaticFS: embed.FS = $.markAsStructValue(new embed.FS`) {
+		t.Fatalf("embedded FS was not emitted as embed.FS:\n%s", string(content))
+	}
+	if !strings.Contains(string(content), `["assets/config.json", new Uint8Array([123, 34, 111, 107, 34, 58, 116, 114, 117, 101, 125])]`) {
+		t.Fatalf("embedded FS file content was not emitted:\n%s", string(content))
+	}
+	if !strings.Contains(string(content), `["assets/nested.txt", new Uint8Array([110, 101, 115, 116, 101, 100])]`) {
+		t.Fatalf("embedded FS directory file content was not emitted:\n%s", string(content))
+	}
+	if !strings.Contains(string(content), `["extra.txt", new Uint8Array([101, 120, 116, 114, 97])]`) {
+		t.Fatalf("embedded FS glob file content was not emitted:\n%s", string(content))
+	}
+}
+
 func TestCompilePackagesEmitsPackageLocalImport(t *testing.T) {
 	moduleDir := writePackageGraphFixture(t, map[string]string{
 		"go.mod": "module example.test/imports\n\ngo 1.25.3\n",

gs/embed/index.test.ts

@@ -0,0 +1,87 @@
+import { describe, expect, it } from 'vitest'
+
+import { cloneStructValue, markAsStructValue } from '@goscript/builtin/index.js'
+import { EOF } from '@goscript/io/index.js'
+import { ReadDir, ReadFile, Stat } from '@goscript/io/fs/index.js'
+
+import { FS } from './index.js'
+
+describe('embed.FS', () => {
+  it('clones embedded files as Go struct values', () => {
+    const original = markAsStructValue(
+      new FS(new Map([['config-set.bin', new Uint8Array([1, 2, 3])]])),
+    )
+
+    const cloned = cloneStructValue(original)
+    const [data, err] = cloned.ReadFile('config-set.bin')
+
+    expect(err).toBeNull()
+    expect(Array.from(data)).toEqual([1, 2, 3])
+  })
+
+  it('supports io/fs read, stat, and directory APIs', () => {
+    const fsys = markAsStructValue(
+      new FS(
+        new Map([
+          ['config-set.bin', new Uint8Array([1, 2, 3])],
+          ['assets/config.json', new Uint8Array([4])],
+        ]),
+      ),
+    )
+
+    const [data, readErr] = ReadFile(fsys, 'config-set.bin')
+    expect(readErr).toBeNull()
+    expect(Array.from(data)).toEqual([1, 2, 3])
+    data[0] = 9
+    const [dataAgain, readAgainErr] = ReadFile(fsys, 'config-set.bin')
+    expect(readAgainErr).toBeNull()
+    expect(Array.from(dataAgain)).toEqual([1, 2, 3])
+
+    const [rootEntries, rootErr] = ReadDir(fsys, '.')
+    expect(rootErr).toBeNull()
+    expect(rootEntries!.map((entry) => entry!.Name())).toEqual([
+      'assets',
+      'config-set.bin',
+    ])
+
+    const [assetInfo, statErr] = Stat(fsys, 'assets')
+    expect(statErr).toBeNull()
+    expect(assetInfo!.IsDir()).toBe(true)
+
+    const [assetEntries, assetErr] = ReadDir(fsys, 'assets')
+    expect(assetErr).toBeNull()
+    expect(assetEntries!.map((entry) => entry!.Name())).toEqual(['config.json'])
+  })
+
+  it('supports Open file reads and directory iteration', () => {
+    const fsys = markAsStructValue(
+      new FS(
+        new Map([
+          ['config-set.bin', new Uint8Array([1, 2, 3])],
+          ['assets/config.json', new Uint8Array([4])],
+        ]),
+      ),
+    )
+
+    const [file, openErr] = fsys.Open('config-set.bin')
+    expect(openErr).toBeNull()
+    const buffer = new Uint8Array(2)
+    const [firstRead, firstErr] = file!.Read(buffer)
+    expect(firstErr).toBeNull()
+    expect(firstRead).toBe(2)
+    expect(Array.from(buffer)).toEqual([1, 2])
+    const [secondRead, secondErr] = file!.Read(buffer)
+    expect(secondErr).toBeNull()
+    expect(secondRead).toBe(1)
+    expect(Array.from(buffer)).toEqual([3, 2])
+    const [eofRead, eofErr] = file!.Read(buffer)
+    expect(eofRead).toBe(0)
+    expect(eofErr).toBe(EOF)
+
+    const [dir, dirOpenErr] = fsys.Open('.')
+    expect(dirOpenErr).toBeNull()
+    const [entries, readDirErr] = dir!.ReadDir(1)
+    expect(readDirErr).toBeNull()
+    expect(entries!.map((entry) => entry!.Name())).toEqual(['assets'])
+  })
+})