From 1425a802d7b27e4193708c241d54e3d7ba6f2513 Mon Sep 17 00:00:00 2001 From: Christian Stewart Date: Thu, 28 May 2026 13:19:03 -0700 Subject: [PATCH] fix(compiler): support goscript deploy embed fs Signed-off-by: Christian Stewart --- compiler/build-flags.go | 11 +- compiler/lowering.go | 213 +++++++++++++++++++++++++++++++--- compiler/skeleton_test.go | 41 +++++++ gs/embed/index.test.ts | 87 ++++++++++++++ gs/embed/index.ts | 234 +++++++++++++++++++++++++++++++++++++- 5 files changed, 561 insertions(+), 25 deletions(-) create mode 100644 gs/embed/index.test.ts diff --git a/compiler/build-flags.go b/compiler/build-flags.go index e26c46a2c..016a8353c 100644 --- a/compiler/build-flags.go +++ b/compiler/build-flags.go @@ -1,6 +1,9 @@ package compiler -import "strings" +import ( + "slices" + "strings" +) const goScriptBuildTag = "goscript" @@ -23,10 +26,8 @@ func appendBuildTag(value string, tag string) string { tags := strings.FieldsFunc(value, func(r rune) bool { return r == ',' || r == ' ' || r == '\t' || r == '\n' }) - for _, existing := range tags { - if existing == tag { - return strings.Join(tags, " ") - } + if slices.Contains(tags, tag) { + return strings.Join(tags, " ") } tags = append(tags, tag) if len(tags) == 0 { diff --git a/compiler/lowering.go b/compiler/lowering.go index 82e7553d2..8a0cd6df5 100644 --- a/compiler/lowering.go +++ b/compiler/lowering.go @@ -2013,35 +2013,218 @@ func (o *LoweringOwner) lowerGoEmbedValue( typ types.Type, patterns []string, ) (string, []Diagnostic) { + if isEmbedFSType(typ) { + return o.lowerGoEmbedFSValue(ctx, patterns) + } if len(patterns) != 1 { return "", []Diagnostic{loweringUnsupported("declaration", ctx.semPkg.pkgPath, "unsupported go:embed pattern list")} } - pattern := strings.Trim(patterns[0], "`\"") + cleanPattern, diagnostics := cleanGoEmbedFilePattern(ctx, patterns[0]) + if len(diagnostics) != 0 { + return "", diagnostics + } + data, diagnostics := readGoEmbedFile(ctx, cleanPattern) + if len(diagnostics) != 0 { + return "", diagnostics + } + if isStringType(typ) { + return strconv.Quote(string(data)), nil + } + if slice, ok := types.Unalias(typ).Underlying().(*types.Slice); ok && isByteType(slice.Elem()) { + return byteSliceLiteral(data), nil + } + diag := loweringUnsupported("declaration", ctx.semPkg.pkgPath, "unsupported go:embed target type") + diag.Detail = "target type: " + types.TypeString(typ, func(pkg *types.Package) string { + if pkg == nil { + return "" + } + return pkg.Path() + }) + return "", []Diagnostic{diag} +} + +func (o *LoweringOwner) lowerGoEmbedFSValue(ctx lowerFileContext, patterns []string) (string, []Diagnostic) { + embedAlias := ctx.importPaths["embed"] + if embedAlias == "" { + return "", []Diagnostic{loweringUnsupported("declaration", ctx.semPkg.pkgPath, "unsupported go:embed FS import")} + } + if len(patterns) == 0 { + return "", []Diagnostic{loweringUnsupported("declaration", ctx.semPkg.pkgPath, "unsupported go:embed pattern list")} + } + + filesByPath := make(map[string][]byte) + for _, pattern := range patterns { + files, diagnostics := expandGoEmbedPattern(ctx, pattern) + if len(diagnostics) != 0 { + return "", diagnostics + } + for _, file := range files { + filesByPath[file.path] = file.data + } + } + paths := make([]string, 0, len(filesByPath)) + for path := range filesByPath { + paths = append(paths, path) + } + slices.Sort(paths) + entries := make([]string, 0, len(paths)) + for _, path := range paths { + entries = append(entries, "["+strconv.Quote(path)+", "+byteSliceLiteral(filesByPath[path])+"]") + } + builtinAlias := o.runtimeOwner.BuiltinImport().Alias + return builtinAlias + ".markAsStructValue(new " + embedAlias + ".FS(new Map([" + strings.Join(entries, ", ") + "])))", nil +} + +type goEmbedFile struct { + path string + data []byte +} + +func cleanGoEmbedFilePattern(ctx lowerFileContext, pattern string) (string, []Diagnostic) { + cleanPattern, _, diagnostics := cleanGoEmbedPattern(ctx, pattern) + if len(diagnostics) != 0 { + return "", diagnostics + } + if strings.Contains(cleanPattern, "*") { + return "", []Diagnostic{loweringUnsupported("declaration", ctx.semPkg.pkgPath, "unsupported go:embed pattern")} + } + info, err := os.Stat(filepath.Join(filepath.Dir(ctx.sourcePath), filepath.FromSlash(cleanPattern))) + if err != nil { + return "", []Diagnostic{goEmbedReadDiagnostic(ctx, err)} + } + if info.IsDir() { + return "", []Diagnostic{loweringUnsupported("declaration", ctx.semPkg.pkgPath, "unsupported go:embed directory target")} + } + return cleanPattern, nil +} + +func cleanGoEmbedPattern(ctx lowerFileContext, pattern string) (string, bool, []Diagnostic) { + pattern = strings.Trim(pattern, "`\"") + all := false + if strings.HasPrefix(pattern, "all:") { + all = true + pattern = strings.TrimPrefix(pattern, "all:") + } cleanPattern := path.Clean(pattern) if pattern == "" || - strings.Contains(pattern, "*") || path.IsAbs(pattern) || cleanPattern == "." || cleanPattern == ".." || strings.HasPrefix(cleanPattern, "../") { - return "", []Diagnostic{loweringUnsupported("declaration", ctx.semPkg.pkgPath, "unsupported go:embed pattern")} + return "", false, []Diagnostic{loweringUnsupported("declaration", ctx.semPkg.pkgPath, "unsupported go:embed pattern")} + } + return cleanPattern, all, nil +} + +func expandGoEmbedPattern(ctx lowerFileContext, pattern string) ([]goEmbedFile, []Diagnostic) { + cleanPattern, all, diagnostics := cleanGoEmbedPattern(ctx, pattern) + if len(diagnostics) != 0 { + return nil, diagnostics + } + pkgDir := filepath.Dir(ctx.sourcePath) + paths := []string{filepath.Join(pkgDir, filepath.FromSlash(cleanPattern))} + if strings.Contains(cleanPattern, "*") { + matches, err := filepath.Glob(filepath.Join(pkgDir, filepath.FromSlash(cleanPattern))) + if err != nil { + return nil, []Diagnostic{loweringUnsupported("declaration", ctx.semPkg.pkgPath, "unsupported go:embed pattern")} + } + if len(matches) == 0 { + return nil, []Diagnostic{loweringUnsupported("declaration", ctx.semPkg.pkgPath, "go:embed pattern matched no files")} + } + paths = matches } - data, err := os.ReadFile(filepath.Join(filepath.Dir(ctx.sourcePath), filepath.FromSlash(cleanPattern))) + + var files []goEmbedFile + for _, path := range paths { + collected, diagnostics := collectGoEmbedPath(ctx, pkgDir, path, all) + if len(diagnostics) != 0 { + return nil, diagnostics + } + files = append(files, collected...) + } + slices.SortFunc(files, func(a, b goEmbedFile) int { + return cmp.Compare(a.path, b.path) + }) + return files, nil +} + +func collectGoEmbedPath(ctx lowerFileContext, pkgDir, absPath string, all bool) ([]goEmbedFile, []Diagnostic) { + info, err := os.Stat(absPath) if err != nil { - return "", []Diagnostic{{ - Severity: DiagnosticSeverityError, - Code: "goscript/lowering:embed", - Message: "failed to read go:embed file", - Detail: ctx.semPkg.pkgPath + ": " + err.Error(), - }} + return nil, []Diagnostic{goEmbedReadDiagnostic(ctx, err)} } - if isStringType(typ) { - return strconv.Quote(string(data)), nil + if !info.IsDir() { + file, diagnostics := readGoEmbedAbsFile(ctx, pkgDir, absPath) + if len(diagnostics) != 0 { + return nil, diagnostics + } + return []goEmbedFile{file}, nil } - if slice, ok := types.Unalias(typ).Underlying().(*types.Slice); ok && isByteType(slice.Elem()) { - return byteSliceLiteral(data), nil + + var files []goEmbedFile + if err := filepath.WalkDir(absPath, func(path string, entry os.DirEntry, err error) error { + if err != nil { + return err + } + if path != absPath && !all && (strings.HasPrefix(entry.Name(), ".") || strings.HasPrefix(entry.Name(), "_")) { + if entry.IsDir() { + return filepath.SkipDir + } + return nil + } + if entry.IsDir() { + return nil + } + file, diagnostics := readGoEmbedAbsFile(ctx, pkgDir, path) + if len(diagnostics) != 0 { + return fmt.Errorf("%s", diagnostics[0].Detail) + } + files = append(files, file) + return nil + }); err != nil { + return nil, []Diagnostic{goEmbedReadDiagnostic(ctx, err)} + } + if len(files) == 0 { + return nil, []Diagnostic{loweringUnsupported("declaration", ctx.semPkg.pkgPath, "go:embed directory matched no files")} + } + return files, nil +} + +func readGoEmbedFile(ctx lowerFileContext, cleanPattern string) ([]byte, []Diagnostic) { + file, diagnostics := readGoEmbedAbsFile(ctx, filepath.Dir(ctx.sourcePath), filepath.Join(filepath.Dir(ctx.sourcePath), filepath.FromSlash(cleanPattern))) + if len(diagnostics) != 0 { + return nil, diagnostics + } + return file.data, nil +} + +func readGoEmbedAbsFile(ctx lowerFileContext, pkgDir, absPath string) (goEmbedFile, []Diagnostic) { + relPath, err := filepath.Rel(pkgDir, absPath) + if err != nil { + return goEmbedFile{}, []Diagnostic{goEmbedReadDiagnostic(ctx, err)} + } + data, err := os.ReadFile(absPath) + if err != nil { + return goEmbedFile{}, []Diagnostic{goEmbedReadDiagnostic(ctx, err)} + } + return goEmbedFile{path: filepath.ToSlash(relPath), data: data}, nil +} + +func goEmbedReadDiagnostic(ctx lowerFileContext, err error) Diagnostic { + return Diagnostic{ + Severity: DiagnosticSeverityError, + Code: "goscript/lowering:embed", + Message: "failed to read go:embed file", + Detail: ctx.semPkg.pkgPath + ": " + err.Error(), + } +} + +func isEmbedFSType(typ types.Type) bool { + named, _ := types.Unalias(typ).(*types.Named) + if named == nil || named.Obj() == nil || named.Obj().Pkg() == nil { + return false } - return "", []Diagnostic{loweringUnsupported("declaration", ctx.semPkg.pkgPath, "unsupported go:embed target type")} + return named.Obj().Pkg().Path() == "embed" && named.Obj().Name() == "FS" } func byteSliceLiteral(data []byte) string { diff --git a/compiler/skeleton_test.go b/compiler/skeleton_test.go index 45ed340e0..6eaf63f5b 100644 --- a/compiler/skeleton_test.go +++ b/compiler/skeleton_test.go @@ -1062,6 +1062,47 @@ func TestCompilePackagesUsesEmbedOverride(t *testing.T) { } } +func TestCompilePackagesEmbedsFS(t *testing.T) { + moduleDir := writePackageGraphFixture(t, map[string]string{ + "go.mod": "module example.test/embedfs\n\ngo 1.25.3\n", + "assets/config.json": `{"ok":true}`, + "assets/nested.txt": "nested", + "extra.txt": "extra", + "main.go": strings.Join([]string{ + "package embedfs", + "import \"embed\"", + "//go:embed assets *.txt", + "var StaticFS embed.FS", + "", + }, "\n"), + }) + outputDir := filepath.Join(t.TempDir(), "output") + comp, err := NewCompiler(&Config{Dir: moduleDir, OutputPath: outputDir, AllDependencies: true}, nil, nil) + if err != nil { + t.Fatal(err.Error()) + } + + if _, err := comp.CompilePackages(context.Background(), "."); err != nil { + t.Fatal(err.Error()) + } + content, err := os.ReadFile(filepath.Join(outputDir, "@goscript", "example.test", "embedfs", "main.gs.ts")) + if err != nil { + t.Fatal(err.Error()) + } + if !strings.Contains(string(content), `export let StaticFS: embed.FS = $.markAsStructValue(new embed.FS`) { + t.Fatalf("embedded FS was not emitted as embed.FS:\n%s", string(content)) + } + if !strings.Contains(string(content), `["assets/config.json", new Uint8Array([123, 34, 111, 107, 34, 58, 116, 114, 117, 101, 125])]`) { + t.Fatalf("embedded FS file content was not emitted:\n%s", string(content)) + } + if !strings.Contains(string(content), `["assets/nested.txt", new Uint8Array([110, 101, 115, 116, 101, 100])]`) { + t.Fatalf("embedded FS directory file content was not emitted:\n%s", string(content)) + } + if !strings.Contains(string(content), `["extra.txt", new Uint8Array([101, 120, 116, 114, 97])]`) { + t.Fatalf("embedded FS glob file content was not emitted:\n%s", string(content)) + } +} + func TestCompilePackagesEmitsPackageLocalImport(t *testing.T) { moduleDir := writePackageGraphFixture(t, map[string]string{ "go.mod": "module example.test/imports\n\ngo 1.25.3\n", diff --git a/gs/embed/index.test.ts b/gs/embed/index.test.ts new file mode 100644 index 000000000..64184e7c5 --- /dev/null +++ b/gs/embed/index.test.ts @@ -0,0 +1,87 @@ +import { describe, expect, it } from 'vitest' + +import { cloneStructValue, markAsStructValue } from '@goscript/builtin/index.js' +import { EOF } from '@goscript/io/index.js' +import { ReadDir, ReadFile, Stat } from '@goscript/io/fs/index.js' + +import { FS } from './index.js' + +describe('embed.FS', () => { + it('clones embedded files as Go struct values', () => { + const original = markAsStructValue( + new FS(new Map([['config-set.bin', new Uint8Array([1, 2, 3])]])), + ) + + const cloned = cloneStructValue(original) + const [data, err] = cloned.ReadFile('config-set.bin') + + expect(err).toBeNull() + expect(Array.from(data)).toEqual([1, 2, 3]) + }) + + it('supports io/fs read, stat, and directory APIs', () => { + const fsys = markAsStructValue( + new FS( + new Map([ + ['config-set.bin', new Uint8Array([1, 2, 3])], + ['assets/config.json', new Uint8Array([4])], + ]), + ), + ) + + const [data, readErr] = ReadFile(fsys, 'config-set.bin') + expect(readErr).toBeNull() + expect(Array.from(data)).toEqual([1, 2, 3]) + data[0] = 9 + const [dataAgain, readAgainErr] = ReadFile(fsys, 'config-set.bin') + expect(readAgainErr).toBeNull() + expect(Array.from(dataAgain)).toEqual([1, 2, 3]) + + const [rootEntries, rootErr] = ReadDir(fsys, '.') + expect(rootErr).toBeNull() + expect(rootEntries!.map((entry) => entry!.Name())).toEqual([ + 'assets', + 'config-set.bin', + ]) + + const [assetInfo, statErr] = Stat(fsys, 'assets') + expect(statErr).toBeNull() + expect(assetInfo!.IsDir()).toBe(true) + + const [assetEntries, assetErr] = ReadDir(fsys, 'assets') + expect(assetErr).toBeNull() + expect(assetEntries!.map((entry) => entry!.Name())).toEqual(['config.json']) + }) + + it('supports Open file reads and directory iteration', () => { + const fsys = markAsStructValue( + new FS( + new Map([ + ['config-set.bin', new Uint8Array([1, 2, 3])], + ['assets/config.json', new Uint8Array([4])], + ]), + ), + ) + + const [file, openErr] = fsys.Open('config-set.bin') + expect(openErr).toBeNull() + const buffer = new Uint8Array(2) + const [firstRead, firstErr] = file!.Read(buffer) + expect(firstErr).toBeNull() + expect(firstRead).toBe(2) + expect(Array.from(buffer)).toEqual([1, 2]) + const [secondRead, secondErr] = file!.Read(buffer) + expect(secondErr).toBeNull() + expect(secondRead).toBe(1) + expect(Array.from(buffer)).toEqual([3, 2]) + const [eofRead, eofErr] = file!.Read(buffer) + expect(eofRead).toBe(0) + expect(eofErr).toBe(EOF) + + const [dir, dirOpenErr] = fsys.Open('.') + expect(dirOpenErr).toBeNull() + const [entries, readDirErr] = dir!.ReadDir(1) + expect(readDirErr).toBeNull() + expect(entries!.map((entry) => entry!.Name())).toEqual(['assets']) + }) +}) diff --git a/gs/embed/index.ts b/gs/embed/index.ts index 15907bbc1..cc9938f74 100644 --- a/gs/embed/index.ts +++ b/gs/embed/index.ts @@ -1,20 +1,244 @@ import * as $ from '@goscript/builtin/index.js' +import * as io from '@goscript/io/index.js' import * as fs from '@goscript/io/fs/index.js' +import * as time from '@goscript/time/index.js' export class FS { + private files: Map + + constructor(files?: Map) { + this.files = files ?? new Map() + } + + clone(): FS { + const files = new Map() + for (const [name, data] of this.files) { + files.set(name, data.slice()) + } + return $.markAsStructValue(new FS(files)) + } + Open(name: string): [fs.File, $.GoError] { - return [null, pathError('open', name)] + const err = validatePath('open', name) + if (err != null) { + return [null, err] + } + const data = this.files.get(name) + if (data !== undefined) { + return [new embedFile(name, data), null] + } + const entries = this.dirEntries(name) + if (entries === null) { + return [null, pathError('open', name, fs.ErrNotExist)] + } + return [new embedFile(name, null, entries), null] } ReadDir(name: string): [$.Slice, $.GoError] { - return [null, pathError('read', name)] + const err = validatePath('read', name) + if (err != null) { + return [null, err] + } + const entries = this.dirEntries(name) + if (entries === null) { + return [null, pathError('read', name, fs.ErrNotExist)] + } + return [entries, null] } ReadFile(name: string): [Uint8Array, $.GoError] { - return [new Uint8Array(0), pathError('read', name)] + const err = validatePath('read', name) + if (err != null) { + return [new Uint8Array(0), err] + } + const data = this.files.get(name) + if (data === undefined) { + const err = this.dirExists(name) ? fs.ErrInvalid : fs.ErrNotExist + return [new Uint8Array(0), pathError('read', name, err)] + } + return [data.slice(), null] + } + + Stat(name: string): [fs.FileInfo, $.GoError] { + const err = validatePath('stat', name) + if (err != null) { + return [null, err] + } + const data = this.files.get(name) + if (data !== undefined) { + return [new embedFileInfo(baseName(name), data.byteLength, 0o444), null] + } + if (!this.dirExists(name)) { + return [null, pathError('stat', name, fs.ErrNotExist)] + } + return [new embedFileInfo(baseName(name), 0, fs.ModeDir | 0o555), null] + } + + private dirEntries(name: string): $.Slice | null { + if (!this.dirExists(name)) { + return null + } + const prefix = name === '.' ? '' : name + '/' + const entries = new Map() + for (const [filePath, data] of this.files) { + if (prefix !== '' && !filePath.startsWith(prefix)) { + continue + } + const rest = prefix === '' ? filePath : filePath.slice(prefix.length) + if (rest === '') { + continue + } + const slash = rest.indexOf('/') + const childName = slash === -1 ? rest : rest.slice(0, slash) + if (entries.has(childName)) { + continue + } + const isDir = slash !== -1 + const info = + isDir ? + new embedFileInfo(childName, 0, fs.ModeDir | 0o555) + : new embedFileInfo(childName, data.byteLength, 0o444) + entries.set(childName, new embedDirEntry(info)) + } + return Array.from(entries.values()).sort((a, b) => + a!.Name().localeCompare(b!.Name()), + ) + } + + private dirExists(name: string): boolean { + if (name === '.') { + return true + } + const prefix = name + '/' + for (const path of this.files.keys()) { + if (path.startsWith(prefix)) { + return true + } + } + return false + } +} + +class embedFile { + private offset = 0 + private dirOffset = 0 + + constructor( + private readonly name: string, + private readonly data: Uint8Array | null, + private readonly entries: $.Slice = [], + ) {} + + Close(): $.GoError { + return null + } + + Read(buffer: Uint8Array): [number, $.GoError] { + if (this.data === null) { + return [0, pathError('read', this.name, fs.ErrInvalid)] + } + if (this.offset >= this.data.byteLength) { + return [0, io.EOF] + } + const n = Math.min(buffer.byteLength, this.data.byteLength - this.offset) + buffer.set(this.data.subarray(this.offset, this.offset + n)) + this.offset += n + return [n, null] + } + + ReadDir(n: number): [$.Slice, $.GoError] { + if (this.data !== null) { + return [null, pathError('readdir', this.name, fs.ErrInvalid)] + } + const allEntries = this.entries ?? [] + if (n <= 0) { + const entries = allEntries.slice(this.dirOffset) + this.dirOffset = allEntries.length + return [entries, null] + } + if (this.dirOffset >= allEntries.length) { + return [[], io.EOF] + } + const entries = allEntries.slice(this.dirOffset, this.dirOffset + n) + this.dirOffset += entries.length + return [entries, null] + } + + Stat(): [fs.FileInfo, $.GoError] { + if (this.data === null) { + return [new embedFileInfo(baseName(this.name), 0, fs.ModeDir | 0o555), null] + } + return [new embedFileInfo(baseName(this.name), this.data.byteLength, 0o444), null] + } +} + +class embedFileInfo { + constructor( + private readonly name: string, + private readonly size: number, + private readonly mode: fs.FileMode, + ) {} + + IsDir(): boolean { + return fs.FileMode_IsDir(this.mode) + } + + ModTime(): time.Time { + return new time.Time() + } + + Mode(): fs.FileMode { + return this.mode + } + + Name(): string { + return this.name + } + + Size(): number { + return this.size + } + + Sys(): null { + return null + } +} + +class embedDirEntry { + constructor(private readonly info: fs.FileInfo) {} + + Info(): [fs.FileInfo, $.GoError] { + return [this.info, null] + } + + IsDir(): boolean { + return this.info!.IsDir() + } + + Name(): string { + return this.info!.Name() + } + + Type(): fs.FileMode { + return fs.fileModeType(this.info!.Mode()) } } -function pathError(op: string, name: string): $.GoError { - return new fs.PathError({ Op: op, Path: name, Err: fs.ErrNotExist }) +function validatePath(op: string, name: string): $.GoError { + if (!fs.ValidPath(name)) { + return pathError(op, name, fs.ErrInvalid) + } + return null +} + +function pathError(op: string, name: string, err: $.GoError): $.GoError { + return new fs.PathError({ Op: op, Path: name, Err: err }) +} + +function baseName(name: string): string { + const idx = name.lastIndexOf('/') + if (idx === -1) { + return name + } + return name.slice(idx + 1) }