feat: support inline package.json and requirements.txt

pkg/engine/cmd.go

@@ -282,7 +282,7 @@ func (e *Engine) newCommand(ctx context.Context, extraEnv []string, tool types.T
 	)
 
 	if strings.TrimSpace(rest) != "" {
-		f, err := os.CreateTemp("", version.ProgramName+requiredFileExtensions[args[0]])
+		f, err := os.CreateTemp(env.Getenv("GPTSCRIPT_TMPDIR", envvars), version.ProgramName+requiredFileExtensions[args[0]])
 		if err != nil {
 			return nil, nil, err
 		}

pkg/env/env.go

@@ -26,6 +26,15 @@ func ToEnvLike(v string) string {
 	return strings.ToUpper(v)
 }
 
+func Getenv(key string, envs []string) string {
+	for i := len(envs) - 1; i >= 0; i-- {
+		if k, v, ok := strings.Cut(envs[i], "="); ok && k == key {
+			return v
+		}
+	}
+	return ""
+}
+
 func Matches(cmd []string, bin string) bool {
 	switch len(cmd) {
 	case 0:

pkg/parser/parser.go

@@ -16,7 +16,7 @@ import (
 var (
 	sepRegex       = regexp.MustCompile(`^\s*---+\s*$`)
 	strictSepRegex = regexp.MustCompile(`^---\n$`)
-	skipRegex      = regexp.MustCompile(`^![-\w]+\s*$`)
+	skipRegex      = regexp.MustCompile(`^![-.:\w]+\s*$`)
 )
 
 func normalize(key string) string {
@@ -308,6 +308,8 @@ func Parse(input io.Reader, opts ...Options) (Document, error) {
 		}
 	}
 
+	nodes = assignMetadata(nodes)
+
 	if !opt.AssignGlobals {
 		return Document{
 			Nodes: nodes,
@@ -359,6 +361,42 @@ func Parse(input io.Reader, opts ...Options) (Document, error) {
 	}, nil
 }
 
+func assignMetadata(nodes []Node) (result []Node) {
+	metadata := map[string]map[string]string{}
+	result = make([]Node, 0, len(nodes))
+	for _, node := range nodes {
+		if node.TextNode != nil {
+			body, ok := strings.CutPrefix(node.TextNode.Text, "!metadata:")
+			if ok {
+				line, rest, ok := strings.Cut(body, "\n")
+				if ok {
+					toolName, metaKey, ok := strings.Cut(strings.TrimSpace(line), ":")
+					if ok {
+						d, ok := metadata[toolName]
+						if !ok {
+							d = map[string]string{}
+							metadata[toolName] = d
+						}
+						d[metaKey] = strings.TrimSpace(rest)
+					}
+				}
+			}
+		}
+	}
+	if len(metadata) == 0 {
+		return nodes
+	}
+
+	for _, node := range nodes {
+		if node.ToolNode != nil {
+			node.ToolNode.Tool.MetaData = metadata[node.ToolNode.Tool.Name]
+		}
+		result = append(result, node)
+	}
+
+	return
+}
+
 func isGPTScriptHashBang(line string) bool {
 	if !strings.HasPrefix(line, "#!") {
 		return false

pkg/parser/parser_test.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/gptscript-ai/gptscript/pkg/types"
 	"github.com/hexops/autogold/v2"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
@@ -239,3 +240,31 @@ share output filters: shared
 		}},
 	}}).Equal(t, out)
 }
+
+func TestParseMetaData(t *testing.T) {
+	input := `
+name: first
+
+body
+---
+!metadata:first:package.json
+foo=base
+f
+
+---
+!metadata:first2:requirements.txt
+asdf2
+
+---
+!metadata:first:requirements.txt
+asdf
+`
+	tools, err := ParseTools(strings.NewReader(input))
+	require.NoError(t, err)
+
+	assert.Len(t, tools, 1)
+	autogold.Expect(map[string]string{
+		"package.json":     "foo=base\nf",
+		"requirements.txt": "asdf",
+	}).Equal(t, tools[0].MetaData)
+}

pkg/repos/get.go

@@ -26,8 +26,8 @@ const credentialHelpersRepo = "github.com/gptscript-ai/gptscript-credential-help
 
 type Runtime interface {
 	ID() string
-	Supports(cmd []string) bool
-	Setup(ctx context.Context, dataRoot, toolSource string, env []string) ([]string, error)
+	Supports(tool types.Tool, cmd []string) bool
+	Setup(ctx context.Context, tool types.Tool, dataRoot, toolSource string, env []string) ([]string, error)
 }
 
 type noopRuntime struct {
@@ -37,11 +37,11 @@ func (n noopRuntime) ID() string {
 	return "none"
 }
 
-func (n noopRuntime) Supports(_ []string) bool {
+func (n noopRuntime) Supports(_ types.Tool, _ []string) bool {
 	return false
 }
 
-func (n noopRuntime) Setup(_ context.Context, _, _ string, _ []string) ([]string, error) {
+func (n noopRuntime) Setup(_ context.Context, _ types.Tool, _, _ string, _ []string) ([]string, error) {
 	return nil, nil
 }
 
@@ -52,7 +52,6 @@ type Manager struct {
 	credHelperDirs   credentials.CredentialHelperDirs
 	runtimes         []Runtime
 	credHelperConfig *credHelperConfig
-	supportLocal     bool
 }
 
 type credHelperConfig struct {
@@ -62,10 +61,6 @@ type credHelperConfig struct {
 	env         []string
 }
 
-func (m *Manager) SetSupportLocal() {
-	m.supportLocal = true
-}
-
 func New(cacheDir string, runtimes ...Runtime) *Manager {
 	root := filepath.Join(cacheDir, "repos")
 	return &Manager{
@@ -216,7 +211,7 @@ func (m *Manager) setup(ctx context.Context, runtime Runtime, tool types.Tool, e
 		}
 	}
 
-	newEnv, err := runtime.Setup(ctx, m.runtimeDir, targetFinal, env)
+	newEnv, err := runtime.Setup(ctx, tool, m.runtimeDir, targetFinal, env)
 	if err != nil {
 		return "", nil, err
 	}
@@ -240,17 +235,10 @@ func (m *Manager) setup(ctx context.Context, runtime Runtime, tool types.Tool, e
 
 func (m *Manager) GetContext(ctx context.Context, tool types.Tool, cmd, env []string) (string, []string, error) {
 	var isLocal bool
-	if !m.supportLocal {
-		if tool.Source.Repo == nil {
-			return tool.WorkingDir, env, nil
-		}
-
-		if tool.Source.Repo.VCS != "git" {
-			return "", nil, fmt.Errorf("only git is supported, found VCS %s for %s", tool.Source.Repo.VCS, tool.ID)
-		}
-	} else if tool.Source.Repo == nil {
+	if tool.Source.Repo == nil {
 		isLocal = true
-		id := hash.Digest(tool)[:12]
+		d, _ := json.Marshal(tool)
+		id := hash.Digest(d)[:12]
 		tool.Source.Repo = &types.Repo{
 			VCS:      "<local>",
 			Root:     id,
@@ -261,7 +249,7 @@ func (m *Manager) GetContext(ctx context.Context, tool types.Tool, cmd, env []st
 	}
 
 	for _, runtime := range m.runtimes {
-		if runtime.Supports(cmd) {
+		if runtime.Supports(tool, cmd) {
 			log.Debugf("Runtime %s supports %v", runtime.ID(), cmd)
 			return m.setup(ctx, runtime, tool, env)
 		}

pkg/repos/runtimes/busybox/busybox.go

@@ -18,6 +18,7 @@ import (
 	runtimeEnv "github.com/gptscript-ai/gptscript/pkg/env"
 	"github.com/gptscript-ai/gptscript/pkg/hash"
 	"github.com/gptscript-ai/gptscript/pkg/repos/download"
+	"github.com/gptscript-ai/gptscript/pkg/types"
 )
 
 //go:embed SHASUMS256.txt
@@ -32,7 +33,7 @@ func (r *Runtime) ID() string {
 	return "busybox"
 }
 
-func (r *Runtime) Supports(cmd []string) bool {
+func (r *Runtime) Supports(_ types.Tool, cmd []string) bool {
 	if runtime.GOOS != "windows" {
 		return false
 	}
@@ -44,7 +45,7 @@ func (r *Runtime) Supports(cmd []string) bool {
 	return false
 }
 
-func (r *Runtime) Setup(ctx context.Context, dataRoot, _ string, env []string) ([]string, error) {
+func (r *Runtime) Setup(ctx context.Context, _ types.Tool, dataRoot, _ string, env []string) ([]string, error) {
 	binPath, err := r.getRuntime(ctx, dataRoot)
 	if err != nil {
 		return nil, err

pkg/repos/runtimes/busybox/busybox_test.go

@@ -11,6 +11,7 @@ import (
 	"testing"
 
 	"github.com/adrg/xdg"
+	"github.com/gptscript-ai/gptscript/pkg/types"
 	"github.com/samber/lo"
 	"github.com/stretchr/testify/require"
 )
@@ -31,7 +32,7 @@ func TestRuntime(t *testing.T) {
 
 	r := Runtime{}
 
-	s, err := r.Setup(context.Background(), testCacheHome, "testdata", os.Environ())
+	s, err := r.Setup(context.Background(), types.Tool{}, testCacheHome, "testdata", os.Environ())
 	require.NoError(t, err)
 	_, err = os.Stat(filepath.Join(firstPath(s), "busybox.exe"))
 	if errors.Is(err, fs.ErrNotExist) {

pkg/repos/runtimes/golang/golang.go

@@ -18,6 +18,7 @@ import (
 	runtimeEnv "github.com/gptscript-ai/gptscript/pkg/env"
 	"github.com/gptscript-ai/gptscript/pkg/hash"
 	"github.com/gptscript-ai/gptscript/pkg/repos/download"
+	"github.com/gptscript-ai/gptscript/pkg/types"
 )
 
 //go:embed digests.txt
@@ -34,11 +35,12 @@ func (r *Runtime) ID() string {
 	return "go" + r.Version
 }
 
-func (r *Runtime) Supports(cmd []string) bool {
-	return len(cmd) > 0 && cmd[0] == "${GPTSCRIPT_TOOL_DIR}/bin/gptscript-go-tool"
+func (r *Runtime) Supports(tool types.Tool, cmd []string) bool {
+	return tool.Source.IsGit() &&
+		len(cmd) > 0 && cmd[0] == "${GPTSCRIPT_TOOL_DIR}/bin/gptscript-go-tool"
 }
 
-func (r *Runtime) Setup(ctx context.Context, dataRoot, toolSource string, env []string) ([]string, error) {
+func (r *Runtime) Setup(ctx context.Context, _ types.Tool, dataRoot, toolSource string, env []string) ([]string, error) {
 	binPath, err := r.getRuntime(ctx, dataRoot)
 	if err != nil {
 		return nil, err

pkg/repos/runtimes/golang/golang_test.go

@@ -10,6 +10,7 @@ import (
 	"testing"
 
 	"github.com/adrg/xdg"
+	"github.com/gptscript-ai/gptscript/pkg/types"
 	"github.com/samber/lo"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -27,7 +28,7 @@ func TestRuntime(t *testing.T) {
 		Version: "1.22.1",
 	}
 
-	s, err := r.Setup(context.Background(), testCacheHome, "testdata", os.Environ())
+	s, err := r.Setup(context.Background(), types.Tool{}, testCacheHome, "testdata", os.Environ())
 	require.NoError(t, err)
 	p, v, _ := strings.Cut(s[0], "=")
 	v, _, _ = strings.Cut(v, string(filepath.ListSeparator))

pkg/repos/runtimes/node/node.go

@@ -17,12 +17,16 @@ import (
 	runtimeEnv "github.com/gptscript-ai/gptscript/pkg/env"
 	"github.com/gptscript-ai/gptscript/pkg/hash"
 	"github.com/gptscript-ai/gptscript/pkg/repos/download"
+	"github.com/gptscript-ai/gptscript/pkg/types"
 )
 
 //go:embed SHASUMS256.txt.asc
 var releasesData []byte
 
-const downloadURL = "https://nodejs.org/dist/%s/"
+const (
+	downloadURL = "https://nodejs.org/dist/%s/"
+	packageJSON = "package.json"
+)
 
 type Runtime struct {
 	// version something like "3.12"
@@ -35,7 +39,10 @@ func (r *Runtime) ID() string {
 	return "node" + r.Version
 }
 
-func (r *Runtime) Supports(cmd []string) bool {
+func (r *Runtime) Supports(tool types.Tool, cmd []string) bool {
+	if _, hasPackageJSON := tool.MetaData[packageJSON]; !hasPackageJSON && !tool.Source.IsGit() {
+		return false
+	}
 	for _, testCmd := range []string{"node", "npx", "npm"} {
 		if r.supports(testCmd, cmd) {
 			return true
@@ -54,17 +61,21 @@ func (r *Runtime) supports(testCmd string, cmd []string) bool {
 	return runtimeEnv.Matches(cmd, testCmd)
 }
 
-func (r *Runtime) Setup(ctx context.Context, dataRoot, toolSource string, env []string) ([]string, error) {
+func (r *Runtime) Setup(ctx context.Context, tool types.Tool, dataRoot, toolSource string, env []string) ([]string, error) {
 	binPath, err := r.getRuntime(ctx, dataRoot)
 	if err != nil {
 		return nil, err
 	}
 
 	newEnv := runtimeEnv.AppendPath(env, binPath)
-	if err := r.runNPM(ctx, toolSource, binPath, append(env, newEnv...)); err != nil {
+	if err := r.runNPM(ctx, tool, toolSource, binPath, append(env, newEnv...)); err != nil {
 		return nil, err
 	}
 
+	if _, ok := tool.MetaData[packageJSON]; ok {
+		newEnv = append(newEnv, "GPTSCRIPT_TMPDIR="+toolSource)
+	}
+
 	return newEnv, nil
 }
 
@@ -100,11 +111,16 @@ func (r *Runtime) getReleaseAndDigest() (string, string, error) {
 	return "", "", fmt.Errorf("failed to find %s release for os=%s arch=%s", r.ID(), osName(), arch())
 }
 
-func (r *Runtime) runNPM(ctx context.Context, toolSource, binDir string, env []string) error {
+func (r *Runtime) runNPM(ctx context.Context, tool types.Tool, toolSource, binDir string, env []string) error {
 	log.InfofCtx(ctx, "Running npm in %s", toolSource)
 	cmd := debugcmd.New(ctx, filepath.Join(binDir, "npm"), "install")
 	cmd.Env = env
 	cmd.Dir = toolSource
+	if contents, ok := tool.MetaData[packageJSON]; ok {
+		if err := os.WriteFile(filepath.Join(toolSource, packageJSON), []byte(contents+"\n"), 0644); err != nil {
+			return err
+		}
+	}
 	return cmd.Run()
 }
 

pkg/repos/runtimes/node/node_test.go

@@ -10,6 +10,7 @@ import (
 	"testing"
 
 	"github.com/adrg/xdg"
+	"github.com/gptscript-ai/gptscript/pkg/types"
 	"github.com/samber/lo"
 	"github.com/stretchr/testify/require"
 )
@@ -28,7 +29,7 @@ func TestRuntime(t *testing.T) {
 		Version: "20",
 	}
 
-	s, err := r.Setup(context.Background(), testCacheHome, "testdata", os.Environ())
+	s, err := r.Setup(context.Background(), types.Tool{}, testCacheHome, "testdata", os.Environ())
 	require.NoError(t, err)
 	_, err = os.Stat(filepath.Join(firstPath(s), "node.exe"))
 	if errors.Is(err, fs.ErrNotExist) {
@@ -42,7 +43,7 @@ func TestRuntime21(t *testing.T) {
 		Version: "21",
 	}
 
-	s, err := r.Setup(context.Background(), testCacheHome, "testdata", os.Environ())
+	s, err := r.Setup(context.Background(), types.Tool{}, testCacheHome, "testdata", os.Environ())
 	require.NoError(t, err)
 	_, err = os.Stat(filepath.Join(firstPath(s), "node.exe"))
 	if errors.Is(err, fs.ErrNotExist) {