Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 13 additions & 2 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -516,10 +516,18 @@ jobs:
matrix:
template:
- default.yaml
create_arg:
- ""
- "--network=vzNAT"
include:
- template: default.yaml
create_arg: "--network=vzNAT"
additional_env: |
_LIMA_DIRECT_IP_PORT_FORWARDER=true
steps:
- name: "Adjust LIMACTL_CREATE_ARGS"
# --cpus=1 is needed for running vz on GHA: https://github.com/lima-vm/lima/pull/1511#issuecomment-1574937888
run: echo "LIMACTL_CREATE_ARGS=${LIMACTL_CREATE_ARGS} --cpus 1 --memory 1" >>$GITHUB_ENV
run: echo "LIMACTL_CREATE_ARGS=${LIMACTL_CREATE_ARGS} --cpus 1 --memory 1 ${{ matrix.create_arg }}" >>$GITHUB_ENV
- uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
- uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v6.0.0
with:
Expand All @@ -536,12 +544,15 @@ jobs:
run: brew install bash coreutils w3m socat
- name: Uninstall qemu
run: brew uninstall --ignore-dependencies --force qemu
- name: Set additional environment variables
if: matrix.additional_env != null
run: echo "${{ matrix.additional_env }}" >>$GITHUB_ENV
- name: Test
run: ./hack/test-templates.sh templates/${{ matrix.template }}
- if: failure()
uses: ./.github/actions/upload_failure_logs_if_exists
with:
suffix: ${{ matrix.template }}
suffix: ${{ matrix.template }}${{ matrix.create_arg }}
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

needs matrix.additional_env too?


# gomodjail is a library sandbox for Go
# https://github.com/AkihiroSuda/gomodjail
Expand Down
5 changes: 3 additions & 2 deletions cmd/limactl/shell.go
Original file line number Diff line number Diff line change
Expand Up @@ -282,10 +282,11 @@ func shellAction(cmd *cobra.Command, args []string) error {
if olderSSH {
logLevel = "QUIET"
}
sshAddress, sshPort := inst.SSHAddressPort()
sshArgs = append(sshArgs, []string{
"-o", fmt.Sprintf("LogLevel=%s", logLevel),
"-p", strconv.Itoa(inst.SSHLocalPort),
inst.SSHAddress,
"-p", strconv.Itoa(sshPort),
sshAddress,
"--",
script,
}...)
Expand Down
5 changes: 3 additions & 2 deletions cmd/limactl/show-ssh.go
Original file line number Diff line number Diff line change
Expand Up @@ -109,8 +109,9 @@ func showSSHAction(cmd *cobra.Command, args []string) error {
if err != nil {
return err
}
opts = append(opts, "Hostname=127.0.0.1")
opts = append(opts, fmt.Sprintf("Port=%d", inst.SSHLocalPort))
sshAddress, sshPort := inst.SSHAddressPort()
opts = append(opts, fmt.Sprintf("Hostname=%s", sshAddress))
opts = append(opts, fmt.Sprintf("Port=%d", sshPort))
return sshutil.Format(w, "ssh", instName, format, opts)
}

Expand Down
5 changes: 3 additions & 2 deletions cmd/limactl/tunnel.go
Original file line number Diff line number Diff line change
Expand Up @@ -103,13 +103,14 @@ func tunnelAction(cmd *cobra.Command, args []string) error {
}
sshArgs := append([]string{}, sshExe.Args...)
sshArgs = append(sshArgs, sshutil.SSHArgsFromOpts(sshOpts)...)
sshAddress, sshPort := inst.SSHAddressPort()
sshArgs = append(sshArgs, []string{
"-q", // quiet
"-f", // background
"-N", // no command
"-D", fmt.Sprintf("127.0.0.1:%d", port),
"-p", strconv.Itoa(inst.SSHLocalPort),
inst.SSHAddress,
"-p", strconv.Itoa(sshPort),
sshAddress,
}...)
sshCmd := exec.CommandContext(ctx, sshExe.Exe, sshArgs...)
sshCmd.Stdout = stderr
Expand Down
3 changes: 2 additions & 1 deletion hack/test-port-forwarding.pl
Original file line number Diff line number Diff line change
Expand Up @@ -191,6 +191,7 @@

my $sudo = $test->{guest_port} < 1024 ? "sudo " : "";
print $lima "${sudo}${cmd} >$listener.${id} 2>/dev/null &\n";
print "Running in guest: ${sudo}${cmd} >$listener.${id} 2>/dev/null &\n";
}

# Make sure the guest- and hostagents had enough time to set up the forwards
Expand All @@ -211,7 +212,7 @@
my $tcp_dest = $test->{host_ip} =~ /:/ ? "TCP6:[$test->{host_ip}]:$test->{host_port}" : "TCP:$test->{host_ip}:$test->{host_port}";
$cmd = $test->{host_socket} eq "" ? "socat -u STDIN $tcp_dest,connect-timeout=$connectionTimeout" : "socat -u STDIN UNIX-CONNECT:$test->{host_socket}";
}
print "Running: $cmd\n";
print "Running in host: $cmd\n";
open(my $netcat, "| $cmd") or die "Can't run '$cmd': $!";
print $netcat "$test->{log_msg}\n";
# Don't check for errors on close; macOS nc seems to return non-zero exit code even on success
Expand Down
29 changes: 13 additions & 16 deletions hack/test-templates.sh
Original file line number Diff line number Diff line change
Expand Up @@ -336,30 +336,20 @@ if [[ -n ${CHECKS["ssh-over-vsock"]} ]]; then
if [[ "$(limactl ls "${NAME}" --yq .vmType)" == "vz" ]]; then
INFO "Testing SSH over vsock"
set -x
INFO "Testing LIMA_SSH_OVER_VSOCK=true environment"
limactl stop "${NAME}"
# Detection of the SSH server on VSOCK may fail; however, a failing log indicates that controlling detection via the environment variable works as expected.
if ! LIMA_SSH_OVER_VSOCK=true limactl start "${NAME}" 2>&1 | grep -i -E "(started vsock forwarder|Failed to detect SSH server on vsock)"; then
set +x
diagnose "${NAME}"
ERROR "LIMA_SSH_OVER_VSOCK=true did not enable vsock forwarder"
exit 1
fi
INFO 'Testing LIMA_SSH_OVER_VSOCK="" environment'
INFO "Testing LIMA_SSH_OVER_VSOCK=false environment"
limactl stop "${NAME}"
# Detection of the SSH server on VSOCK may fail; however, a failing log indicates that controlling detection via the environment variable works as expected.
if ! LIMA_SSH_OVER_VSOCK="" limactl start "${NAME}" 2>&1 | grep -i -E "(started vsock forwarder|Failed to detect SSH server on vsock)"; then
if ! LIMA_SSH_OVER_VSOCK=false limactl start "${NAME}" 2>&1 | grep -i "skipping detection of SSH server on vsock port"; then
set +x
diagnose "${NAME}"
ERROR "LIMA_SSH_OVER_VSOCK= did not enable vsock forwarder"
ERROR "LIMA_SSH_OVER_VSOCK=false did not disable vsock forwarder"
exit 1
fi
INFO "Testing LIMA_SSH_OVER_VSOCK=false environment"
INFO "Testing LIMA_SSH_OVER_VSOCK=true environment"
limactl stop "${NAME}"
if ! LIMA_SSH_OVER_VSOCK=false limactl start "${NAME}" 2>&1 | grep -i "skipping detection of SSH server on vsock port"; then
if ! LIMA_SSH_OVER_VSOCK=true limactl start "${NAME}" 2>&1 | grep -i -E "(started vsock forwarder|Failed to detect SSH server on vsock)"; then
set +x
diagnose "${NAME}"
ERROR "LIMA_SSH_OVER_VSOCK=false did not disable vsock forwarder"
ERROR "LIMA_SSH_OVER_VSOCK=true did not enable vsock forwarder"
exit 1
fi
set +x
Expand Down Expand Up @@ -445,6 +435,13 @@ if [[ -n ${CHECKS["port-forwards"]} ]]; then
if limactl shell "${NAME}" command -v dnf; then
limactl shell "${NAME}" sudo dnf install -y nc socat
fi
# print routing table for debugging
case "${OS_HOST}" in
"Darwin") netstat -rn ;;
"GNU/Linux") ip route show ;;
"Msys") route print ;;
*) ;;
esac
if "${scriptdir}/test-port-forwarding.pl" "${NAME}" socat $PORT_FORWARDING_CONNECTION_TIMEOUT; then
INFO "Port forwarding rules work"
else
Expand Down
6 changes: 5 additions & 1 deletion pkg/hostagent/api/api.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,13 @@

package api

import "net"

type Info struct {
// indicate instance is started by launchd or systemd if not empty
AutoStartedIdentifier string `json:"autoStartedIdentifier,omitempty"`
// SSHLocalPort is the local port on the host for SSH access to the VM.
// Guest IP address directly accessible from the host.
GuestIP net.IP `json:"guestIP,omitempty"`
// SSH local port on the host forwarded to the guest's port 22.
SSHLocalPort int `json:"sshLocalPort,omitempty"`
}
4 changes: 4 additions & 0 deletions pkg/hostagent/events/events.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
package events

import (
"net"
"time"
)

Expand All @@ -16,6 +17,9 @@ type Status struct {

Errors []string `json:"errors,omitempty"`

// Guest IP address directly accessible from the host.
GuestIP net.IP `json:"guestIP,omitempty"`
// SSH local port on the host forwarded to the guest's port 22.
SSHLocalPort int `json:"sshLocalPort,omitempty"`

// Cloud-init progress information
Expand Down
137 changes: 134 additions & 3 deletions pkg/hostagent/hostagent.go
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,11 @@ type HostAgent struct {

statusMu sync.RWMutex
currentStatus events.Status

// Guest IP address on the same subnet as the host.
guestIPv4 net.IP
guestIPv6 net.IP
guestIPMu sync.RWMutex
}

type options struct {
Expand Down Expand Up @@ -258,6 +263,27 @@ func New(ctx context.Context, instName string, stdout io.Writer, signalCh chan o
return a, nil
}

func (a *HostAgent) WriteSSHConfigFile(ctx context.Context) error {
sshExe, err := sshutil.NewSSHExe()
if err != nil {
return err
}
sshOpts, err := sshutil.SSHOpts(
ctx,
sshExe,
a.instDir,
*a.instConfig.User.Name,
*a.instConfig.SSH.LoadDotSSHPubKeys,
*a.instConfig.SSH.ForwardAgent,
*a.instConfig.SSH.ForwardX11,
*a.instConfig.SSH.ForwardX11Trusted)
if err != nil {
return err
}
sshAddress, sshPort := a.sshAddressPort()
return writeSSHConfigFile(sshExe.Exe, a.instName, a.instDir, sshAddress, sshPort, sshOpts)
}

func writeSSHConfigFile(sshPath, instName, instDir, instSSHAddress string, sshLocalPort int, sshOpts []string) error {
if instDir == "" {
return fmt.Errorf("directory is unknown for the instance %q", instName)
Expand Down Expand Up @@ -336,6 +362,17 @@ func (a *HostAgent) emitCloudInitProgressEvent(ctx context.Context, progress *ev
a.emitEvent(ctx, ev)
}

func (a *HostAgent) emitGuestIPEvent(ctx context.Context, ip string) {
a.statusMu.RLock()
currentStatus := a.currentStatus
a.statusMu.RUnlock()

currentStatus.GuestIP = net.ParseIP(ip)

ev := events.Event{Status: currentStatus}
a.emitEvent(ctx, ev)
}

func generatePassword(length int) (string, error) {
// avoid any special symbols, to make it easier to copy/paste
return password.Generate(length, length/4, 0, false, false)
Expand Down Expand Up @@ -481,9 +518,31 @@ func (a *HostAgent) startRoutinesAndWait(ctx context.Context, errCh <-chan error
return a.driver.Stop(ctx)
}

// GuestIP returns the guest's IPv4 address if available; otherwise the IPv6 address.
// It returns nil if the guest is not reachable by a direct IP.
func (a *HostAgent) GuestIP() net.IP {
a.guestIPMu.RLock()
defer a.guestIPMu.RUnlock()
if a.guestIPv4 != nil {
return a.guestIPv4
} else if a.guestIPv6 != nil {
return a.guestIPv6
}
return nil
}

// GuestIPs returns the guest's IPv4 and IPv6 addresses if available; otherwise nil.
func (a *HostAgent) GuestIPs() (ipv4, ipv6 net.IP) {
a.guestIPMu.RLock()
defer a.guestIPMu.RUnlock()
return a.guestIPv4, a.guestIPv6
}

func (a *HostAgent) Info(_ context.Context) (*hostagentapi.Info, error) {
guestIP := a.GuestIP()
info := &hostagentapi.Info{
AutoStartedIdentifier: autostart.AutoStartedIdentifier(),
GuestIP: guestIP,
SSHLocalPort: a.sshLocalPort,
}
return info, nil
Expand All @@ -492,6 +551,12 @@ func (a *HostAgent) Info(_ context.Context) (*hostagentapi.Info, error) {
func (a *HostAgent) sshAddressPort() (sshAddress string, sshPort int) {
sshAddress = a.instSSHAddress
sshPort = a.sshLocalPort
guestIP := a.GuestIP()
if guestIP != nil {
sshAddress = guestIP.String()
sshPort = 22
logrus.Debugf("Using the guest IP address %q directly", sshAddress)
}
return sshAddress, sshPort
}

Expand All @@ -513,7 +578,8 @@ func (a *HostAgent) startHostAgentRoutines(ctx context.Context) error {
return nil
}
logrus.Debugf("shutting down the SSH master")
if exitMasterErr := ssh.ExitMaster(a.instSSHAddress, a.sshLocalPort, a.sshConfig); exitMasterErr != nil {
sshAddress, sshPort := a.sshAddressPort()
if exitMasterErr := ssh.ExitMaster(sshAddress, sshPort, a.sshConfig); exitMasterErr != nil {
logrus.WithError(exitMasterErr).Warn("failed to exit SSH master")
}
return nil
Expand All @@ -529,7 +595,8 @@ sudo mkdir -p -m 700 /run/host-services
sudo ln -sf "${SSH_AUTH_SOCK}" /run/host-services/ssh-auth.sock
sudo chown -R "${USER}" /run/host-services`
faDesc := "linking ssh auth socket to static location /run/host-services/ssh-auth.sock"
stdout, stderr, err := ssh.ExecuteScript(a.instSSHAddress, a.sshLocalPort, a.sshConfig, faScript, faDesc)
sshAddress, sshPort := a.sshAddressPort()
stdout, stderr, err := ssh.ExecuteScript(sshAddress, sshPort, a.sshConfig, faScript, faDesc)
logrus.Debugf("stdout=%q, stderr=%q, err=%v", stdout, stderr, err)
if err != nil {
errs = append(errs, fmt.Errorf("stdout=%q, stderr=%q: %w", stdout, stderr, err))
Expand Down Expand Up @@ -836,7 +903,53 @@ func (a *HostAgent) processGuestAgentEvents(ctx context.Context, client *guestag
if useSSHFwd {
a.portForwarder.OnEvent(ctx, ev)
} else {
dialContext := portfwd.DialContextToGRPCTunnel(client)
useDirectIPPortForwarding := false
if envVar := os.Getenv("_LIMA_DIRECT_IP_PORT_FORWARDER"); envVar != "" {
b, err := strconv.ParseBool(envVar)
if err != nil {
logrus.WithError(err).Warnf("invalid _LIMA_DIRECT_IP_PORT_FORWARDER value %q", envVar)
} else {
useDirectIPPortForwarding = b
}
}
var dialContext func(ctx context.Context, network string, guestAddress string) (net.Conn, error)
if useDirectIPPortForwarding {
logrus.Warn("Direct IP Port forwarding is enabled. It may fall back to GRPC Port Forwarding in some cases.")
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This should also support falling back to SSH ?

Copy link
Member

@AkihiroSuda AkihiroSuda Oct 30, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Probably we should rather remove the env var and define the YAML to specify the candidates of the forwarder implementation per protocols (TCP, UDP, ...)

portForwarderTypes:
  tcp: [direct, ssh]
  udp: [none]

Similar to:

Probably we have no time to cover this in v2.0, so let me postpone this PR to v2.1+, sorry

dialContext = func(ctx context.Context, network, guestAddress string) (net.Conn, error) {
guestIPv4, guestIPv6 := a.GuestIPs()
if guestIPv4 == nil && guestIPv6 == nil {
return portfwd.DialContextToGRPCTunnel(client)(ctx, network, guestAddress)
}
// Check if the host part of guestAddress is either unspecified address or matches the known guest IP.
// If so, replace it with the known guest IP to avoid issues with dual-stack setups and DNS resolution.
// Otherwise, fall back to the gRPC tunnel.
if host, _, err := net.SplitHostPort(guestAddress); err != nil {
return nil, err
} else if ip := net.ParseIP(host); ip.IsUnspecified() || ip.Equal(guestIPv4) || ip.Equal(guestIPv6) {
if ip.To4() != nil {
if guestIPv4 != nil {
conn, err := DialContextToGuestIP(guestIPv4)(ctx, network, guestAddress)
if err == nil {
return conn, nil
}
logrus.WithError(err).Warn("failed to connect to the guest IPv4 directly, falling back to gRPC tunnel")
}
} else if ip.To16() != nil {
if guestIPv6 != nil {
conn, err := DialContextToGuestIP(guestIPv6)(ctx, network, guestAddress)
if err == nil {
return conn, nil
}
logrus.WithError(err).Warn("failed to connect to the guest IPv6 directly, falling back to gRPC tunnel")
}
}
// If we reach here, it means we couldn't find a suitable guest IP
}
return portfwd.DialContextToGRPCTunnel(client)(ctx, network, guestAddress)
}
} else {
dialContext = portfwd.DialContextToGRPCTunnel(client)
}
a.grpcPortForwarder.OnEvent(ctx, dialContext, ev)
}
}
Expand All @@ -850,6 +963,24 @@ func (a *HostAgent) processGuestAgentEvents(ctx context.Context, client *guestag
return io.EOF
}

// DialContextToGuestIP returns a DialContext function that connects to the guest IP directly.
// If the guest IP is not known, it returns nil.
func DialContextToGuestIP(guestIP net.IP) func(ctx context.Context, network, address string) (net.Conn, error) {
if guestIP == nil {
return nil
}
return func(ctx context.Context, network, address string) (net.Conn, error) {
var d net.Dialer
_, port, err := net.SplitHostPort(address)
if err != nil {
return nil, err
}
// Host part of address is ignored, because it already has been checked by forwarding rules
// and we want to connect to the guest IP directly.
return d.DialContext(ctx, network, net.JoinHostPort(guestIP.String(), port))
}
}

const (
verbForward = "forward"
verbCancel = "cancel"
Expand Down
Loading
Loading