[Windows] Support run antrea-agent directly from scripts (antrea-io#1013

)

Currently Antrea runs antrea-agent and kube-proxy from management
Pods. But there is still need to run these two components directly
using scripts. This patch:
- Adds scripts help to install and run antrea-agent and kube-proxy.
- Support read kubeconfig from file for antrea-agent apiserver.
  Which is needed when antrea-agent is running as process.

Signed-off-by: Rui Cao <rcao@vmware.com>

cmd/antrea-agent/agent.go

@@ -225,7 +225,8 @@ func run(o *Options) error {
 		agentQuerier,
 		networkPolicyController,
 		o.config.APIPort,
-		o.config.EnablePrometheusMetrics)
+		o.config.EnablePrometheusMetrics,
+		o.config.ClientConnection.Kubeconfig)
 	if err != nil {
 		return fmt.Errorf("error when creating agent API server: %v", err)
 	}

docs/windows.md

@@ -21,6 +21,10 @@ The following components should be configured and run on the Windows Node.
 
 antrea-agent and kube-proxy run as processes on host and are managed by
 management Pods. It is recommended to run OVS daemons as Windows services.
+If you don't want to run antrea-agent and kube-proxy from management Pods
+Antrea also provide scripts which help install and run these two components
+directlty without Pod, please see [Manually run kube-proxy and antrea-agent on Windows worker Nodes](#Manually-run-kube-proxy-and-antrea-agent-on-Windows-worker-Nodes)
+section for details.
 
 ## Deploying Antrea on Windows Worker Node
 
@@ -219,7 +223,24 @@ kubectl get pods -o wide -nkube-system | grep windows
 antrea-agent-windows-6hvkw                             1/1     Running     0          100s
 kube-proxy-windows-2d45w                               1/1     Running     0          102s
 ```
+### Manually run kube-proxy and antrea-agent on Windows worker Nodes
 
+Besides start kube-proxy and antrea-agent from management Pods, Antrea also
+provides powershell scripts which help install and run these two components
+directly without Pod. Please complete the steps in [Installation](#Installation)
+section, skip [Add Windows kube-proxy DaemonSet](#Add-Windows-kube-proxy-DaemonSet)
+and [Add Windows antrea-agent DaemonSet](#Add-Windows-antrea-agent-DaemonSet)
+stpes. And then run following commands in powershell.
+```
+mkdir c:\k\antrea
+cd c:\k\antrea
+curl.exe -LO https://raw.githubusercontent.com/vmware-tanzu/antrea/master/hack/windows/Start.ps1
+./Start.ps1
+``` 
+
+Please skip [Add Windows kube-proxy DaemonSet](#Add-Windows-kube-proxy-DaemonSet)
+and [Add Windows antrea-agent DaemonSet](#Add-Windows-antrea-agent-DaemonSet)
+sections if you choose manually run kube-proxy and antrea-agent.
 
 ## Known issues
 1. HNS Network is not persistent on Windows. So after the Windows Node reboots,

hack/windows/Helper.psm1

@@ -0,0 +1,223 @@
+function Get-WebFileIfNotExist($Path, $URL) {
+    if (Test-Path $Path) {
+        return
+    }
+    Write-Host "Downloading $URL to $PATH"
+    curl.exe -sLo $Path $URL
+}
+
+function New-DirectoryIfNotExist($Path)
+{
+    if (!(Test-Path $Path))
+    {
+        mkdir $Path
+    }
+}
+
+function Test-ConnectionWithRetry($ComputerName, $Port, $MaxRetry, $Interval) {
+    $RetryCountRange = 1..$MaxRetry
+    foreach ($RetryCount in $RetryCountRange) {
+        Write-Host "Testing connection to ($ComputerName,$Port) ($RetryCount/$MaxRetry)..."
+        if (Test-NetConnection -ComputerName $ComputerName -Port $Port | ? { $_.TcpTestSucceeded }) {
+            return $true
+        }
+        if ($RetryCount -eq $MaxRetry) {
+            return $false
+        }
+        Start-Sleep -Seconds $Interval
+    }
+}
+
+function Get-GithubLatestReleaseTag($Owner, $Repo) {
+    return (curl.exe -s "https://api.github.com/repos/$Owner/$Repo/releases" | ConvertFrom-Json)[0].tag_name
+}
+
+function Install-AntreaAgent {
+    Param(
+        [parameter(Mandatory = $false, HelpMessage="Kubernetes version to use")] [string] $KubernetesVersion="v1.18.0",
+        [parameter(Mandatory = $false, HelpMessage="Kubernetes home path")] [string] $KubernetesHome="c:\k",
+        [parameter(Mandatory = $false, HelpMessage="kubeconfig file path")] [string] $KubeConfig="c:\k\config",
+        [parameter(Mandatory = $false, HelpMessage="Antrea version to use")] [string] $AntreaVersion="latest",
+        [parameter(Mandatory = $false, HelpMessage="Antrea home path")] [string] $AntreaHome="c:\k\antrea"
+    )
+    $ErrorActionPreference = "Stop"
+
+    $kubectl = "$KubernetesHome\kubectl.exe"
+    $KubeProxy = "$KubernetesHome\kube-proxy.exe"
+    $yq = "$KubernetesHome\yq.exe"
+
+    $CNIPath = "c:\opt\cni\bin"
+    $CNIConfigPath = "c:\etc\cni\net.d"
+    $AntreaCNIConfigFile = "$CNIConfigPath\10-antrea.conflist"
+    $HostLocalIpam = "$CNIPath\host-local.exe"
+
+    $AntreaEtc = "$AntreaHome\etc"
+    $AntreaAgentConfigPath = "$AntreaEtc\antrea-agent.conf"
+    $AntreaAgent = "$AntreaHome\bin\antrea-agent.exe"
+    $AntreaCNI = "$CNIPath\antrea.exe"
+    $StopScript = "$AntreaHome\Stop.ps1"
+    $Owner = "vmware-tanzu"
+    $Repo = "antrea"
+
+    $env:Path = "$KubernetesHome;" + $env:Path
+
+    if ($AntreaVersion -eq "latest") {
+        $AntreaVersion = (Get-GithubLatestReleaseTag $Owner $Repo)
+    }
+    Write-Host "Antrea version: $AntreaVersion"
+    $AntreaRawUrlBase = "https://raw.githubusercontent.com/$Owner/$Repo/$AntreaVersion"
+    $AntreaReleaseUrlBase = "https://github.com/$Owner/$Repo/releases/download"
+    $AntreaRawUrlBase = "https://raw.githubusercontent.com/$Owner/$Repo/$AntreaVersion"
+
+
+    New-DirectoryIfNotExist $KubernetesHome
+    # Download kubectl
+    Get-WebFileIfNotExist $kubectl  "https://dl.k8s.io/$KubernetesVersion/bin/windows/amd64/kubectl.exe"
+    # Download kube-proxy
+    Get-WebFileIfNotExist $KubeProxy "https://dl.k8s.io/$KubernetesVersion/bin/windows/amd64/kube-proxy.exe"
+    # Download yq
+    Get-WebFileIfNotExist $yq "https://github.com/mikefarah/yq/releases/download/3.3.2/yq_windows_amd64.exe"
+
+    New-DirectoryIfNotExist $AntreaHome
+    New-DirectoryIfNotExist "$AntreaHome\bin"
+    New-DirectoryIfNotExist "$CNIPath"
+    New-DirectoryIfNotExist "$CNIConfigPath"
+    # Download antrea-agent for windows
+    Get-WebFileIfNotExist $AntreaAgent  "$AntreaReleaseUrlBase/$AntreaVersion/antrea-agent-windows-x86_64.exe"
+    Get-WebFileIfNotExist $AntreaCNI  "$AntreaReleaseUrlBase/$AntreaVersion/antrea-cni-windows-x86_64.exe"
+    # Prepare antrea scripts
+    Get-WebFileIfNotExist $StopScript  "$AntreaRawUrlBase/hack/windows/Stop.ps1"
+
+    # Download host-local IPAM plugin
+    if (!(Test-Path $HostLocalIpam)) {
+        curl.exe -sLO https://github.com/containernetworking/plugins/releases/download/v0.8.1/cni-plugins-windows-amd64-v0.8.1.tgz
+        C:\Windows\system32\tar.exe -xzf cni-plugins-windows-amd64-v0.8.1.tgz  -C $CNIPath "./host-local.exe"
+        Remove-Item cni-plugins-windows-amd64-v0.8.1.tgz
+    }
+
+    New-DirectoryIfNotExist $AntreaEtc
+    Get-WebFileIfNotExist $AntreaCNIConfigFile "$AntreaRawUrlBase/build/yamls/windows/base/conf/antrea-cni.conflist"
+    if (!(Test-Path $AntreaAgentConfigPath)) {
+        Get-WebFileIfNotExist $AntreaAgentConfigPath "$AntreaRawUrlBase/build/yamls/windows/base/conf/antrea-agent.conf"
+        yq w -i $AntreaAgentConfigPath featureGates.AntreaProxy true
+        yq w -i $AntreaAgentConfigPath clientConnection.kubeconfig $AntreaEtc\antrea-agent.kubeconfig
+        yq w -i $AntreaAgentConfigPath antreaClientConnection.kubeconfig $AntreaEtc\antrea-agent.antrea.kubeconfig
+        $env:kubeconfig = $KubeConfig
+        # Create the kubeconfig file that contains the K8s APIServer service and the token of antrea ServiceAccount.
+        $APIServer=$(kubectl get service kubernetes -o jsonpath='{.spec.clusterIP}')
+        $APIServerPort=$(kubectl get service kubernetes -o jsonpath='{.spec.ports[0].port}')
+        $APIServer="https://$APIServer" + ":" + $APIServerPort
+        $TOKEN=$(kubectl get secrets -n kube-system -o jsonpath="{.items[?(@.metadata.annotations['kubernetes\.io/service-account\.name']=='antrea-agent')].data.token}")
+        $TOKEN=$([System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($TOKEN)))
+        kubectl config --kubeconfig=$AntreaEtc\antrea-agent.kubeconfig set-cluster kubernetes --server=$APIServer --insecure-skip-tls-verify
+        kubectl config --kubeconfig=$AntreaEtc\antrea-agent.kubeconfig set-credentials antrea-agent --token=$TOKEN
+        kubectl config --kubeconfig=$AntreaEtc\antrea-agent.kubeconfig set-context antrea-agent@kubernetes --cluster=kubernetes --user=antrea-agent
+        kubectl config --kubeconfig=$AntreaEtc\antrea-agent.kubeconfig use-context antrea-agent@kubernetes
+
+        # Create the kubeconfig file that contains the antrea-controller APIServer service and the token of antrea ServiceAccount.
+        $AntreaAPISServer=$(kubectl get service -n kube-system antrea -o jsonpath='{.spec.clusterIP}')
+        $AntreaAPISServerPort=$(kubectl get service -n kube-system antrea -o jsonpath='{.spec.ports[0].port}')
+        $AntreaAPISServer="https://$AntreaAPISServer" + ":" + $AntreaAPISServerPort
+        kubectl config --kubeconfig=$AntreaEtc\antrea-agent.antrea.kubeconfig set-cluster antrea --server=$AntreaAPISServer --insecure-skip-tls-verify
+        kubectl config --kubeconfig=$AntreaEtc\antrea-agent.antrea.kubeconfig set-credentials antrea-agent --token=$TOKEN
+        kubectl config --kubeconfig=$AntreaEtc\antrea-agent.antrea.kubeconfig set-context antrea-agent@antrea --cluster=antrea --user=antrea-agent
+        kubectl config --kubeconfig=$AntreaEtc\antrea-agent.antrea.kubeconfig use-context antrea-agent@antrea
+    }
+}
+
+function New-KubeProxyServiceInterface {
+    Param(
+        [parameter(Mandatory = $false, HelpMessage="Interface to be added service IPs by kube-proxy")] [string] $InterfaceAlias="HNS Internal NIC"
+    )
+    $ErrorActionPreference = "Stop"
+
+    $hnsSwitchName = "KubeProxyInternalSwitch"
+    $INTERFACE_TO_ADD_SERVICE_IP = "vEthernet ($InterfaceAlias)"
+    if (Get-NetAdapter -InterfaceAlias $INTERFACE_TO_ADD_SERVICE_IP -ErrorAction SilentlyContinue) {
+        Write-Host "Network adapter $INTERFACE_TO_ADD_SERVICE_IP exists, exit."
+        return
+    }
+    if (!(Get-VMSwitch -Name $hnsSwitchName -ErrorAction SilentlyContinue)) {
+        Write-Host "Creating internal switch: $hnsSwitchName for kube-proxy"
+        New-VMSwitch -name $hnsSwitchName -SwitchType Internal
+    }
+    Write-Host "Creating network adapter: $INTERFACE_TO_ADD_SERVICE_IP for kube-proxy"
+    [Environment]::SetEnvironmentVariable("INTERFACE_TO_ADD_SERVICE_IP", $INTERFACE_TO_ADD_SERVICE_IP, [System.EnvironmentVariableTarget]::Machine)
+    Add-VMNetworkAdapter -ManagementOS -Name $InterfaceAlias -SwitchName $hnsSwitchName
+    Set-NetIPInterface -ifAlias $INTERFACE_TO_ADD_SERVICE_IP -Forwarding Enabled
+}
+
+function Start-KubeProxy {
+    Param(
+        [parameter(Mandatory = $false, HelpMessage="kubeconfig file path")] [string] $KubeProxy = "c:\k\kube-proxy.exe",
+        [parameter(Mandatory = $false, HelpMessage="kubeconfig file path")] [string] $KubeConfig="c:\k\config",
+        [parameter(Mandatory = $false)] [string] $LogDir = "c:\var\log\kube-proxy"
+    )
+    $ErrorActionPreference = "Stop"
+
+    if (Get-Process -Name kube-proxy -ErrorAction SilentlyContinue) {
+        Write-Host "kube-proxy is already in running"
+        return
+    }
+
+    New-DirectoryIfNotExist $LogDir
+
+    New-KubeProxyServiceInterface
+
+    Start-Process -FilePath $KubeProxy -ArgumentList "--proxy-mode=userspace --kubeconfig=$KubeConfig --log-dir=$LogDir --logtostderr=false --alsologtostderr --v=4"
+}
+
+function Start-AntreaAgent {
+    Param(
+        [parameter(Mandatory = $false, HelpMessage="Antrea home path")] [string] $AntreaHome="c:\k\antrea",
+        [parameter(Mandatory = $false, HelpMessage="kubeconfig file path")] [string] $KubeConfig="c:\k\config",
+        [parameter(Mandatory = $false)] [string] $LogDir
+    )
+    $ErrorActionPreference = "Stop"
+
+    if (Get-Process -Name antrea-agent -ErrorAction SilentlyContinue) {
+        Write-Host "antrea-agent is already in running"
+        return
+    }
+
+    # Try to cleanup ovsdb-server configurations if the antrea-hnsnetwork is not existing.
+    if (!(Get-VMswitch -Name "antrea-hnsnetwork" -SwitchType External -ErrorAction SilentlyContinue)) {
+        while ($true) {
+            & ovs-vsctl.exe --no-wait --if-exists del-br br-int
+            if (-Not $LASTEXITCODE) {
+                break
+            } else {
+                Write-Host "Waiting for ovsdb-server running"
+                Start-Sleep -Seconds 1
+            }
+        }
+    }
+    while ($true) {
+        $service = Get-Service -Name ovs-vswitchd
+        if ($service.Status -eq "Running") {
+            break
+        }
+        if ($service.Status -eq "Stopped") {
+            Start-Service ovs-vswitchd
+        }
+        Write-Host "Waiting for ovs-vswitchd running"
+        Start-Sleep -Seconds 2
+    }
+
+    $AntreaAgent = "$AntreaHome\bin\antrea-agent.exe"
+    $AntreaAgentConfigPath = "$AntreaHome\etc\antrea-agent.conf"
+    if ($LogDir -eq "") {
+        $LogDir = "$AntreaHome\logs"
+    }
+    New-DirectoryIfNotExist $LogDir
+    [Environment]::SetEnvironmentVariable("NODE_NAME", (hostname).ToLower())
+    Start-Process -FilePath $AntreaAgent  -ArgumentList "--config=$AntreaAgentConfigPath --logtostderr=false --log_dir=$LogDir --alsologtostderr --log_file_max_size=100 --log_file_max_num=4"
+}
+
+Export-ModuleMember Get-WebFileIfNotExist
+Export-ModuleMember New-DirectoryIfNotExist
+Export-ModuleMember Test-ConnectionWithRetry
+Export-ModuleMember Install-AntreaAgent
+Export-ModuleMember New-KubeProxyServiceInterface
+Export-ModuleMember Start-KubeProxy
+Export-ModuleMember Start-AntreaAgent

hack/windows/Start.ps1

@@ -0,0 +1,63 @@
+Param(
+    [parameter(Mandatory = $false, HelpMessage="Kubernetes version to use")] [string] $KubernetesVersion="v1.18.0",
+    [parameter(Mandatory = $false, HelpMessage="Kubernetes home path")] [string] $KubernetesHome="c:\k",
+    [parameter(Mandatory = $false, HelpMessage="kubeconfig file path")] [string] $KubeConfig="c:\k\config",
+    [parameter(Mandatory = $false, HelpMessage="Antrea version to use")] [string] $AntreaVersion="latest",
+    [parameter(Mandatory = $false, HelpMessage="Antrea home path")] [string] $AntreaHome="c:\k\antrea"
+)
+$ErrorActionPreference = "Stop"
+
+$Owner = "vmware-tanzu"
+$Repo = "antrea"
+if ($AntreaVersion -eq "latest") {
+    $AntreaReleases = (curl.exe -s "https://api.github.com/repos/$Owner/$Repo/releases" | ConvertFrom-Json)
+    $AntreaVersion = $AntreaReleases[0].tag_name
+}
+
+if ($AntreaVersion -eq "") {
+    $ErrorMsg = ($AntreaReleases).message
+    Write-Host "Failed to get Antrea latest version: $ErrorMsg, exit"
+    exit 1
+}
+
+Write-Host "KubernetesVersion version: $KubernetesVersion"
+Write-Host "Antrea version: $AntreaVersion"
+$AntreaRawUrlBase = "https://raw.githubusercontent.com/$Owner/$Repo/$AntreaVersion"
+
+if (!(Test-Path $AntreaHome)) {
+    mkdir $AntreaHome
+}
+
+$helper = "$AntreaHome\Helper.psm1"
+if (!(Test-Path $helper))
+{
+    curl.exe -sLo $helper "$AntreaRawUrlBase/hack/windows/Helper.psm1"
+}
+Import-Module $helper
+
+Write-Host "Checking kube-proxy and antrea-agent installation..."
+Install-AntreaAgent -KubernetesVersion $KubernetesVersion -KubernetesHome $KubernetesHome -KubeConfig $KubeConfig -AntreaVersion $AntreaVersion -AntreaHome $AntreaHome
+
+if ($LastExitCode) {
+    Write-Host "Install antrea-agent failed, exit"
+    exit 1
+}
+
+Write-Host "Starting kube-proxy..."
+Start-KubeProxy -KubeProxy $KubernetesHome\kube-proxy.exe -KubeConfig $KubeConfig
+
+$env:kubeconfig = $KubeConfig
+$APIServer=$(kubectl get service kubernetes -o jsonpath='{.spec.clusterIP}')
+$APIServerPort=$(kubectl get service kubernetes -o jsonpath='{.spec.ports[0].port}')
+$APIServer="https://$APIServer" + ":" + $APIServerPort
+$APIServer=[System.Uri]$APIServer
+
+Write-Host "Test connection to kubernetes API server"
+$result = Test-ConnectionWithRetry $APIServer.Host $APIServer.Port 20 3
+if (!$result) {
+    Write-Host "Failed to connection to kubernetes API server service, exit"
+    exit 1
+}
+
+Write-Host "Starting antrea-agent..."
+Start-AntreaAgent -AntreaHome $AntreaHome -KubeConfig $KubeConfig

hack/windows/Stop.ps1

@@ -0,0 +1,2 @@
+taskkill /im antrea-agent.exe /f
+taskkill /im kube-proxy.exe /f

pkg/agent/apiserver/apiserver.go

@@ -89,8 +89,8 @@ func installAPIGroup(s *genericapiserver.GenericAPIServer, aq agentquerier.Agent
 
 // New creates an APIServer for running in antrea agent.
 func New(aq agentquerier.AgentQuerier, npq querier.AgentNetworkPolicyInfoQuerier, bindPort int,
-	enableMetrics bool) (*agentAPIServer, error) {
-	cfg, err := newConfig(bindPort, enableMetrics)
+	enableMetrics bool, kubeconfig string) (*agentAPIServer, error) {
+	cfg, err := newConfig(bindPort, enableMetrics, kubeconfig)
 	if err != nil {
 		return nil, err
 	}
@@ -105,11 +105,17 @@ func New(aq agentquerier.AgentQuerier, npq querier.AgentNetworkPolicyInfoQuerier
 	return &agentAPIServer{GenericAPIServer: s}, nil
 }
 
-func newConfig(bindPort int, enableMetrics bool) (*genericapiserver.CompletedConfig, error) {
+func newConfig(bindPort int, enableMetrics bool, kubeconfig string) (*genericapiserver.CompletedConfig, error) {
 	secureServing := genericoptions.NewSecureServingOptions().WithLoopback()
 	authentication := genericoptions.NewDelegatingAuthenticationOptions()
 	authorization := genericoptions.NewDelegatingAuthorizationOptions().WithAlwaysAllowPaths("/healthz")
 
+	// kubeconfig file is useful when antrea-agent isn't not running as a pod
+	if len(kubeconfig) > 0 {
+		authentication.RemoteKubeConfigFile = kubeconfig
+		authorization.RemoteKubeConfigFile = kubeconfig
+	}
+
 	// Set the PairName but leave certificate directory blank to generate in-memory by default.
 	secureServing.ServerCert.CertDirectory = ""
 	secureServing.ServerCert.PairName = Name