From d64288032d4b8c31af9112808d89fe8ddff207f4 Mon Sep 17 00:00:00 2001 From: Rui Cao Date: Thu, 30 Jul 2020 16:09:44 +0800 Subject: [PATCH] [Windows] Support run antrea-agent directly from scripts (#1013) Currently Antrea runs antrea-agent and kube-proxy from management Pods. But there is still need to run these two components directly using scripts. This patch: - Adds scripts help to install and run antrea-agent and kube-proxy. - Support read kubeconfig from file for antrea-agent apiserver. Which is needed when antrea-agent is running as process. Signed-off-by: Rui Cao --- cmd/antrea-agent/agent.go | 3 +- docs/windows.md | 21 ++++++ hack/windows/Helper.psm1 | 37 +++++++++++ hack/windows/Install-AntreaAgent.ps1 | 99 ++++++++++++++++++++++++++++ hack/windows/Start-AntreaAgent.ps1 | 22 +++++++ hack/windows/Start-KubeProxy.ps1 | 25 +++++++ hack/windows/Start.ps1 | 59 +++++++++++++++++ hack/windows/Stop.ps1 | 2 + pkg/agent/apiserver/apiserver.go | 12 +++- 9 files changed, 276 insertions(+), 4 deletions(-) create mode 100644 hack/windows/Helper.psm1 create mode 100644 hack/windows/Install-AntreaAgent.ps1 create mode 100644 hack/windows/Start-AntreaAgent.ps1 create mode 100644 hack/windows/Start-KubeProxy.ps1 create mode 100644 hack/windows/Start.ps1 create mode 100644 hack/windows/Stop.ps1 diff --git a/cmd/antrea-agent/agent.go b/cmd/antrea-agent/agent.go index 2dd9386f02e..c48f0252354 100644 --- a/cmd/antrea-agent/agent.go +++ b/cmd/antrea-agent/agent.go @@ -225,7 +225,8 @@ func run(o *Options) error { agentQuerier, networkPolicyController, o.config.APIPort, - o.config.EnablePrometheusMetrics) + o.config.EnablePrometheusMetrics, + o.config.ClientConnection.Kubeconfig) if err != nil { return fmt.Errorf("error when creating agent API server: %v", err) } diff --git a/docs/windows.md b/docs/windows.md index d0da7596192..3c03d5b572b 100644 --- a/docs/windows.md +++ b/docs/windows.md @@ -21,6 +21,10 @@ The following components should be configured and run on the Windows Node. antrea-agent and kube-proxy run as processes on host and are managed by management Pods. It is recommended to run OVS daemons as Windows services. +If you don't want to run antrea-agent and kube-proxy from management Pods +Antrea also provide scripts which help install and run these two components +directlty without Pod, please see [Manually run kube-proxy and antrea-agent on Windows worker Nodes](#Manually-run-kube-proxy-and-antrea-agent-on-Windows-worker-Nodes) +section for details. ## Deploying Antrea on Windows Worker Node @@ -219,7 +223,24 @@ kubectl get pods -o wide -nkube-system | grep windows antrea-agent-windows-6hvkw 1/1 Running 0 100s kube-proxy-windows-2d45w 1/1 Running 0 102s ``` +### Manually run kube-proxy and antrea-agent on Windows worker Nodes +Besides start kube-proxy and antrea-agent from management Pods, Antrea also +provides powershell scripts which help install and run these two components +directly without Pod. Please complete the steps in [Installation](#Installation) +section, skip [Add Windows kube-proxy DaemonSet](#Add-Windows-kube-proxy-DaemonSet) +and [Add Windows antrea-agent DaemonSet](#Add-Windows-antrea-agent-DaemonSet) +stpes. And then run following commands in powershell. +``` +mkdir c:\k\antrea +cd c:\k\antrea +curl.exe -LO https://raw.githubusercontent.com/vmware-tanzu/antrea/master/hack/windows/Start.ps1 +./Start.ps1 +``` + +Please skip [Add Windows kube-proxy DaemonSet](#Add-Windows-kube-proxy-DaemonSet) +and [Add Windows antrea-agent DaemonSet](#Add-Windows-antrea-agent-DaemonSet) +sections if you choose manually run kube-proxy and antrea-agent. ## Known issues 1. HNS Network is not persistent on Windows. So after the Windows Node reboots, diff --git a/hack/windows/Helper.psm1 b/hack/windows/Helper.psm1 new file mode 100644 index 00000000000..461ad9af1a2 --- /dev/null +++ b/hack/windows/Helper.psm1 @@ -0,0 +1,37 @@ +function DownloadFileIfNotExist($Path, $URL) { + if (Test-Path $Path) { + return + } + Write-Host "Downloading $URL to $PATH" + curl.exe -sLo $Path $URL +} + +function CreateDirectory($Path) +{ + if (!(Test-Path $Path)) + { + mkdir $Path + } +} + +Function TestConnection($ComputerName, $Port, $MaxRetry, $Interval) { + $RetryCountRange = 1..$MaxRetry + foreach ($RetryCount in $RetryCountRange) { + Write-Host "Testing connection to ($ComputerName,$Port) ($RetryCount/$MaxRetry)..." + if (Test-NetConnection -ComputerName $ComputerName -Port $Port | ? { $_.TcpTestSucceeded }) { + return $true + } + if ($RetryCount -eq $MaxRetry) { + return $false + } + Start-Sleep -Seconds $Interval + } +} + +Function GetGithubLatestReleaseTag($Owner, $Repo) { + return (curl.exe -s "https://api.github.com/repos/$Owner/$Repo/releases" | ConvertFrom-Json)[0].tag_name +} + +Export-ModuleMember DownloadFileIfNotExist +Export-ModuleMember CreateDirectory +Export-ModuleMember TestConnection diff --git a/hack/windows/Install-AntreaAgent.ps1 b/hack/windows/Install-AntreaAgent.ps1 new file mode 100644 index 00000000000..2efb95700c9 --- /dev/null +++ b/hack/windows/Install-AntreaAgent.ps1 @@ -0,0 +1,99 @@ +Param( + [parameter(Mandatory = $false, HelpMessage="Kubernetes version to use")] [string] $KubernetesVersion="v1.18.0", + [parameter(Mandatory = $false, HelpMessage="Kubernetes home path")] [string] $KubernetesHome="c:\k", + [parameter(Mandatory = $false, HelpMessage="kubeconfig file path")] [string] $KubeConfig="c:\k\config", + [parameter(Mandatory = $false, HelpMessage="Antrea version to use")] [string] $AntreaVersion="latest", + [parameter(Mandatory = $false, HelpMessage="Antrea home path")] [string] $AntreaHome="c:\k\antrea" +) +$ErrorActionPreference = "Stop" + + +$kubectl = "$KubernetesHome\kubectl.exe" +$KubeProxy = "$KubernetesHome\kube-proxy.exe" +$yq = "$KubernetesHome\yq.exe" + +$CNIPath = "c:\opt\cni\bin" +$CNIConfigPath = "c:\etc\cni\net.d" +$AntreaCNIConfigFile = "$CNIPath\10-antrea.conflist" +$HostLocalIpam = "$CNIPath\host-local.exe" + +$AntreaEtc = "$AntreaHome\etc" +$AntreaAgentConfigPath = "$AntreaEtc\antrea-agent.conf" +$AntreaAgent = "$AntreaHome\bin\antrea-agent.exe" +$AntreaCNI = "$CNIPath\antrea.exe" +$PrepareServiceInterface = "$AntreaHome\Prepare-ServiceInterface.ps1" +$StartKubeProxy = "$AntreaHome\Start-KubeProxy.ps1" +$StartAntreaAgent = "$AntreaHome\Start-AntreaAgent.ps1" +$StopScript = "$AntreaHome\Stop.ps1" +$Owner = "vmware-tanzu" +$Repo = "antrea" + +$env:Path = "$KubernetesHome;" + $env:Path +$helper = "$AntreaHome\Helper.psm1" +Import-Module $helper + +if ($AntreaVersion -eq "latest") { + $AntreaVersion = (GetGithubLatestReleaseTag $Owner $Repo) +} +Write-Host "Antrea version: $AntreaVersion" +$AntreaRawUrlBase = "https://raw.githubusercontent.com/$Owner/$Repo/$AntreaVersion" +$AntreaReleaseUrlBase = "https://github.com/$Owner/$Repo/releases/download" +$AntreaRawUrlBase = "https://raw.githubusercontent.com/$Owner/$Repo/$AntreaVersion" + + +CreateDirectory $KubernetesHome +# Download kubectl +DownloadFileIfNotExist $kubectl "https://dl.k8s.io/$KubernetesVersion/bin/windows/amd64/kubectl.exe" +# Download kube-proxy +DownloadFileIfNotExist $KubeProxy "https://dl.k8s.io/$KubernetesVersion/bin/windows/amd64/kube-proxy.exe" +# Download yq +DownloadFileIfNotExist $yq "https://github.com/mikefarah/yq/releases/download/3.3.2/yq_windows_amd64.exe" + +CreateDirectory $AntreaHome +CreateDirectory "$AntreaHome\bin" +CreateDirectory "$CNIPath" +CreateDirectory "$CNIConfigPath" +# Download antrea-agent for windows +DownloadFileIfNotExist $AntreaAgent "$AntreaReleaseUrlBase/$AntreaVersion/antrea-agent-windows-x86_64.exe" +DownloadFileIfNotExist $AntreaCNI "$AntreaReleaseUrlBase/$AntreaVersion/antrea-cni-windows-x86_64.exe" +# Prepare antrea scripts +DownloadFileIfNotExist $PrepareServiceInterface "$AntreaRawUrlBase/hack/windows/Prepare-ServiceInterface.ps1" +DownloadFileIfNotExist $StartKubeProxy "$AntreaRawUrlBase/hack/windows/Start-KubeProxy.ps1" +DownloadFileIfNotExist $StartAntreaAgent "$AntreaRawUrlBase/hack/windows/Start-AntreaAgent.ps1" +DownloadFileIfNotExist $StopScript "$AntreaRawUrlBase/hack/windows/Stop.ps1" + +# Download host-local IPAM plugin +if (!(Test-Path $HostLocalIpam)) { + curl.exe -sLO https://github.com/containernetworking/plugins/releases/download/v0.8.1/cni-plugins-windows-amd64-v0.8.1.tgz + C:\Windows\system32\tar.exe -xzf cni-plugins-windows-amd64-v0.8.1.tgz -C $CNIPath "./host-local.exe" + Remove-Item cni-plugins-windows-amd64-v0.8.1.tgz +} + +CreateDirectory $AntreaEtc +DownloadFileIfNotExist $AntreaCNIConfigFile "$AntreaRawUrlBase/build/yamls/windows/base/conf/antrea-cni.conflist" +if (!(Test-Path $AntreaAgentConfigPath)) { + DownloadFileIfNotExist $AntreaAgentConfigPath "$AntreaRawUrlBase/build/yamls/windows/base/conf/antrea-agent.conf" + yq w -i $AntreaAgentConfigPath featureGates.AntreaProxy true + yq w -i $AntreaAgentConfigPath clientConnection.kubeconfig $AntreaEtc\antrea-agent.kubeconfig + yq w -i $AntreaAgentConfigPath antreaClientConnection.kubeconfig $AntreaEtc\antrea-agent.antrea.kubeconfig + $env:kubeconfig = $KubeConfig + # Create the kubeconfig file that contains the K8s APIServer service and the token of antrea ServiceAccount. + $APIServer=$(kubectl get service kubernetes -o jsonpath='{.spec.clusterIP}') + $APIServerPort=$(kubectl get service kubernetes -o jsonpath='{.spec.ports[0].port}') + $APIServer="https://$APIServer" + ":" + $APIServerPort + $TOKEN=$(kubectl get secrets -n kube-system -o jsonpath="{.items[?(@.metadata.annotations['kubernetes\.io/service-account\.name']=='antrea-agent')].data.token}") + $TOKEN=$([System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($TOKEN))) + kubectl config --kubeconfig=$AntreaEtc\antrea-agent.kubeconfig set-cluster kubernetes --server=$APIServer --insecure-skip-tls-verify + kubectl config --kubeconfig=$AntreaEtc\antrea-agent.kubeconfig set-credentials antrea-agent --token=$TOKEN + kubectl config --kubeconfig=$AntreaEtc\antrea-agent.kubeconfig set-context antrea-agent@kubernetes --cluster=kubernetes --user=antrea-agent + kubectl config --kubeconfig=$AntreaEtc\antrea-agent.kubeconfig use-context antrea-agent@kubernetes + + # Create the kubeconfig file that contains the antrea-controller APIServer service and the token of antrea ServiceAccount. + $AntreaAPISServer=$(kubectl get service -n kube-system antrea -o jsonpath='{.spec.clusterIP}') + $AntreaAPISServerPort=$(kubectl get service -n kube-system antrea -o jsonpath='{.spec.ports[0].port}') + $AntreaAPISServer="https://$AntreaAPISServer" + ":" + $AntreaAPISServerPort + kubectl config --kubeconfig=$AntreaEtc\antrea-agent.antrea.kubeconfig set-cluster antrea --server=$AntreaAPISServer --insecure-skip-tls-verify + kubectl config --kubeconfig=$AntreaEtc\antrea-agent.antrea.kubeconfig set-credentials antrea-agent --token=$TOKEN + kubectl config --kubeconfig=$AntreaEtc\antrea-agent.antrea.kubeconfig set-context antrea-agent@antrea --cluster=antrea --user=antrea-agent + kubectl config --kubeconfig=$AntreaEtc\antrea-agent.antrea.kubeconfig use-context antrea-agent@antrea +} diff --git a/hack/windows/Start-AntreaAgent.ps1 b/hack/windows/Start-AntreaAgent.ps1 new file mode 100644 index 00000000000..159365e6c7e --- /dev/null +++ b/hack/windows/Start-AntreaAgent.ps1 @@ -0,0 +1,22 @@ +Param( + [parameter(Mandatory = $false, HelpMessage="Antrea home path")] [string] $AntreaHome="c:\k\antrea", + [parameter(Mandatory = $false, HelpMessage="kubeconfig file path")] [string] $KubeConfig="c:\k\config", + [parameter(Mandatory = $false)] [string] $LogDir +) +$ErrorActionPreference = "Stop" + +if (Get-Process -Name antrea-agent -ErrorAction SilentlyContinue) { + Write-Host "antrea-agent is already in running, exit" + exit 0 +} + +Import-Module c:\k\antrea\Helper.psm1 +$AntreaAgent = "$AntreaHome\bin\antrea-agent.exe" +$AntreaAgentConfigPath = "$AntreaHome\etc\antrea-agent.conf" +if ($LogDir -eq "") { + $LogDir = "$AntreaHome\logs" +} + +CreateDirectory $LogDir +[Environment]::SetEnvironmentVariable("NODE_NAME", (hostname).ToLower()) +& $AntreaAgent --config=$AntreaAgentConfigPath --logtostderr=false --log_dir=$LogDir --alsologtostderr --log_file_max_size=100 --log_file_max_num=4 diff --git a/hack/windows/Start-KubeProxy.ps1 b/hack/windows/Start-KubeProxy.ps1 new file mode 100644 index 00000000000..5b52adb1295 --- /dev/null +++ b/hack/windows/Start-KubeProxy.ps1 @@ -0,0 +1,25 @@ +Param( + [parameter(Mandatory = $false, HelpMessage="kubeconfig file path")] [string] $KubeProxy = "c:\k\kube-proxy.exe", + [parameter(Mandatory = $false, HelpMessage="kubeconfig file path")] [string] $KubeConfig="c:\k\config", + [parameter(Mandatory = $false)] [string] $LogDir = "c:\var\log\kube-proxy" +) +$ErrorActionPreference = "Stop" + +$PrepareServiceInterface = "c:\k\antrea\Prepare-ServiceInterface.ps1" +Import-Module c:\k\antrea\Helper.psm1 + +if (Get-Process -Name kube-proxy -ErrorAction SilentlyContinue) { + Write-Host "kube-proxy is already in running, exit" + exit 0 +} + +CreateDirectory $LogDir + +powershell $PrepareServiceInterface + +if ($LastExitCode) { + Write-Host "Prepare kube-proxy service interface failed, exit" + exit 1 +} + +& $KubeProxy --proxy-mode=userspace --kubeconfig=$KubeConfig --log-dir=$LogDir --logtostderr=false --alsologtostderr --v=4 diff --git a/hack/windows/Start.ps1 b/hack/windows/Start.ps1 new file mode 100644 index 00000000000..301c78c249b --- /dev/null +++ b/hack/windows/Start.ps1 @@ -0,0 +1,59 @@ +Param( + [parameter(Mandatory = $false, HelpMessage="Kubernetes version to use")] [string] $KubernetesVersion="v1.18.0", + [parameter(Mandatory = $false, HelpMessage="Kubernetes home path")] [string] $KubernetesHome="c:\k", + [parameter(Mandatory = $false, HelpMessage="kubeconfig file path")] [string] $KubeConfig="c:\k\config", + [parameter(Mandatory = $false, HelpMessage="Antrea version to use")] [string] $AntreaVersion="latest", + [parameter(Mandatory = $false, HelpMessage="Antrea home path")] [string] $AntreaHome="c:\k\antrea" +) +$ErrorActionPreference = "Stop" + +$InstallAntreaAgent = "$AntreaHome\Install-AntreaAgent.ps1" +$Owner = "vmware-tanzu" +$Repo = "antrea" +if ($AntreaVersion -eq "latest") { + $AntreaVersion = (curl.exe -s "https://api.github.com/repos/$Owner/$Repo/releases" | ConvertFrom-Json)[0].tag_name +} +Write-Host "KubernetesVersion version: $KubernetesVersion" +Write-Host "Antrea version: $AntreaVersion" +$AntreaRawUrlBase = "https://raw.githubusercontent.com/$Owner/$Repo/$AntreaVersion" + +if (!(Test-Path $AntreaHome)) { + mkdir $AntreaHome +} + +$helper = "$AntreaHome\Helper.psm1" +if (!(Test-Path $helper)) +{ + curl.exe -sLo $helper "$AntreaRawUrlBase/hack/windows/Helper.psm1" +} +Import-Module $helper + +# Download Install-AntreaAgent.ps1 +DownloadFileIfNotExist $InstallAntreaAgent "$AntreaRawUrlBase/hack/windows/Install-AntreaAgent.ps1" + +Write-Host "Checking kube-proxy and antrea-agent installation..." +powershell $InstallAntreaAgent -KubernetesVersion $KubernetesVersion -KubernetesHome $KubernetesHome -KubeConfig $KubeConfig -AntreaVersion $AntreaVersion -AntreaHome $AntreaHome + +if ($LastExitCode) { + Write-Host "Install antrea-agent failed, exit" + exit 1 +} + +Write-Host "Starting kube-proxy..." +Start-Process powershell -ArgumentList "-File $AntreaHome\Start-KubeProxy.ps1 -KubeProxy $KubernetesHome\kube-proxy.exe -KubeConfig $KubeConfig" + +$env:kubeconfig = $KubeConfig +$APIServer=$(kubectl get service kubernetes -o jsonpath='{.spec.clusterIP}') +$APIServerPort=$(kubectl get service kubernetes -o jsonpath='{.spec.ports[0].port}') +$APIServer="https://$APIServer" + ":" + $APIServerPort +$APIServer=[System.Uri]$APIServer + +Write-Host "Test connection to kubernetes API server" +$result = TestConnection $APIServer.Host $APIServer.Port 20 3 +if (!$result) { + Write-Host "Failed to connection to kubernetes API server service, exit" + exit 1 +} + +Write-Host "Starting antrea-agent..." +Start-Process powershell -ArgumentList "-File $AntreaHome\Start-AntreaAgent.ps1 -AntreaHome $AntreaHome -KubeConfig $KubeConfig" diff --git a/hack/windows/Stop.ps1 b/hack/windows/Stop.ps1 new file mode 100644 index 00000000000..2ec6b634bf9 --- /dev/null +++ b/hack/windows/Stop.ps1 @@ -0,0 +1,2 @@ +taskkill /im antrea-agent.exe /f +taskkill /im kube-proxy.exe /f diff --git a/pkg/agent/apiserver/apiserver.go b/pkg/agent/apiserver/apiserver.go index 53157ddb3cc..e4d763a6aba 100644 --- a/pkg/agent/apiserver/apiserver.go +++ b/pkg/agent/apiserver/apiserver.go @@ -89,8 +89,8 @@ func installAPIGroup(s *genericapiserver.GenericAPIServer, aq agentquerier.Agent // New creates an APIServer for running in antrea agent. func New(aq agentquerier.AgentQuerier, npq querier.AgentNetworkPolicyInfoQuerier, bindPort int, - enableMetrics bool) (*agentAPIServer, error) { - cfg, err := newConfig(bindPort, enableMetrics) + enableMetrics bool, kubeconfig string) (*agentAPIServer, error) { + cfg, err := newConfig(bindPort, enableMetrics, kubeconfig) if err != nil { return nil, err } @@ -105,11 +105,17 @@ func New(aq agentquerier.AgentQuerier, npq querier.AgentNetworkPolicyInfoQuerier return &agentAPIServer{GenericAPIServer: s}, nil } -func newConfig(bindPort int, enableMetrics bool) (*genericapiserver.CompletedConfig, error) { +func newConfig(bindPort int, enableMetrics bool, kubeconfig string) (*genericapiserver.CompletedConfig, error) { secureServing := genericoptions.NewSecureServingOptions().WithLoopback() authentication := genericoptions.NewDelegatingAuthenticationOptions() authorization := genericoptions.NewDelegatingAuthorizationOptions().WithAlwaysAllowPaths("/healthz") + // kubeconfig file is useful when antrea-agent isn't not running as a pod + if len(kubeconfig) > 0 { + authentication.RemoteKubeConfigFile = kubeconfig + authorization.RemoteKubeConfigFile = kubeconfig + } + // Set the PairName but leave certificate directory blank to generate in-memory by default. secureServing.ServerCert.CertDirectory = "" secureServing.ServerCert.PairName = Name