Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Nginx Ingress Controller on sky local up #3223

Merged
merged 24 commits into from
Apr 2, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 0 additions & 10 deletions sky/clouds/kubernetes.py
Original file line number Diff line number Diff line change
Expand Up @@ -77,16 +77,6 @@ def _unsupported_features_for_resources(
cls, resources: 'resources_lib.Resources'
) -> Dict[clouds.CloudImplementationFeatures, str]:
unsupported_features = cls._CLOUD_UNSUPPORTED_FEATURES
curr_context = kubernetes_utils.get_current_kube_config_context_name()
if curr_context == kubernetes_utils.KIND_CONTEXT_NAME:
# If we are using KIND, the loadbalancer service will never be
# assigned an external IP. Users may use ingress, but that requires
# blocking HTTP port 80.
# For now, we disable port opening feature on kind clusters.
unsupported_features[
clouds.CloudImplementationFeatures.OPEN_PORTS] = (
'Opening ports is not supported in Kubernetes when '
'using local kind cluster.')
return unsupported_features

@classmethod
Expand Down
9 changes: 9 additions & 0 deletions sky/provision/kubernetes/network_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
from sky import exceptions
from sky import skypilot_config
from sky.adaptors import kubernetes
from sky.provision.kubernetes import utils as kubernetes_utils
from sky.utils import kubernetes_enums
from sky.utils import ux_utils

Expand All @@ -19,6 +20,14 @@
def get_port_mode(
mode_str: Optional[str] = None) -> kubernetes_enums.KubernetesPortMode:
"""Get the port mode from the provider config."""

curr_kube_config = kubernetes_utils.get_current_kube_config_context_name()
running_kind = curr_kube_config == kubernetes_utils.KIND_CONTEXT_NAME

if running_kind:
# If running in kind (`sky local up`), use ingress mode
return kubernetes_enums.KubernetesPortMode.INGRESS

mode_str = mode_str or skypilot_config.get_nested(
('kubernetes', 'ports'),
kubernetes_enums.KubernetesPortMode.LOADBALANCER.value)
Expand Down
28 changes: 28 additions & 0 deletions sky/utils/kubernetes/create_cluster.sh
Original file line number Diff line number Diff line change
Expand Up @@ -166,6 +166,31 @@ wait_for_skypilot_cpu_image_pull() {
echo "SkyPilot CPU image loaded into kind cluster."
}

wait_for_nginx_ingress_controller_install() {
echo "Starting installation of Nginx Ingress Controller..."

SECONDS=0
TIMEOUT=600 # 10 minutes in seconds

kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/main/deploy/static/provider/kind/deploy.yaml

while true; do
if kubectl get pod -n ingress-nginx -l app.kubernetes.io/component=controller -o wide | grep 'Running'; then
echo "Nginx Ingress Controller installed."
break
elif [ $SECONDS -ge $TIMEOUT ]; then
echo "Timed out waiting for installation of Nginx Ingress Controller."
exit 1
else
echo "Waiting for Nginx Ingress Controller Installation..."
echo "To check status, check Nginx Ingress Controller pods:"
echo "kubectl get pod -n ingress-nginx -l app.kubernetes.io/component=controller -o wide"
sleep 5
fi
done

}

if $ENABLE_GPUS; then
echo "Enabling GPU support..."
# Run patch for missing ldconfig.real
Expand Down Expand Up @@ -196,6 +221,9 @@ fi
# Load local skypilot image on to the cluster for faster startup
wait_for_skypilot_cpu_image_pull

# Install the Nginx Ingress Controller
wait_for_nginx_ingress_controller_install

# Print CPUs available on the local cluster
NUM_CPUS=$(kubectl get nodes -o jsonpath='{.items[0].status.capacity.cpu}')
echo "Kubernetes cluster ready! Run `sky check` to setup Kubernetes access."
Expand Down
9 changes: 8 additions & 1 deletion sky/utils/kubernetes/generate_kind_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,14 @@ def generate_kind_config(path: str,
extraArgs:
"service-node-port-range": {port_start}-{port_end}
nodes:
- role: control-plane""")
- role: control-plane
kubeadmConfigPatches:
- |
kind: InitConfiguration
nodeRegistration:
kubeletExtraArgs:
node-labels: "ingress-ready=true"
""")
if gpus:
preamble += textwrap.indent(
textwrap.dedent("""
Expand Down
7 changes: 7 additions & 0 deletions sky/utils/log_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,13 @@ def process_line(self, log_line):
if 'SkyPilot CPU image loaded into kind cluster' in log_line:
logger.info(f'{colorama.Fore.GREEN}SkyPilot CPU image pulled.'
f'{colorama.Style.RESET_ALL}')
if 'Starting installation of Nginx Ingress Controller...' in log_line:
self.status_display.update(
'[bold cyan]Creating Nginx Ingress Controller')
if 'Nginx Ingress Controller installed' in log_line:
logger.info(
f'{colorama.Fore.GREEN}Nginx Ingress Controller installed.'
f'{colorama.Style.RESET_ALL}')

def __exit__(self, except_type, except_value, traceback):
del except_type, except_value, traceback # unused
Expand Down
Loading