fix: tls remove failed (#106)

* fix: tls remove failed & Add a node placeholder when the num of nodes in upstream is 0

* protect: will retry when any upstream nodes is empty

* add FAQ

* Explanation of failure reason in FAQ

docs/FAQ.md

@@ -38,3 +38,15 @@ This is because CRDs are generally declared in the file system, and Apply to ent
 So far apisix-ingress-controller doesn't support set admin_key for Apache APISIX, so when you deploy your APISIX cluster, admin_key should be removed from config.
 
 Note since APISIX have two configuration files, the first is config.yaml, which contains the user specified configs, the other is config-default.yaml, which has all default items, config items in these two files will be merged. So admin_key in both files should be removed. You can customize these two configuration files and mount them to APISIX deloyment.
+
+5. Failed to create route with `ApisixRoute`?
+
+When `apisix-ingress-controller` creates a route with CRD, it checks the `Endpoint` resources in Kubernetes (matched by namespace_name_port). If the corresponding endpoint information is not found, the route will not be created and wait for the next retry.
+
+Tips: The failure caused by empty upstream nodes is a limitation of Apache APISIX, related [issue](https://github.com/apache/apisix/issues/3072)
+
+6. What is the retry rule of `apisix-ingress-controller`?
+
+If an error occurs during the process of `apisix-ingress-controller` parsing CRD and distributing the configuration to APISIX, a retry will be triggered.
+
+The delayed retry method is adopted. After the first failure, it is retried once per second. After 5 retries are triggered, the slow retry strategy will be enabled, and the retry will be performed every 1 minute until it succeeds.

go.mod

@@ -8,7 +8,7 @@ require (
 	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
 	github.com/gxthrj/apisix-ingress-types v0.1.3
 	github.com/gxthrj/apisix-types v0.1.3
-	github.com/gxthrj/seven v0.2.6
+	github.com/gxthrj/seven v0.2.7
 	github.com/julienschmidt/httprouter v1.3.0
 	github.com/spf13/cobra v1.1.1
 	github.com/stretchr/testify v1.4.0

go.sum

@@ -127,7 +127,10 @@ github.com/gxthrj/seven v0.1.9/go.mod h1:epDVCYT1ibfV6fgaeM918QAer9u0/M2LXW6OcIR
 github.com/gxthrj/seven v0.2.0 h1:S85ZI2TzVQILsiI9BE+Sa0VqrkU3kzNYJFOOh4+5Vu8=
 github.com/gxthrj/seven v0.2.0/go.mod h1:Uf0JHSRmhZyV3tPLV1oVzq/Dw19ya9rXFsECiLKrgVk=
 github.com/gxthrj/seven v0.2.4/go.mod h1:SYs/veqEMdwRF5BL3nf/nxfypoDMO2E6Odgp17m+J9U=
+github.com/gxthrj/seven v0.2.6 h1:d7OteigPwnv9lLHjq6jWQ0IbTD3g+RnhS8ogDieskeg=
 github.com/gxthrj/seven v0.2.6/go.mod h1:SYs/veqEMdwRF5BL3nf/nxfypoDMO2E6Odgp17m+J9U=
+github.com/gxthrj/seven v0.2.7 h1:DNRi3HGXiTEC2O87jq9MqEMHjwf7eHvYQXhJxv1Qa5E=
+github.com/gxthrj/seven v0.2.7/go.mod h1:SYs/veqEMdwRF5BL3nf/nxfypoDMO2E6Odgp17m+J9U=
 github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
 github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=

pkg/ingress/apisix/tls.go

@@ -18,6 +18,7 @@ import (
 	ingressConf "github.com/api7/ingress-controller/pkg/kube"
 	ingress "github.com/gxthrj/apisix-ingress-types/pkg/apis/config/v1"
 	apisix "github.com/gxthrj/apisix-types/pkg/apis/apisix/v1"
+	"github.com/gxthrj/seven/conf"
 	"k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
@@ -32,6 +33,9 @@ type ApisixTlsCRD ingress.ApisixTls
 func (as *ApisixTlsCRD) Convert(sc Secreter) (*apisix.Ssl, error) {
 	name := as.Name
 	namespace := as.Namespace
+	_, group := BuildAnnotation(as.Annotations)
+	conf.AddGroup(group)
+
 	id := namespace + "_" + name
 	secretName := as.Spec.Secret.Name
 	secretNamespace := as.Spec.Secret.Namespace
@@ -52,6 +56,7 @@ func (as *ApisixTlsCRD) Convert(sc Secreter) (*apisix.Ssl, error) {
 		Cert:   &cert,
 		Key:    &key,
 		Status: &status,
+		Group:  &group,
 	}
 	return ssl, nil
 }

pkg/ingress/apisix/tls_test.go

@@ -45,22 +45,64 @@ spec:
 	status := int(1)
 	cert := "root"
 	key := "123456"
+	group := ""
 	sslExpect := &a6Type.Ssl{
 		ID:     &id,
 		Snis:   snis,
 		Cert:   &cert,
 		Key:    &key,
 		Status: &status,
+		Group:  &group,
 	}
 	atlsCRD := &ApisixTlsCRD{}
 	err := yaml.Unmarshal([]byte(atlsStr), atlsCRD)
 	assert.Nil(t, err, "yaml decode failed")
 	sc := &SecretClientMock{}
 	ssl, err := atlsCRD.Convert(sc)
-	assert.EqualValues(t, sslExpect.Key, ssl.Key, "ssl convert error")
-	assert.EqualValues(t, sslExpect.ID, ssl.ID, "ssl convert error")
-	assert.EqualValues(t, sslExpect.Cert, ssl.Cert, "ssl convert error")
-	assert.EqualValues(t, sslExpect.Snis, ssl.Snis, "ssl convert error")
+	assert.EqualValues(t, sslExpect.Key, ssl.Key, "key convert error")
+	assert.EqualValues(t, sslExpect.ID, ssl.ID, "id convert error")
+	assert.EqualValues(t, sslExpect.Cert, ssl.Cert, "cert convert error")
+	assert.EqualValues(t, sslExpect.Snis, ssl.Snis, "snis convert error")
+	assert.EqualValues(t, sslExpect.Group, ssl.Group, "group convert error")
+}
+
+func TestConvert_group_annotation(t *testing.T) {
+	atlsStr := `
+apiVersion: apisix.apache.org/v1
+kind: ApisixTls
+metadata:
+  annotations:
+    k8s.apisix.apache.org/ingress.class: 127.0.0.1:9080
+  name: foo
+  namespace: helm
+spec:
+  hosts:
+  - api6.com
+  secret:
+    name: test-atls
+    namespace: helm
+`
+	id := "helm_foo"
+	host := "api6.com"
+	snis := []*string{&host}
+	status := int(1)
+	cert := "root"
+	key := "123456"
+	group := "127.0.0.1:9080"
+	sslExpect := &a6Type.Ssl{
+		ID:     &id,
+		Snis:   snis,
+		Cert:   &cert,
+		Key:    &key,
+		Status: &status,
+		Group:  &group,
+	}
+	atlsCRD := &ApisixTlsCRD{}
+	err := yaml.Unmarshal([]byte(atlsStr), atlsCRD)
+	assert.Nil(t, err, "yaml decode failed")
+	sc := &SecretClientMock{}
+	ssl, err := atlsCRD.Convert(sc)
+	assert.EqualValues(t, sslExpect.Group, ssl.Group, "group convert error")
 }
 
 func TestConvert_Error(t *testing.T) {

pkg/ingress/controller/apisix_route.go

@@ -22,6 +22,7 @@ import (
 	api6Scheme "github.com/gxthrj/apisix-ingress-types/pkg/client/clientset/versioned/scheme"
 	api6Informers "github.com/gxthrj/apisix-ingress-types/pkg/client/informers/externalversions/config/v1"
 	"github.com/gxthrj/apisix-ingress-types/pkg/client/listers/config/v1"
+	apisixV1 "github.com/gxthrj/apisix-types/pkg/apis/apisix/v1"
 	"github.com/gxthrj/seven/state"
 	"k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/util/runtime"
@@ -208,8 +209,22 @@ func (c *ApisixRouteController) add(key string) error {
 	apisixRoute := apisix.ApisixRoute(*apisixIngressRoute)
 	routes, services, upstreams, _ := apisixRoute.Convert()
 	comb := state.ApisixCombination{Routes: routes, Services: services, Upstreams: upstreams}
-	_, err = comb.Solver()
-	return err
+	// protect: will retry when any upstream nodes is empty
+	retry := false
+	upstreamWithEmptyNodes := &apisixV1.Upstream{}
+	for _, upstream := range upstreams {
+		if len(upstream.Nodes) < 1 {
+			upstreamWithEmptyNodes = upstream
+			break
+		}
+	}
+	if !retry {
+		_, err = comb.Solver()
+		return err
+	} else {
+		return fmt.Errorf("upstream %s which nodes is empty", *upstreamWithEmptyNodes.Name)
+	}
+
 }
 
 // sync

pkg/ingress/controller/apisix_tls.go

@@ -149,9 +149,8 @@ func (c *ApisixTlsController) syncHandler(tqo *TlsQueueObj) error {
 		// sync to apisix
 		log.Debug(tls)
 		log.Debug(tqo)
-		state.SyncSsl(tls, tqo.Ope)
+		return state.SyncSsl(tls, tqo.Ope)
 	}
-	return err
 }
 
 func (c *ApisixTlsController) addFunc(obj interface{}) {