From 7386e02a57059e5f7f7047c5c4fec6a6c9f668b7 Mon Sep 17 00:00:00 2001 From: Vincent Marguerie <24724195+vincentmrg@users.noreply.github.com> Date: Fri, 8 Nov 2024 18:26:29 +0100 Subject: [PATCH] WIP --- Makefile | 2 +- config/samples/v1alpha1_firewallrule.yaml | 2 +- go.mod | 14 + go.sum | 28 + internal/controller/externalip_controller.go | 303 ++------ .../controller/firewallrule_controller.go | 549 +------------ internal/controller/utils.go | 21 +- .../aws/converter/security_group_decoder.go | 37 + .../aws/converter/security_group_encoder.go | 34 +- internal/provider/aws/imds.go | 109 +++ internal/provider/aws/provider.go | 735 ++++++++++++------ internal/provider/provider.go | 26 +- internal/provider/utils.go | 2 +- 13 files changed, 787 insertions(+), 1075 deletions(-) create mode 100644 internal/provider/aws/imds.go diff --git a/Makefile b/Makefile index 94f9188..54a1749 100644 --- a/Makefile +++ b/Makefile @@ -123,7 +123,7 @@ docker-push: ## Push docker image with the manager. # - have enabled BuildKit. More info: https://docs.docker.com/develop/develop-images/build_enhancements/ # - be able to push the image to your registry (i.e. if you do not set a valid value via IMG=> then the export will fail) # To adequately provide solutions that are compatible with multiple platforms, you should consider using this option. -PLATFORMS ?= linux/arm64,linux/amd64,linux/s390x,linux/ppc64le +PLATFORMS ?= linux/arm64,linux/amd64 .PHONY: docker-buildx docker-buildx: ## Build and push docker image for the manager for cross-platform support # copy existing Dockerfile and insert --platform=${BUILDPLATFORM} into Dockerfile.cross, and preserve the original Dockerfile diff --git a/config/samples/v1alpha1_firewallrule.yaml b/config/samples/v1alpha1_firewallrule.yaml index 7321802..da48cd5 100644 --- a/config/samples/v1alpha1_firewallrule.yaml +++ b/config/samples/v1alpha1_firewallrule.yaml @@ -3,7 +3,7 @@ kind: FirewallRule metadata: name: firewallrule-sample spec: - nodeName: ip-10-136-0-108.eu-west-1.compute.internal + nodeName: ip-10-100-212-255.eu-west-1.compute.internal description: An amazing firewall rule ! direction: Ingress protocol: tcp diff --git a/go.mod b/go.mod index 8c1db1a..7d25d98 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,20 @@ require ( require ( github.com/antlr4-go/antlr/v4 v4.13.0 // indirect github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a // indirect + github.com/aws/aws-sdk-go-v2 v1.32.3 // indirect + github.com/aws/aws-sdk-go-v2/config v1.28.1 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.17.42 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.18 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.22 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.22 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect + github.com/aws/aws-sdk-go-v2/service/ec2 v1.186.1 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.3 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.24.3 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.3 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.32.3 // indirect + github.com/aws/smithy-go v1.22.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect diff --git a/go.sum b/go.sum index 4c30ec5..b108257 100644 --- a/go.sum +++ b/go.sum @@ -6,6 +6,34 @@ github.com/aws/aws-sdk-go v1.44.211 h1:YNr5DwdzG/8y9Tl0QrPTnC99aFUHgT5hhy6GpnnzH github.com/aws/aws-sdk-go v1.44.211/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU= github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= +github.com/aws/aws-sdk-go-v2 v1.32.3 h1:T0dRlFBKcdaUPGNtkBSwHZxrtis8CQU17UpNBZYd0wk= +github.com/aws/aws-sdk-go-v2 v1.32.3/go.mod h1:2SK5n0a2karNTv5tbP1SjsX0uhttou00v/HpXKM1ZUo= +github.com/aws/aws-sdk-go-v2/config v1.28.1 h1:oxIvOUXy8x0U3fR//0eq+RdCKimWI900+SV+10xsCBw= +github.com/aws/aws-sdk-go-v2/config v1.28.1/go.mod h1:bRQcttQJiARbd5JZxw6wG0yIK3eLeSCPdg6uqmmlIiI= +github.com/aws/aws-sdk-go-v2/credentials v1.17.42 h1:sBP0RPjBU4neGpIYyx8mkU2QqLPl5u9cmdTWVzIpHkM= +github.com/aws/aws-sdk-go-v2/credentials v1.17.42/go.mod h1:FwZBfU530dJ26rv9saAbxa9Ej3eF/AK0OAY86k13n4M= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.18 h1:68jFVtt3NulEzojFesM/WVarlFpCaXLKaBxDpzkQ9OQ= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.18/go.mod h1:Fjnn5jQVIo6VyedMc0/EhPpfNlPl7dHV916O6B+49aE= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.22 h1:Jw50LwEkVjuVzE1NzkhNKkBf9cRN7MtE1F/b2cOKTUM= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.22/go.mod h1:Y/SmAyPcOTmpeVaWSzSKiILfXTVJwrGmYZhcRbhWuEY= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.22 h1:981MHwBaRZM7+9QSR6XamDzF/o7ouUGxFzr+nVSIhrs= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.22/go.mod h1:1RA1+aBEfn+CAB/Mh0MB6LsdCYCnjZm7tKXtnk499ZQ= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc= +github.com/aws/aws-sdk-go-v2/service/ec2 v1.186.1 h1:s3en74URaTjlhpJqOUCHlmombBFo88jxZqs3qjRmXrI= +github.com/aws/aws-sdk-go-v2/service/ec2 v1.186.1/go.mod h1:ossaD9Z1ugYb6sq9QIqQLEOorCGcqUoxlhud9M9yE70= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 h1:TToQNkvGguu209puTojY/ozlqy2d/SFNcoLIqTFi42g= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0/go.mod h1:0jp+ltwkf+SwG2fm/PKo8t4y8pJSgOCO4D8Lz3k0aHQ= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.3 h1:qcxX0JYlgWH3hpPUnd6U0ikcl6LLA9sLkXE2w1fpMvY= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.3/go.mod h1:cLSNEmI45soc+Ef8K/L+8sEA3A3pYFEYf5B5UI+6bH4= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.3 h1:UTpsIf0loCIWEbrqdLb+0RxnTXfWh2vhw4nQmFi4nPc= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.3/go.mod h1:FZ9j3PFHHAR+w0BSEjK955w5YD2UwB/l/H0yAK3MJvI= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.3 h1:2YCmIXv3tmiItw0LlYf6v7gEHebLY45kBEnPezbUKyU= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.3/go.mod h1:u19stRyNPxGhj6dRm+Cdgu6N75qnbW7+QN0q0dsAk58= +github.com/aws/aws-sdk-go-v2/service/sts v1.32.3 h1:wVnQ6tigGsRqSWDEEyH6lSAJ9OyFUsSnbaUWChuSGzs= +github.com/aws/aws-sdk-go-v2/service/sts v1.32.3/go.mod h1:VZa9yTFyj4o10YGsmDO4gbQJUvvhY72fhumT8W4LqsE= +github.com/aws/smithy-go v1.22.0 h1:uunKnWlcoL3zO7q+gG2Pk53joueEOsnNB28QdMsmiMM= +github.com/aws/smithy-go v1.22.0/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= diff --git a/internal/controller/externalip_controller.go b/internal/controller/externalip_controller.go index df8c8d8..4da2d62 100644 --- a/internal/controller/externalip_controller.go +++ b/internal/controller/externalip_controller.go @@ -20,7 +20,7 @@ import ( "context" "encoding/json" "fmt" - "time" + "reflect" "github.com/go-logr/logr" "github.com/google/uuid" @@ -83,285 +83,86 @@ func (r *ExternalIPReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, nil } - // Lifecycle reconciliation - if externalIP.ObjectMeta.DeletionTimestamp.IsZero() { - return r.reconcileExternalIP(ctx, log, externalIP) - } - - // Deletion reconciliation - return r.reconcileExternalIPDeletion(ctx, log, externalIP) -} + // Get node from ExternalIP spec + var node corev1.Node + if externalIP.Spec.NodeName != "" { + if err := r.Get(ctx, types.NamespacedName{Name: externalIP.Spec.NodeName}, &node); err != nil { + if errors.IsNotFound(err) { + // Invalid nodeName, remove ExternalIP nodeName attribute. + log.Info("Node not found. Removing it from ExternalIP spec", "nodeName", externalIP.Spec.NodeName) + externalIP.Spec.NodeName = "" + return ctrl.Result{}, r.Update(ctx, externalIP) + } + // Error reading the object - requeue the request. + log.Error(err, "Failed to get Node") + return ctrl.Result{}, err + } -func (r *ExternalIPReconciler) reconcileExternalIP(ctx context.Context, log logr.Logger, externalIP *v1alpha1.ExternalIP) (ctrl.Result, error) { - // 1st STEP - // - // Add finalizer - if !controllerutil.ContainsFinalizer(externalIP, externalIPFinalizer) { - controllerutil.AddFinalizer(externalIP, externalIPFinalizer) - log.V(1).Info("Updating ExternalIP", "finalizer", externalIPFinalizer) - return ctrl.Result{}, r.Update(ctx, externalIP) } + instanceID := r.Provider.GetInstanceID(node) - // 2nd STEP - // - // Reserve external IP address - if externalIP.Status.State == v1alpha1.ExternalIPStateNone { - // Create external address - res, err := r.Provider.CreateAddress(ctx) + var status v1alpha1.ExternalIPStatus + var err error + if externalIP.ObjectMeta.DeletionTimestamp.IsZero() { + status, err = r.Provider.ReconcileExternalIP(ctx, instanceID, externalIP) if err != nil { - log.Error(err, "Failed to create address") return ctrl.Result{}, err } - log.Info("Created address", "id", res.AddressID, "publicIP", res.PublicIP) - - // Update status - externalIP.Status.State = v1alpha1.ExternalIPStateReserved - externalIP.Status.AddressID = &res.AddressID - externalIP.Status.PublicIPAddress = &res.PublicIP - log.V(1).Info( - "Updating ExternalIP", - "state", externalIP.Status.State, - "addressID", externalIP.Status.AddressID, - "PublicIPAddress", externalIP.Status.PublicIPAddress, - ) - return ctrl.Result{RequeueAfter: time.Second * 5}, r.Status().Update(ctx, externalIP) - } - - // 3rd STEP - // - // Finally associate external ip to instance network interface. - // This must be the last step, since this exposes the instance on the outside. - if externalIP.IsReserved() { - if externalIP.Spec.NodeName != "" { - // Get node from ExternalIP spec - var node corev1.Node - if err := r.Get(ctx, types.NamespacedName{Name: externalIP.Spec.NodeName}, &node); err != nil { - if errors.IsNotFound(err) { - // Invalid nodeName, remove ExternalIP nodeName attribute. - log.Info("Node not found. Removing it from ExternalIP spec", "nodeName", externalIP.Spec.NodeName) - externalIP.Spec.NodeName = "" - return ctrl.Result{}, r.Update(ctx, externalIP) - } - // Error reading the object - requeue the request. - log.Error(err, "Failed to get Node") - return ctrl.Result{}, err - } - - // Retrieve node instance - instanceID := r.Provider.GetInstanceID(node) - res, err := r.Provider.GetInstance(ctx, instanceID) + // Node not being deleted, reconcile externalip label + if node.ObjectMeta.DeletionTimestamp.IsZero() { + // Marshal node, ... + old, err := json.Marshal(node) if err != nil { - log.Error(err, "Failed to get instance", "id", instanceID) + log.Error(err, "Failed to marshal node") return ctrl.Result{}, err } - // Interface with index 0 is the first attached to node, the one we're interested in. - // Each instance has a default network interface, called the primary network interface. - // You cannot detach a primary network interface from an instance. - var networkInterface *provider.NetworkInterface - for _, elem := range res.NetworkInterfaces { - if elem != nil && *elem.DeviceID == 0 { - networkInterface = elem - break - } - } - if networkInterface == nil { - err := fmt.Errorf("no network interface with public IP found for instance %s", instanceID) - log.Error(err, "Cannot associate an address with this instance", "instanceID", instanceID) + // ... then compute new node to marshal it... + node.Labels[externalIPLabel] = *status.PublicIPAddress + new, err := json.Marshal(node) + if err != nil { + log.Error(err, "Failed to marshal new node") return ctrl.Result{}, err } - // Finally, associate address to instance network interface, then update status. - if err := r.Provider.AssociateAddress(ctx, provider.AssociateAddressRequest{ - AddressID: *externalIP.Status.AddressID, - NetworkInterfaceID: networkInterface.NetworkInterfaceID, - }); err != nil { - log.Error( - err, - "Failed to associate address", - "addressID", *externalIP.Status.AddressID, - "instanceID", instanceID, - "networkInterfaceID", networkInterface.NetworkInterfaceID, - ) + // ... and create a patch. + patch, err := strategicpatch.CreateTwoWayMergePatch(old, new, node) + if err != nil { + log.Error(err, "Failed to create patch for node") return ctrl.Result{}, err } - log.Info( - "Associated address", - "addressID", *externalIP.Status.AddressID, - "instanceID", instanceID, - "networkInterfaceID", networkInterface.NetworkInterfaceID, - ) - - // Update status - externalIP.Status.State = v1alpha1.ExternalIPStateAssociated - externalIP.Status.InstanceID = &instanceID - log.V(1).Info("Updating ExternalIP", "state", externalIP.Status.State, "InstanceID", externalIP.Status.InstanceID) - return ctrl.Result{RequeueAfter: time.Second * 5}, r.Status().Update(ctx, externalIP) - } - - // No spec.nodeName, no association, end reconciliation for ExternalIP. - log.V(1).Info("No No spec.nodeName, no association, end reconciliation for ExternalIP.") - return ctrl.Result{}, nil - } - - // ExternalIP reliability check - // - // Check if the associated node still exists and disassociate it if it does not. - // No nodeName or no living node, set state back to "Reserved" - if externalIP.IsAssociated() { - if externalIP.Spec.NodeName != "" { - // Get node from ExternalIP spec - var node corev1.Node - if err := r.Get(ctx, types.NamespacedName{Name: externalIP.Spec.NodeName}, &node); err != nil { - if errors.IsNotFound(err) { - // Invalid nodeName, remove ExternalIP nodeName attribute. - log.Info("Node not found. Removing it from ExternalIP spec", "nodeName", externalIP.Spec.NodeName) - - // Set status back to Reserved - externalIP.Status.State = v1alpha1.ExternalIPStateReserved - log.V(1).Info("Updating ExternalIP", "state", externalIP.Status.State, "InstanceID", externalIP.Status.InstanceID) - if err = r.Status().Update(ctx, externalIP); err != nil { - log.Error(err, "Failed to update ExternalIP status", "externalIP", externalIP.Name, "status", externalIP.Status.State) - return ctrl.Result{}, err - } - externalIP.Spec.NodeName = "" - return ctrl.Result{}, r.Update(ctx, externalIP) - } - // Error reading the object - requeue the request. - log.Error(err, "Failed to get Node") + // Apply patch to set node's wanted labels. + if err = r.Client.Patch(ctx, &node, client.RawPatch(types.MergePatchType, patch)); err != nil { + log.Error(err, "Failed to patch node") return ctrl.Result{}, err } - - // Node not being deleted, reconcile externalip label - if node.ObjectMeta.DeletionTimestamp.IsZero() { - // Marshal node, ... - old, err := json.Marshal(node) - if err != nil { - log.Error(err, "Failed to marshal node") - return ctrl.Result{}, err - } - - // ... then compute new node to marshal it... - node.Labels[externalIPLabel] = *externalIP.Status.PublicIPAddress - new, err := json.Marshal(node) - if err != nil { - log.Error(err, "Failed to marshal new node") - return ctrl.Result{}, err - } - - // ... and create a patch. - patch, err := strategicpatch.CreateTwoWayMergePatch(old, new, node) - if err != nil { - log.Error(err, "Failed to create patch for node") - return ctrl.Result{}, err - } - - // Apply patch to set node's wanted labels. - if err = r.Client.Patch(ctx, &node, client.RawPatch(types.MergePatchType, patch)); err != nil { - log.Error(err, "Failed to patch node") - return ctrl.Result{}, err - } - - return ctrl.Result{}, nil - } } - - // Set state back to "Reserved", disassociate address and end reconciliation - return r.disassociateAddress(ctx, r.Provider, log, externalIP) - } - - return ctrl.Result{}, nil -} - -func (r *ExternalIPReconciler) reconcileExternalIPDeletion( - ctx context.Context, - log logr.Logger, - externalIP *v1alpha1.ExternalIP, -) (ctrl.Result, error) { - // 1st STEP - // - // Reconciliation of a possible external IP associated with the instance. - // If an IP is associated with the instance, disassociate it. - if externalIP.IsAssociated() { - return r.disassociateAddress(ctx, r.Provider, log, externalIP) - } - - // 2nd STEP - // - // Release unassociated address. - if externalIP.IsReserved() { - // Do not delete EIP if flag PreventEIPDeallocation is set - if externalIP.Status.AddressID != nil && !externalIP.Spec.PreventEIPDeallocation { - if err := r.Provider.DeleteAddress(ctx, *externalIP.Status.AddressID); err != nil { - if !provider.IsErrNotFound(err) { - log.Error(err, "Failed to delete Address", "addressID", *externalIP.Status.AddressID) - return ctrl.Result{}, err - } - log.V(1).Info("Address not found", "addressID", *externalIP.Status.AddressID) - } - log.Info("Deleted Address", "addressID", *externalIP.Status.AddressID) - - // Update status - externalIP.Status.AddressID = nil + } else { + if err := r.Provider.ReconcileExternalIPDeletion(ctx, externalIP); err != nil { + return ctrl.Result{}, err } - // set State to None for finalizer to delete externalIP object - externalIP.Status.State = v1alpha1.ExternalIPStateNone - log.V(1).Info("Updating ExternalIP", "state", externalIP.Status.State) - return ctrl.Result{RequeueAfter: time.Second * 5}, r.Status().Update(ctx, externalIP) } - // 3rd STEP - // - // Remove finalizer to release ExternalIP - if externalIP.Status.State == v1alpha1.ExternalIPStateNone { - if controllerutil.ContainsFinalizer(externalIP, externalIPFinalizer) { - controllerutil.RemoveFinalizer(externalIP, externalIPFinalizer) - return ctrl.Result{}, r.Update(ctx, externalIP) - } + if !externalIP.DeletionTimestamp.IsZero() && controllerutil.ContainsFinalizer(externalIP, externalIPFinalizer) { + controllerutil.RemoveFinalizer(externalIP, externalIPFinalizer) + return ctrl.Result{}, r.Update(ctx, externalIP) } - return ctrl.Result{}, nil -} + // Copy the existing ExternalIP to avoid mutating the original + existingExternalIP := externalIP.DeepCopy() -// disassociateAddress performs address disassociation tasks -func (r *ExternalIPReconciler) disassociateAddress( - ctx context.Context, - pvd provider.Provider, - log logr.Logger, - externalIP *v1alpha1.ExternalIP, -) (ctrl.Result, error) { - // Get address and disassociate it - if externalIP.Status.AddressID != nil { - res, err := pvd.GetAddress(ctx, *externalIP.Status.AddressID) + // Patch the Pool status if it differs from the desired status + externalIP.Status = status + if !reflect.DeepEqual(externalIP.Status, existingExternalIP.Status) { + err := r.Status().Patch(ctx, externalIP, client.MergeFrom(existingExternalIP)) if err != nil { - log.Error(err, "Failed to retrieve address", "addressID", *externalIP.Status.AddressID) - return ctrl.Result{}, err - } - - if res.AssociationID != nil { - if err := pvd.DisassociateAddress(ctx, provider.DisassociateAddressRequest{ - AssociationID: *res.AssociationID, - }); err != nil { - log.Error(err, "Failed to disassociate address", "addressID", *externalIP.Status.AddressID, "instanceID", *externalIP.Status.InstanceID) - return ctrl.Result{}, err - } - log.Info("Disassociated address", "addressID", *externalIP.Status.AddressID, "instanceID", *externalIP.Status.InstanceID) + log.Error(err, "Failed to patch ExternalIP status") + return ctrl.Result{}, fmt.Errorf("failed to patch ExternalIP status: %w", err) } } - // Update status - externalIP.Status.State = v1alpha1.ExternalIPStateReserved - externalIP.Status.InstanceID = nil - log.V(1).Info("Updating ExternalIP", "state", externalIP.Status.State) - if err := r.Status().Update(ctx, externalIP); err != nil { - log.Error(err, "Failed to update ExternalIP state", "externalIP", externalIP.Name) - return ctrl.Result{}, err - } - - log.V(1).Info("Removing ExternalIP NodeName", "externalIP", externalIP.Name) - externalIP.Spec.NodeName = "" - return ctrl.Result{}, r.Update(ctx, externalIP) + return ctrl.Result{}, nil } // SetupWithManager sets up the controller with the Manager. diff --git a/internal/controller/firewallrule_controller.go b/internal/controller/firewallrule_controller.go index bf6c545..21d3a82 100644 --- a/internal/controller/firewallrule_controller.go +++ b/internal/controller/firewallrule_controller.go @@ -18,10 +18,7 @@ package controller import ( "context" - "encoding/json" - "fmt" - "reflect" - "time" + "slices" "github.com/go-logr/logr" "github.com/google/uuid" @@ -84,441 +81,67 @@ func (r *FirewallRuleReconciler) Reconcile(ctx context.Context, req ctrl.Request return ctrl.Result{}, nil } - // LastTransitionTime is not set. This should happen when kubestatic is - // upgraded from a version that does not support this field, we set it with - // the current time. - if firewallRule.Status.LastTransitionTime.IsZero() { - firewallRule.Status.LastTransitionTime = metav1.Now() - if err := r.Status().Update(ctx, firewallRule); err != nil { - log.Error(err, "Failed to update FirewallRule state", "firewallRule", firewallRule.Name) - return ctrl.Result{}, err - } - r.frLastTransitionTime[firewallRule.Name] = firewallRule.Status.LastTransitionTime - } - - // Check for LastTransitionTime consistency, if not, requeueing. - knownLastTransitionTime := r.frLastTransitionTime[firewallRule.Name] - if firewallRule.Status.LastTransitionTime.Before(&knownLastTransitionTime) { - log.V(1).Info("FirewallRule LastTransitionTime inconsistency, requeuing in 1 second") - return ctrl.Result{RequeueAfter: time.Second}, nil - } - r.frLastTransitionTime[firewallRule.Name] = firewallRule.Status.LastTransitionTime - - // Lifecycle reconciliation - if firewallRule.ObjectMeta.DeletionTimestamp.IsZero() { - return r.reconcileFirewallRule(ctx, log, firewallRule) + // TODO: Check what to do with the case where the nodeName is nil. + if firewallRule.Spec.NodeName == nil { + return ctrl.Result{}, nil } - // Deletion reconciliation - return r.reconcileFirewallRuleDeletion(ctx, log, firewallRule) -} - -//nolint:gocyclo -func (r *FirewallRuleReconciler) reconcileFirewallRule( - ctx context.Context, - log logr.Logger, - rule *v1alpha1.FirewallRule, -) (ctrl.Result, error) { - // 1st STEP - // // Add finalizer - if !controllerutil.ContainsFinalizer(rule, firewallRuleFinalizer) { - rule.ObjectMeta.Finalizers = append(rule.ObjectMeta.Finalizers, firewallRuleFinalizer) + if !controllerutil.ContainsFinalizer(firewallRule, firewallRuleFinalizer) { + firewallRule.ObjectMeta.Finalizers = append(firewallRule.ObjectMeta.Finalizers, firewallRuleFinalizer) log.V(1).Info("Updating FirewallRule", "finalizer", firewallRuleFinalizer) - return ctrl.Result{}, r.Update(ctx, rule) + return ctrl.Result{}, r.Update(ctx, firewallRule) } - // 2nd STEP - // - // Reserve firewall - if rule.Status.State == v1alpha1.FirewallRuleStateNone && rule.Spec.NodeName != nil { - // Create firewall rule - // In the case of standalone firewall rules, we create it, - // otherwise, we update the group dedicated to the node. - var id string - var err error - if r.Provider.HasGroupedFirewallRules() { - // List FirewallRules with identical nodeName - frs := &v1alpha1.FirewallRuleList{} - if err := r.List(ctx, frs, client.MatchingFields{firewallRuleNodeNameKey: *rule.Spec.NodeName}); err != nil { - log.Error(err, "Unable to list FirewallRules") - return ctrl.Result{}, err - } - - // Check for other rules associated to the node. - // If there is already one, we update the group of rules, if not, we create a new group. - rulesAssociated := []v1alpha1.FirewallRule{} - for _, fr := range frs.Items { - knownLastTransitionTime := r.frLastTransitionTime[fr.Name] - if !fr.Status.LastTransitionTime.Equal(&knownLastTransitionTime) { - log.V(1).Info("FirewallRule LastTransitionTime inconsistency, requeuing in 1 second", "firewallRuleName", fr.Name) - return ctrl.Result{RequeueAfter: time.Second}, nil - } - if fr.Name != rule.Name && fr.Status.State != v1alpha1.FirewallRuleStateNone { - rulesAssociated = append(rulesAssociated, fr) - } - } - - if len(rulesAssociated) > 0 { - firewallRuleID := ptr.Deref(rulesAssociated[0].Status.FirewallRuleID, "") - log.V(1).Info("Updating FirewallRule group", "firewallRuleID", firewallRuleID) - id, err = r.Provider.UpdateFirewallRuleGroup(ctx, encodeUpdateFirewallRuleGroupRequest(firewallRuleID, frs.Items)) - if err != nil { - log.Error(err, "Unable to update FirewallRules") - return ctrl.Result{}, err - } - } else { - // No existing group, we create a new one. - log.V(1).Info("Creating FirewallRule group") - id, err = r.Provider.CreateFirewallRuleGroup( - ctx, - encodeCreateFirewallRuleGroupRequest( - fmt.Sprintf("kubestatic-%s", randomString(10)), - fmt.Sprintf("Kubestatic managed group for node %s", *rule.Spec.NodeName), - frs.Items, - ), - ) - } - } else { - // Standalone rules, we simply create a rule. - log.V(1).Info("Creating FirewallRule") - id, err = r.Provider.CreateFirewallRule(ctx, encodeCreateFirewallRuleRequest(rule)) - } - - if err != nil { - log.Error(err, "Failed to create firewall rule") - return ctrl.Result{}, err - } - log.Info("Created firewall rule", "id", id) - - // Update status - rule.Status.LastTransitionTime = metav1.Now() - rule.Status.State = v1alpha1.FirewallRuleStateReserved - rule.Status.FirewallRuleID = &id - lastApplied, err := json.Marshal(rule.Spec) - if err != nil { - return ctrl.Result{}, fmt.Errorf("Failed to marshal last applied firewallrule: %w", err) - } - rule.Status.LastApplied = nil - if len(lastApplied) > 0 { - rule.Status.LastApplied = ptr.To(string(lastApplied)) - } - log.V(1).Info("Updating FirewallRule", "state", rule.Status.State, "firewallRuleID", rule.Status.FirewallRuleID) - if err = r.Status().Update(ctx, rule); err != nil { - log.Error(err, "Failed to update FirewallRule status", "firewallRule", rule.Name, "status", rule.Status.State) - return ctrl.Result{}, err - } - r.frLastTransitionTime[rule.Name] = rule.Status.LastTransitionTime - return ctrl.Result{RequeueAfter: time.Second * 5}, nil - - } else if rule.Spec.NodeName != nil { - lastApplied := &v1alpha1.FirewallRuleSpec{} - if err := json.Unmarshal([]byte(ptr.Deref(rule.Status.LastApplied, "")), lastApplied); err != nil { - return ctrl.Result{}, fmt.Errorf("Failed to unmarshal last applied firewallrule: %w", err) - } - - // Update firewall rule. - // In the case of standalone firewall rules, we update it, - // otherwise, we update the group dedicated to the node. - if !reflect.DeepEqual(rule.Spec, *lastApplied) { - // Update firewall rule - firewallRuleID := ptr.Deref(rule.Status.FirewallRuleID, "") - var err error - if r.Provider.HasGroupedFirewallRules() { - // List FirewallRules with identical nodeName - frs := &v1alpha1.FirewallRuleList{} - if err := r.List(ctx, frs, client.MatchingFields{firewallRuleNodeNameKey: *rule.Spec.NodeName}); err != nil { - log.Error(err, "Unable to list FirewallRules") - return ctrl.Result{}, err - } - rules := []v1alpha1.FirewallRule{} - for _, fr := range frs.Items { - knownLastTransitionTime := r.frLastTransitionTime[fr.Name] - if !fr.Status.LastTransitionTime.Equal(&knownLastTransitionTime) { - log.V(1).Info("FirewallRule LastTransitionTime inconsistency, requeuing in 1 second", "firewallRuleName", fr.Name) - return ctrl.Result{RequeueAfter: time.Second}, nil - } - rules = append(rules, fr) - } - log.V(1).Info("Updating FirewallRule group", "firewallRuleID", firewallRuleID) - _, err = r.Provider.UpdateFirewallRuleGroup(ctx, encodeUpdateFirewallRuleGroupRequest(firewallRuleID, rules)) - } else { - log.V(1).Info("Updating FirewallRule", "firewallRuleID", firewallRuleID) - _, err = r.Provider.UpdateFirewallRule(ctx, encodeUpdateFirewallRuleRequest(firewallRuleID, rule)) - } - - if err != nil { - log.Error(err, "Failed to update firewall rule", "id", firewallRuleID) - return ctrl.Result{}, err - } - log.Info("Updated firewall rule", "id", firewallRuleID) - - // Update status - lastApplied, err := json.Marshal(rule.Spec) - if err != nil { - return ctrl.Result{}, fmt.Errorf("Failed to marshal last applied firewallrule: %w", err) - } - rule.Status.LastTransitionTime = metav1.Now() - rule.Status.LastApplied = nil - if len(lastApplied) > 0 { - rule.Status.LastApplied = ptr.To(string(lastApplied)) - } - log.V(1).Info("Updating FirewallRule", "state", rule.Status.State, "firewallRuleID", rule.Status.FirewallRuleID) - if err = r.Status().Update(ctx, rule); err != nil { - log.Error(err, "Failed to update FirewallRule status", "firewallRule", rule.Name, "status", rule.Status.State) - return ctrl.Result{}, err - } - r.frLastTransitionTime[rule.Name] = rule.Status.LastTransitionTime - return ctrl.Result{RequeueAfter: time.Second * 5}, nil + // Get node from FirewallRule spec + var node corev1.Node + if err := r.Get(ctx, types.NamespacedName{Name: *firewallRule.Spec.NodeName}, &node); err != nil { + if errors.IsNotFound(err) { + // Invalid nodeName, remove FirewallRule nodeName attribute. + log.Info("Node not found. Removing it from FirewallRule spec", "nodeName", firewallRule.Spec.NodeName) + firewallRule.Spec.NodeName = nil + return ctrl.Result{}, r.Update(ctx, firewallRule) } + // Error reading the object - requeue the request. + log.Error(err, "Failed to get Node") + return ctrl.Result{}, err } - // 3rd STEP - // - // Finally associate firewall rule to instance network interface. - if rule.IsReserved() && rule.Spec.NodeName != nil { - // Get node from FirewallRule spec - var node corev1.Node - if err := r.Get(ctx, types.NamespacedName{Name: *rule.Spec.NodeName}, &node); err != nil { - if errors.IsNotFound(err) { - // Invalid nodeName, remove FirewallRule nodeName attribute. - log.Info("Node not found. Removing it from FirewallRule spec", "nodeName", rule.Spec.NodeName) - rule.Spec.NodeName = nil - return ctrl.Result{}, r.Update(ctx, rule) - } - // Error reading the object - requeue the request. - log.Error(err, "Failed to get Node") - return ctrl.Result{}, err - } + // Retrieve node instance + instanceID := r.Provider.GetInstanceID(node) - // Retrieve node instance - instanceID := r.Provider.GetInstanceID(node) - res, err := r.Provider.GetInstance(ctx, instanceID) - if err != nil { - log.Error(err, "Failed to get instance", "id", instanceID) - return ctrl.Result{}, err - } + // List FirewallRules with identical nodeName + firewallRuleList := &v1alpha1.FirewallRuleList{} + if err := r.List(ctx, firewallRuleList, client.MatchingFields{firewallRuleNodeNameKey: *firewallRule.Spec.NodeName}); err != nil { + log.Error(err, "Unable to list FirewallRules") + return ctrl.Result{}, err + } - // Get the first network interface with a public IP address - // This is needed because we could have multiple network interfaces, - // for example on EKS we have the public one, as well as one or more created by the EKS CNI. - var networkInterface *provider.NetworkInterface - for _, elem := range res.NetworkInterfaces { - if elem != nil && elem.PublicIP != nil { - networkInterface = elem - break - } - } - if networkInterface == nil { - err := fmt.Errorf("no network interface with public IP found for instance %s", instanceID) - log.Error(err, "Cannot associate a firewall rule with this instance", "instanceID", instanceID) - return ctrl.Result{}, err - } + // Remove deleted FirewallRules from the list + frs := slices.DeleteFunc(firewallRuleList.Items, func(fr v1alpha1.FirewallRule) bool { + return !fr.DeletionTimestamp.IsZero() + }) - // Finally, associate firewall rule to instance network interface, then update status. - if err := r.Provider.AssociateFirewallRule(ctx, provider.AssociateFirewallRuleRequest{ - FirewallRuleID: *rule.Status.FirewallRuleID, - NetworkInterfaceID: networkInterface.NetworkInterfaceID, - }); err != nil { - log.Error( - err, - "Failed to associate firewall rule", - "firewallRuleID", *rule.Status.FirewallRuleID, - "instanceID", instanceID, - "networkInterfaceID", networkInterface.NetworkInterfaceID, - ) + if len(frs) > 0 { + if err := r.Provider.ReconcileFirewallRules(ctx, instanceID, frs); err != nil { return ctrl.Result{}, err } - log.Info( - "Associated firewall rule", - "firewallRuleID", *rule.Status.FirewallRuleID, - "instanceID", instanceID, - "networkInterfaceID", networkInterface.NetworkInterfaceID, - ) - - // Update status - rule.Status.LastTransitionTime = metav1.Now() - rule.Status.State = v1alpha1.FirewallRuleStateAssociated - rule.Status.InstanceID = &instanceID - rule.Status.NetworkInterfaceID = &networkInterface.NetworkInterfaceID - log.V(1).Info( - "Updating FirewallRule", - "state", rule.Status.State, - "instanceID", rule.Status.InstanceID, - "networkInterfaceID", rule.Status.NetworkInterfaceID, - ) - if err = r.Status().Update(ctx, rule); err != nil { - log.Error(err, "Failed to update FirewallRule status", "firewallRule", rule.Name, "status", rule.Status.State) + } else { + if err := r.Provider.ReconcileFirewallRulesDeletion(ctx, instanceID); err != nil { return ctrl.Result{}, err } - r.frLastTransitionTime[rule.Name] = rule.Status.LastTransitionTime - return ctrl.Result{RequeueAfter: time.Second * 5}, nil - } - - // FirewallRule reliability check - // - // Check if the associated node still exists and disassociate it if it does not. - // No nodeName or no living node, set state back to "Reserved" - if rule.Status.State != v1alpha1.FirewallRuleStateNone { - if rule.Spec.NodeName != nil { - // Get node from FirewallRule spec - var node corev1.Node - if err := r.Get(ctx, types.NamespacedName{Name: *rule.Spec.NodeName}, &node); err != nil { - if errors.IsNotFound(err) { - // Invalid nodeName, remove FirewallRule nodeName attribute. - log.Info("Node not found. Set state back to Reserved", "nodeName", rule.Spec.NodeName) - - // Set status back to Reserved - rule.Status.LastTransitionTime = metav1.Now() - rule.Status.State = v1alpha1.FirewallRuleStateReserved - log.V(1).Info("Updating FirewallRule", "state", rule.Status.State, "InstanceID", rule.Status.InstanceID) - if err = r.Status().Update(ctx, rule); err != nil { - log.Error(err, "Failed to update FirewallRule status", "firewallRule", rule.Name, "status", rule.Status.State) - return ctrl.Result{}, err - } - r.frLastTransitionTime[rule.Name] = rule.Status.LastTransitionTime - - rule.Spec.NodeName = nil - return ctrl.Result{}, r.Update(ctx, rule) - } - // Error reading the object - requeue the request. - log.Error(err, "Failed to get Node") - return ctrl.Result{}, err - } - - // If the node is not being deleted and has an instance corresponding to its node name, the reconciliation is done - // This check exist to disassociate the rule of the old instance if the node name change - if node.ObjectMeta.DeletionTimestamp.IsZero() && ptr.Deref(rule.Status.InstanceID, "") == r.Provider.GetInstanceID(node) { - return ctrl.Result{}, nil - } - } - - // If the rule has no node name, has an node name not matching its instance ID, or its node is being deleted - // clear firewall rule from provider and set state back to "None" - return r.clearFirewallRule(ctx, log, rule) } - return ctrl.Result{}, nil -} - -func (r *FirewallRuleReconciler) reconcileFirewallRuleDeletion( - ctx context.Context, - log logr.Logger, - rule *v1alpha1.FirewallRule, -) (ctrl.Result, error) { - // 1st STEP - // - // Reconciliation of a possible firewall rule associated with the instance. - // If a rule is associated with an instance or reserved, clear it. - if rule.Status.State != v1alpha1.FirewallRuleStateNone { - return r.clearFirewallRule(ctx, log, rule) + if !firewallRule.DeletionTimestamp.IsZero() && controllerutil.ContainsFinalizer(firewallRule, firewallRuleFinalizer) { + controllerutil.RemoveFinalizer(firewallRule, firewallRuleFinalizer) + return ctrl.Result{}, r.Update(ctx, firewallRule) } - // 2nd STEP - // - // Remove finalizer to release FirewallRule - if controllerutil.ContainsFinalizer(rule, firewallRuleFinalizer) { - controllerutil.RemoveFinalizer(rule, firewallRuleFinalizer) - return ctrl.Result{}, r.Update(ctx, rule) - } + // TODO: Set Status return ctrl.Result{}, nil } -// clearFirewallRule remove the rule from the provider rule -// In the case of grouped rules, the provider rule is updated and deleted if needed -// In the case of standalone rules the provider rule is deleted -func (r *FirewallRuleReconciler) clearFirewallRule(ctx context.Context, log logr.Logger, rule *v1alpha1.FirewallRule) (ctrl.Result, error) { - log = log.WithValues("ruleName", rule.Name) - - if rule.Status.FirewallRuleID != nil { - firewallRuleID := ptr.Deref(rule.Status.FirewallRuleID, "") - - toDelete := false - if r.Provider.HasGroupedFirewallRules() { - // List FirewallRules - frs := &v1alpha1.FirewallRuleList{} - if err := r.List(ctx, frs); err != nil { - log.Error(err, "Unable to list FirewallRules") - return ctrl.Result{}, err - } - - // Check for other rules associated to the node. - // If there is other ones, we only update the group of rules, if not, we also disassociate the group. - rules := []v1alpha1.FirewallRule{} - for _, fr := range frs.Items { - knownLastTransitionTime := r.frLastTransitionTime[fr.Name] - if !fr.Status.LastTransitionTime.Equal(&knownLastTransitionTime) { - log.V(1).Info("FirewallRule LastTransitionTime inconsistency, requeuing in 1 second", "firewallRuleName", fr.Name) - return ctrl.Result{RequeueAfter: time.Second}, nil - } - if fr.Name != rule.Name && ptr.Deref(fr.Status.FirewallRuleID, "") == ptr.Deref(rule.Status.FirewallRuleID, "") { - rules = append(rules, fr) - } - } - if len(rules) > 0 { - log.V(1).Info("Updating FirewallRule", "firewallRuleID", firewallRuleID) - if _, err := r.Provider.UpdateFirewallRuleGroup(ctx, encodeUpdateFirewallRuleGroupRequest(firewallRuleID, rules)); err != nil { - log.Error(err, "Unable to update FirewallRules") - return ctrl.Result{}, err - } - } else { - toDelete = true - } - } else { - toDelete = true - } - - // Perform firewallrule deletion if needed - if toDelete { - if rule.Status.NetworkInterfaceID != nil { - log.V(1).Info("Disassociating firewall rule on provider", "firewallRuleID", firewallRuleID) - err := r.Provider.DisassociateFirewallRule(ctx, provider.AssociateFirewallRuleRequest{ - FirewallRuleID: *rule.Status.FirewallRuleID, - NetworkInterfaceID: *rule.Status.NetworkInterfaceID, - }) - if err != nil { - if !provider.IsErrNotFound(err) { - log.Error( - err, - "Failed to disassociate firewall rule", - "firewallRuleID", *rule.Status.FirewallRuleID, - "networkInterfaceID", *rule.Status.NetworkInterfaceID, - ) - return ctrl.Result{}, err - } - log.V(1).Info("Firewall rule already disassociated", "firewallRuleID", *rule.Status.FirewallRuleID) - } else { - log.Info("Disassociated firewall rule", "firewallRuleID", *rule.Status.FirewallRuleID, "networkInterfaceID", *rule.Status.NetworkInterfaceID) - } - } - - log.V(1).Info("Deleting firewall rule on provider", "firewallRuleID", firewallRuleID) - err := r.Provider.DeleteFirewallRule(ctx, *rule.Status.FirewallRuleID) - if err != nil { - if !provider.IsErrNotFound(err) { - log.Error(err, "Failed to delete firewall rule", "firewallRuleID", *rule.Status.FirewallRuleID) - return ctrl.Result{}, err - } - log.V(1).Info("Firewall rule already deleted", "firewallRuleID", *rule.Status.FirewallRuleID) - } else { - log.Info("Deleted firewall rule", "firewallRuleID", *rule.Status.FirewallRuleID) - } - } - } - - // Update status - rule.Status = v1alpha1.FirewallRuleStatus{State: v1alpha1.FirewallRuleStateNone, LastTransitionTime: metav1.Now()} - log.V(1).Info("Updating FirewallRule", "state", rule.Status.State) - if err := r.Status().Update(ctx, rule); err != nil { - log.Error(err, "Failed to update FirewallRule state", "firewallRule", rule.Name) - return ctrl.Result{}, err - } - r.frLastTransitionTime[rule.Name] = rule.Status.LastTransitionTime - - return ctrl.Result{RequeueAfter: time.Second * 5}, nil -} - // SetupWithManager sets up the controller with the Manager. func (r *FirewallRuleReconciler) SetupWithManager(mgr ctrl.Manager) error { // Index FirewallRule NodeName to list FirewallRules by node. @@ -535,99 +158,3 @@ func (r *FirewallRuleReconciler) SetupWithManager(mgr ctrl.Manager) error { For(&v1alpha1.FirewallRule{}). Complete(r) } - -// encodeCreateFirewallRuleGroupRequest converts an api FirewallRule slice to a CreateFirewallRuleGroupRequest slice. -func encodeCreateFirewallRuleGroupRequest(name, description string, data []v1alpha1.FirewallRule) provider.CreateFirewallRuleGroupRequest { - return provider.CreateFirewallRuleGroupRequest{ - Name: name, - Description: description, - FirewallRules: encodeFirewallRuleSpecs(data), - } -} - -// encodeCreateFirewallRuleRequest converts an api FirewallRule to a CreateFirewallRuleRequest. -func encodeCreateFirewallRuleRequest(data *v1alpha1.FirewallRule) provider.CreateFirewallRuleRequest { - return provider.CreateFirewallRuleRequest{ - FirewallRuleSpec: encodeFirewallRuleSpec(data), - } -} - -// encodeUpdateFirewallRuleGroupRequest converts an api FirewallRule slice to a UpdateFirewallRuleGroupRequest slice. -func encodeUpdateFirewallRuleGroupRequest(id string, data []v1alpha1.FirewallRule) provider.UpdateFirewallRuleGroupRequest { - return provider.UpdateFirewallRuleGroupRequest{ - FirewallRuleGroupID: id, - FirewallRules: encodeFirewallRuleSpecs(data), - } -} - -// encodeUpdateFirewallRuleRequest converts an api FirewallRule to a UpdateFirewallRuleRequest. -func encodeUpdateFirewallRuleRequest(id string, data *v1alpha1.FirewallRule) provider.UpdateFirewallRuleRequest { - return provider.UpdateFirewallRuleRequest{ - FirewallRuleID: id, - FirewallRuleSpec: encodeFirewallRuleSpec(data), - } -} - -// encodeFirewallRuleSpecs converts an api FirewallRule slice to a FirewallRuleSpec slice. -func encodeFirewallRuleSpecs(data []v1alpha1.FirewallRule) []provider.FirewallRuleSpec { - if data == nil { - return make([]provider.FirewallRuleSpec, 0) - } - - res := make([]provider.FirewallRuleSpec, len(data)) - for i, e := range data { - res[i] = encodeFirewallRuleSpec(&e) - } - return res -} - -// encodeFirewallRuleSpec converts an api FirewallRule to a FirewallRuleSpec. -func encodeFirewallRuleSpec(data *v1alpha1.FirewallRule) provider.FirewallRuleSpec { - return provider.FirewallRuleSpec{ - Name: data.Name, - Description: data.Spec.Description, - Direction: encodeDirection(data.Spec.Direction), - IPPermission: &provider.IPPermission{ - FromPort: data.Spec.FromPort, - Protocol: data.Spec.Protocol, - IPRanges: encodeIPRanges(data.Spec.IPRanges), - ToPort: data.Spec.ToPort, - }, - } -} - -// encodeIPRange converts an api IPRange to an IPRange. -func encodeIPRange(data *v1alpha1.IPRange) *provider.IPRange { - if data == nil { - return nil - } - - return &provider.IPRange{ - CIDR: data.CIDR, - Description: data.Description, - } -} - -// encodeIPRange converts an api IPRange slice to an IPRange slice. -func encodeIPRanges(data []*v1alpha1.IPRange) []*provider.IPRange { - if data == nil { - return make([]*provider.IPRange, 0) - } - - res := make([]*provider.IPRange, len(data)) - for i, e := range data { - res[i] = encodeIPRange(e) - } - return res -} - -// encodeDirection converts an api Direction to a Direction. -func encodeDirection(data v1alpha1.Direction) provider.Direction { - switch data { - case v1alpha1.DirectionEgress: - return provider.DirectionEgress - case v1alpha1.DirectionIngress: - return provider.DirectionIngress - } - return provider.Direction("") -} diff --git a/internal/controller/utils.go b/internal/controller/utils.go index fd0f660..2916f90 100644 --- a/internal/controller/utils.go +++ b/internal/controller/utils.go @@ -17,9 +17,6 @@ limitations under the License. package controller import ( - "math/rand" - "time" - corev1 "k8s.io/api/core/v1" "github.com/quortex/kubestatic/api/v1alpha1" @@ -62,13 +59,13 @@ func getMostReferencedIP(pods []corev1.Pod, eips []v1alpha1.ExternalIP) (ip *v1a return } -const charset = "abcdefghijklmnopqrstuvwxyz" +// const charset = "abcdefghijklmnopqrstuvwxyz" -func randomString(length int) string { - seededRand := rand.New(rand.NewSource(time.Now().UnixNano())) - b := make([]byte, length) - for i := range b { - b[i] = charset[seededRand.Intn(len(charset))] - } - return string(b) -} +// func randomString(length int) string { +// seededRand := rand.New(rand.NewSource(time.Now().UnixNano())) +// b := make([]byte, length) +// for i := range b { +// b[i] = charset[seededRand.Intn(len(charset))] +// } +// return string(b) +// } diff --git a/internal/provider/aws/converter/security_group_decoder.go b/internal/provider/aws/converter/security_group_decoder.go index d9a8483..8a1267a 100644 --- a/internal/provider/aws/converter/security_group_decoder.go +++ b/internal/provider/aws/converter/security_group_decoder.go @@ -2,6 +2,7 @@ package converter import ( + "github.com/aws/aws-sdk-go-v2/service/ec2/types" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" @@ -63,6 +64,16 @@ func DecodeIpPermission(data *ec2.IpPermission) *provider.IPPermission { } } +// DecodeIpPermission converts an ec2 IpPermission to an IPPermission. +func DecodeNewIpPermission(data types.IpPermission) *provider.IPPermission { + return &provider.IPPermission{ + FromPort: int64(aws.Int32Value(data.FromPort)), + Protocol: aws.StringValue(data.IpProtocol), + IPRanges: DecodeNewIpRanges(data.IpRanges), + ToPort: aws.Int64(int64(aws.Int32Value(data.ToPort))), + } +} + // DecodeIpPermissions converts an ec2 IpPermission slice to an IPPermission slice. func DecodeIpPermissions(data []*ec2.IpPermission) []*provider.IPPermission { if data == nil { @@ -76,6 +87,15 @@ func DecodeIpPermissions(data []*ec2.IpPermission) []*provider.IPPermission { return res } +// DecodeIpPermissions converts an ec2 IpPermission slice to an IPPermission slice. +func DecodeNewIpPermissions(data []types.IpPermission) []*provider.IPPermission { + res := make([]*provider.IPPermission, len(data)) + for i, e := range data { + res[i] = DecodeNewIpPermission(e) + } + return res +} + // DecodeIpRange converts an ec2 IpRange to an IPRange. func DecodeIpRange(data *ec2.IpRange) *provider.IPRange { if data == nil { @@ -100,3 +120,20 @@ func DecodeIpRanges(data []*ec2.IpRange) []*provider.IPRange { } return res } + +// DecodeIpRange converts an ec2 IpRange to an IPRange. +func DecodeNewIpRange(data types.IpRange) *provider.IPRange { + return &provider.IPRange{ + CIDR: aws.StringValue(data.CidrIp), + Description: aws.StringValue(data.Description), + } +} + +// DecodeIpRanges converts an ec2 IpRange slice to an IPRange slice. +func DecodeNewIpRanges(data []types.IpRange) []*provider.IPRange { + res := make([]*provider.IPRange, len(data)) + for i, e := range data { + res[i] = DecodeNewIpRange(e) + } + return res +} diff --git a/internal/provider/aws/converter/security_group_encoder.go b/internal/provider/aws/converter/security_group_encoder.go index aaf1e96..7baf426 100644 --- a/internal/provider/aws/converter/security_group_encoder.go +++ b/internal/provider/aws/converter/security_group_encoder.go @@ -4,16 +4,16 @@ package converter import ( "slices" + "github.com/aws/aws-sdk-go-v2/service/ec2/types" "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/service/ec2" "k8s.io/utils/ptr" "github.com/quortex/kubestatic/internal/provider" ) // EncodeIPPermission converts an IPPermission to an ec2 IpPermission. -func EncodeIPPermission(req provider.IPPermission) *ec2.IpPermission { - res := &ec2.IpPermission{ +func EncodeIPPermission(req provider.IPPermission) types.IpPermission { + res := types.IpPermission{ IpProtocol: aws.String(req.Protocol), IpRanges: EncodeIpRanges(req.IPRanges), } @@ -21,28 +21,26 @@ func EncodeIPPermission(req provider.IPPermission) *ec2.IpPermission { // fromport / toport must be specified for the tcp / udp protocol even if they are zero. // On the contrary, they must be omitted for the other protocols if they are zero. if slices.Contains([]string{"udp", "tcp", "UDP", "TCP"}, req.Protocol) { - res.FromPort = aws.Int64(req.FromPort) - res.ToPort = req.ToPort + res.FromPort = aws.Int32(int32(req.FromPort)) + if req.ToPort != nil { + res.ToPort = aws.Int32(int32(*req.ToPort)) + } if res.ToPort == nil { - res.ToPort = aws.Int64(req.FromPort) + res.ToPort = aws.Int32(int32(req.FromPort)) } } else { if req.FromPort != 0 { - res.FromPort = ptr.To(req.FromPort) + res.FromPort = aws.Int32(int32(req.FromPort)) } - res.ToPort = req.ToPort + res.ToPort = aws.Int32(int32(*req.ToPort)) } return res } // EncodeIpRange converts an IPRange to an ec2 IpRange. -func EncodeIpRange(data *provider.IPRange) *ec2.IpRange { - if data == nil { - return nil - } - - res := &ec2.IpRange{ +func EncodeIpRange(data *provider.IPRange) types.IpRange { + res := types.IpRange{ CidrIp: aws.String(data.CIDR), } if data.Description != "" { @@ -52,12 +50,8 @@ func EncodeIpRange(data *provider.IPRange) *ec2.IpRange { } // EncodeIpRanges converts an IPRange slice to an ec2 IpRange slice. -func EncodeIpRanges(data []*provider.IPRange) []*ec2.IpRange { - if data == nil { - return make([]*ec2.IpRange, 0) - } - - res := make([]*ec2.IpRange, len(data)) +func EncodeIpRanges(data []*provider.IPRange) []types.IpRange { + res := make([]types.IpRange, len(data)) for i, e := range data { res[i] = EncodeIpRange(e) } diff --git a/internal/provider/aws/imds.go b/internal/provider/aws/imds.go new file mode 100644 index 0000000..b80eaa1 --- /dev/null +++ b/internal/provider/aws/imds.go @@ -0,0 +1,109 @@ +package aws + +import ( + "fmt" + "io" + "net/http" + "time" +) + +const ( + // Retrieve instance metadata for AWS EC2 instance + // https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instancedata-data-retrieval.html + instanceMetadataEndpoint = "http://169.254.169.254/latest/meta-data" + + // IMDSv2 token related constants + tokenEndpoint = "http://169.254.169.254/latest/api/token" + tokenTTLHeader = "X-aws-ec2-metadata-token-ttl-seconds" + tokenRequestHeader = "X-aws-ec2-metadata-token" +) + +// The VPC identifier +// Automatically retrieved with GetVPCID function. +// For run outside of the cluster, can be set through linker flag, e.g. +// go build -ldflags "-X github.com/quortex/kubestatic/internalprovider/aws.vpcID=$VPC_ID" -a -o manager main.go +var vpcID string + +func init() { + // Get vpc ID from the running instance + id, err := retrieveVPCID() + if err != nil { + panic(err) + } + vpcID = id +} + +// retrieveVPCID retrieves the VPC ID of the instance. +// It first checks if the VPC ID is already cached. If not, it attempts to get an IMDSv2 token. +// If the token retrieval fails, it falls back to IMDSv1. It then retrieves the MAC address +// and uses it to get the VPC ID. +func retrieveVPCID() (string, error) { + if vpcID != "" { + return vpcID, nil + } + + client := http.Client{Timeout: 3 * time.Second} + + token, err := getV2Token(client) + if err != nil { + fmt.Printf("failed getting IMDSv2 token falling back to IMDSv1 : %s", err) + } + + mac, err := retrieveInstanceMetadata(client, "/mac", token) + if err != nil { + return "", err + } + + body, err := retrieveInstanceMetadata(client, "/network/interfaces/macs/"+mac+"/vpc-id", token) + if err != nil { + return "", err + } + + return body, nil +} + +// getV2Token retrieves an IMDSv2 token using the provided HTTP client. +// It sends a PUT request to the token endpoint and returns the token if successful. +func getV2Token(client http.Client) (string, error) { + req, err := http.NewRequest(http.MethodPut, tokenEndpoint, nil) + if err != nil { + return "", err + } + req.Header.Set(tokenTTLHeader, "21600") + res, err := client.Do(req) + if err != nil { + return "", err + } + defer func() { _ = res.Body.Close() }() + + token, err := io.ReadAll(res.Body) + if err != nil { + return "", err + } + + return string(token), nil +} + +// retrieveInstanceMetadata retrieves instance metadata from the specified context path using the provided HTTP client and token. +// It sends a GET request to the instance metadata endpoint and returns the response body if successful. +func retrieveInstanceMetadata(client http.Client, contextPath string, token string) (string, error) { + req, err := http.NewRequest(http.MethodGet, instanceMetadataEndpoint+contextPath, nil) + if err != nil { + return "", err + } + + if token != "" { + req.Header.Set(tokenRequestHeader, token) + } + res, err := client.Do(req) + if err != nil { + return "", err + } + + defer func() { _ = res.Body.Close() }() + body, err := io.ReadAll(res.Body) + if err != nil { + return "", err + } + return string(body), nil +} diff --git a/internal/provider/aws/provider.go b/internal/provider/aws/provider.go index 7e32cd6..5909bb5 100644 --- a/internal/provider/aws/provider.go +++ b/internal/provider/aws/provider.go @@ -4,175 +4,239 @@ package aws import ( "context" "fmt" - "io" - "net/http" + "math/rand" "path" + "slices" "time" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/ec2" + "github.com/aws/aws-sdk-go-v2/service/ec2/types" "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/ec2" corev1 "k8s.io/api/core/v1" + "github.com/quortex/kubestatic/api/v1alpha1" "github.com/quortex/kubestatic/internal/provider" "github.com/quortex/kubestatic/internal/provider/aws/converter" ) +type TagKey string + const ( - // Retrieve instance metadata for AWS EC2 instance - // https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instancedata-data-retrieval.html - instanceMetadataEndpoint = "http://169.254.169.254/latest/meta-data" - - // IMDSv2 token related constants - tokenEndpoint = "http://169.254.169.254/latest/api/token" - tokenTTLHeader = "X-aws-ec2-metadata-token-ttl-seconds" - tokenRequestHeader = "X-aws-ec2-metadata-token" + TagKeyDomain = "kubestatic.quortex.io" + TagKeyManaged TagKey = TagKeyDomain + "/managed" + TagKeyInstanceID TagKey = TagKeyDomain + "/instance-id" + TagKeyExternalIPName TagKey = TagKeyDomain + "/external-ip-name" ) -// The VPC identifier -// Automatically retrieved with GetVPCID function. -// For run outside of the cluster, can be set through linker flag, e.g. -// go build -ldflags "-X github.com/quortex/kubestatic/internalprovider/aws.vpcID=$VPC_ID" -a -o manager main.go -var vpcID string - type awsProvider struct { - ec2 *ec2.EC2 + ec2 *ec2.Client } // NewProvider instantiate a Provider implementation for AWS func NewProvider() (provider.Provider, error) { - // By default NewSession loads credentials from the shared credentials file (~/.aws/credentials) - // - // The Session will attempt to load configuration and credentials from the environment, - // configuration files, and other credential sources. The order configuration is loaded in is: - // * Environment Variables - // * Shared Credentials file - // * Shared Configuration file (if SharedConfig is enabled) - // * EC2 Instance Metadata (credentials only) - session, err := session.NewSession() - if err != nil { - return nil, err - } - - // Get vpc ID from the running instance - id, err := retrieveVPCID() + // Load the Shared AWS Configuration (~/.aws/config) + cfg, err := config.LoadDefaultConfig(context.TODO()) if err != nil { - return nil, err + panic(err) } - vpcID = id return &awsProvider{ - ec2: ec2.New(session), + ec2: ec2.NewFromConfig(cfg), }, nil } -func getV2Token(client http.Client) (string, error) { - req, err := http.NewRequest(http.MethodPut, tokenEndpoint, nil) - if err != nil { - return "", err - } - req.Header.Set(tokenTTLHeader, "21600") - res, err := client.Do(req) - if err != nil { - return "", err - } - defer func() { _ = res.Body.Close() }() +func (p *awsProvider) GetInstanceID(node corev1.Node) string { + return path.Base(node.Spec.ProviderID) +} - token, err := io.ReadAll(res.Body) +func (p *awsProvider) deleteSecurityGroup(ctx context.Context, securityGroupID string) error { + _, err := p.ec2.DeleteSecurityGroup(ctx, &ec2.DeleteSecurityGroupInput{ + GroupId: aws.String(securityGroupID), + }) if err != nil { - return "", err + return converter.DecodeEC2Error("failed to delete security group", err) } - return string(token), nil + return nil } -func retrieveInstanceMetadata(client http.Client, contextPath string, token string) (string, error) { - req, err := http.NewRequest(http.MethodGet, instanceMetadataEndpoint+contextPath, nil) - if err != nil { - return "", err - } - - if token != "" { - req.Header.Set(tokenRequestHeader, token) - } - res, err := client.Do(req) +func (p *awsProvider) authorizeSecurityGroupIngress(ctx context.Context, firewallRuleID string, req provider.IPPermission) error { + _, err := p.ec2.AuthorizeSecurityGroupIngress(ctx, &ec2.AuthorizeSecurityGroupIngressInput{ + GroupId: &firewallRuleID, + IpPermissions: []types.IpPermission{ + converter.EncodeIPPermission(req), + }, + }) if err != nil { - return "", err + return converter.DecodeEC2Error("failed to authorize security group ingress permission", err) } - defer func() { _ = res.Body.Close() }() - body, err := io.ReadAll(res.Body) - if err != nil { - return "", err - } - return string(body), nil + return nil } -func retrieveVPCID() (string, error) { - if vpcID != "" { - return vpcID, nil - } - - client := http.Client{Timeout: 3 * time.Second} - - token, err := getV2Token(client) +func (p *awsProvider) revokeSecurityGroupIngress(ctx context.Context, firewallRuleID string, req provider.IPPermission) error { + _, err := p.ec2.RevokeSecurityGroupIngress(ctx, &ec2.RevokeSecurityGroupIngressInput{ + GroupId: aws.String(firewallRuleID), + IpPermissions: []types.IpPermission{ + converter.EncodeIPPermission(req), + }, + }) if err != nil { - fmt.Printf("failed getting IMDSv2 token falling back to IMDSv1 : %s", err) + return converter.DecodeEC2Error("failed to revoke security group ingress permission", err) } - mac, err := retrieveInstanceMetadata(client, "/mac", token) - if err != nil { - return "", err - } + return nil +} - body, err := retrieveInstanceMetadata(client, "/network/interfaces/macs/"+mac+"/vpc-id", token) +func (p *awsProvider) authorizeSecurityGroupEgress(ctx context.Context, firewallRuleID string, req provider.IPPermission) error { + _, err := p.ec2.AuthorizeSecurityGroupEgress(ctx, &ec2.AuthorizeSecurityGroupEgressInput{ + GroupId: aws.String(firewallRuleID), + IpPermissions: []types.IpPermission{ + converter.EncodeIPPermission(req), + }, + }) if err != nil { - return "", err + return converter.DecodeEC2Error("failed to authorize security group egress permission", err) } - return body, nil + return nil } -func (p *awsProvider) GetInstanceID(node corev1.Node) string { - return path.Base(node.Spec.ProviderID) -} +func (p *awsProvider) revokeSecurityGroupEgress(ctx context.Context, firewallRuleID string, req provider.IPPermission) error { + _, err := p.ec2.RevokeSecurityGroupEgress(ctx, &ec2.RevokeSecurityGroupEgressInput{ + GroupId: aws.String(firewallRuleID), + IpPermissions: []types.IpPermission{ + converter.EncodeIPPermission(req), + }, + }) + if err != nil { + return converter.DecodeEC2Error("failed to revoke security group egress permission", err) + } -// Firewall rule groups are supported by AWS (EC2 SecurityGroups). -func (p *awsProvider) HasGroupedFirewallRules() bool { - return true + return nil } -func (p *awsProvider) GetInstance(ctx context.Context, instanceID string) (*provider.Instance, error) { - res, err := p.ec2.DescribeInstances(&ec2.DescribeInstancesInput{ - InstanceIds: aws.StringSlice([]string{instanceID}), +func (p *awsProvider) getInstance(ctx context.Context, instanceID string) (*types.Instance, error) { + res, err := p.ec2.DescribeInstances(ctx, &ec2.DescribeInstancesInput{ + InstanceIds: []string{instanceID}, }) if err != nil { return nil, converter.DecodeEC2Error("failed to get instance", err) } - if len(res.Reservations) == 0 || len(res.Reservations[0].Instances) == 0 { return nil, &provider.Error{ Code: provider.NotFoundError, Msg: fmt.Sprintf("failed to get instance: instance with instance-id %s not found", instanceID), } } - - return converter.DecodeInstance(res.Reservations[0].Instances[0]), nil + return &res.Reservations[0].Instances[0], nil } -func (p *awsProvider) GetAddress(ctx context.Context, addressID string) (*provider.Address, error) { - res, err := p.ec2.DescribeAddresses(&ec2.DescribeAddressesInput{ - Filters: []*ec2.Filter{ +func (p *awsProvider) getNetworkInterfaces(ctx context.Context, securityGroupID string) ([]types.NetworkInterface, error) { + res, err := p.ec2.DescribeNetworkInterfaces(ctx, &ec2.DescribeNetworkInterfacesInput{ + Filters: []types.Filter{ { - Name: aws.String("domain"), - Values: aws.StringSlice([]string{"vpc"}), + Name: aws.String("group-id"), + Values: []string{securityGroupID}, }, + }, + }) + if err != nil { + return nil, converter.DecodeEC2Error("failed to list network interfaces", err) + } + return res.NetworkInterfaces, nil +} + +func (p *awsProvider) getSecurityGroup(ctx context.Context, vpcID, instanceID, securityGroupID string) (*types.SecurityGroup, error) { + filters := []types.Filter{ + { + Name: aws.String(fmt.Sprintf("tag:%s", TagKeyManaged)), + Values: []string{"true"}, + }, + } + if vpcID != "" { + filters = append(filters, types.Filter{ + Name: aws.String("vpc-id"), + Values: []string{vpcID}, + }) + } + if instanceID != "" { + filters = append(filters, types.Filter{ + Name: aws.String(fmt.Sprintf("tag:%s", TagKeyInstanceID)), + Values: []string{instanceID}, + }) + } + if securityGroupID != "" { + filters = append(filters, types.Filter{ + Name: aws.String("group-id"), + Values: []string{securityGroupID}, + }) + } + + res, err := p.ec2.DescribeSecurityGroups(ctx, &ec2.DescribeSecurityGroupsInput{Filters: filters}) + if err != nil { + return nil, converter.DecodeEC2Error("failed to list security groups", err) + } + + if len(res.SecurityGroups) == 0 { + return nil, &provider.Error{ + Code: provider.NotFoundError, + Msg: fmt.Sprintf("failed to get security group: security group with instance-id %s not found", instanceID), + } + } + return &res.SecurityGroups[0], nil +} + +func (p *awsProvider) createSecurityGroup(ctx context.Context, vpcID, instanceID string) (string, error) { + res, err := p.ec2.CreateSecurityGroup(ctx, &ec2.CreateSecurityGroupInput{ + GroupName: aws.String(fmt.Sprintf("kubestatic-%s", randomString(10))), + Description: aws.String(fmt.Sprintf("Kubestatic managed group for instance %s", instanceID)), + VpcId: aws.String(vpcID), + TagSpecifications: []types.TagSpecification{ { - Name: aws.String("allocation-id"), - Values: aws.StringSlice([]string{addressID}), + ResourceType: types.ResourceTypeSecurityGroup, + Tags: []types.Tag{ + { + Key: aws.String(string(TagKeyManaged)), + Value: aws.String("true"), + }, + { + Key: aws.String(string(TagKeyInstanceID)), + Value: aws.String(instanceID), + }, + }, }, }, }) + if err != nil { + return "", converter.DecodeEC2Error("failed to create security group", err) + } + + return aws.StringValue(res.GroupId), nil +} + +func (p *awsProvider) getAddress(ctx context.Context, externalIPName, addressID string) (*types.Address, error) { + filters := []types.Filter{ + { + Name: aws.String(fmt.Sprintf("tag:%s", TagKeyManaged)), + Values: []string{"true"}, + }, + { + Name: aws.String(fmt.Sprintf("tag:%s", TagKeyExternalIPName)), + Values: []string{externalIPName}, + }, + } + if addressID != "" { + filters = append(filters, types.Filter{ + Name: aws.String("allocation-id"), + Values: []string{addressID}, + }) + } + + res, err := p.ec2.DescribeAddresses(ctx, &ec2.DescribeAddressesInput{ + Filters: filters, + }) if err != nil { return nil, converter.DecodeEC2Error("failed to get address", err) } @@ -180,266 +244,421 @@ func (p *awsProvider) GetAddress(ctx context.Context, addressID string) (*provid if len(res.Addresses) == 0 { return nil, &provider.Error{ Code: provider.NotFoundError, - Msg: fmt.Sprintf("failed to get address: address with allocation-id %s not found", addressID), + Msg: fmt.Sprintf("failed to get address: address for ExternalIP %s not found", externalIPName), } } - return converter.DecodeAddress(res.Addresses[0]), nil + return &res.Addresses[0], nil } -func (p *awsProvider) CreateAddress(ctx context.Context) (*provider.Address, error) { - res, err := p.ec2.AllocateAddress(&ec2.AllocateAddressInput{ - Domain: aws.String("vpc"), +func (p *awsProvider) createAddress(ctx context.Context, externalIPName, instanceID string) (string, error) { + res, err := p.ec2.AllocateAddress(ctx, &ec2.AllocateAddressInput{ + Domain: "vpc", + TagSpecifications: []types.TagSpecification{ + { + ResourceType: types.ResourceTypeElasticIp, + Tags: []types.Tag{ + { + Key: aws.String(string(TagKeyManaged)), + Value: aws.String("true"), + }, + { + Key: aws.String(string(TagKeyExternalIPName)), + Value: aws.String(externalIPName), + }, + { + Key: aws.String(string(TagKeyInstanceID)), + Value: aws.String(instanceID), + }, + }, + }, + }, }) if err != nil { - return nil, converter.DecodeEC2Error("failed to create address", err) + return "", converter.DecodeEC2Error("failed to create address", err) } - return p.GetAddress(ctx, aws.StringValue(res.AllocationId)) + return aws.StringValue(res.AllocationId), nil } -func (p *awsProvider) DeleteAddress(ctx context.Context, addressID string) error { - _, err := p.ec2.ReleaseAddress(&ec2.ReleaseAddressInput{ - AllocationId: aws.String(addressID), +func (p *awsProvider) associateAddress(ctx context.Context, addressID, networkInterfaceID string) error { + _, err := p.ec2.AssociateAddress(ctx, &ec2.AssociateAddressInput{ + AllocationId: &addressID, + NetworkInterfaceId: &networkInterfaceID, }) if err != nil { - return converter.DecodeEC2Error("failed to delete address", err) + return converter.DecodeEC2Error("failed to associate address", err) } return nil } -func (p *awsProvider) AssociateAddress(ctx context.Context, req provider.AssociateAddressRequest) error { - _, err := p.ec2.AssociateAddress(&ec2.AssociateAddressInput{ - AllocationId: aws.String(req.AddressID), - NetworkInterfaceId: aws.String(req.NetworkInterfaceID), +func (p *awsProvider) disassociateAddress(ctx context.Context, associationID string) error { + _, err := p.ec2.DisassociateAddress(ctx, &ec2.DisassociateAddressInput{ + AssociationId: &associationID, }) if err != nil { - return converter.DecodeEC2Error("failed to associate address", err) + return converter.DecodeEC2Error("failed to disassociate address", err) } return nil } -func (p *awsProvider) DisassociateAddress(ctx context.Context, req provider.DisassociateAddressRequest) error { - _, err := p.ec2.DisassociateAddress(&ec2.DisassociateAddressInput{ - AssociationId: aws.String(req.AssociationID), +func (p *awsProvider) deleteAddress(ctx context.Context, addressID string) error { + _, err := p.ec2.ReleaseAddress(ctx, &ec2.ReleaseAddressInput{ + AllocationId: &addressID, }) if err != nil { - return converter.DecodeEC2Error("failed to disassociate address", err) + return converter.DecodeEC2Error("failed to delete address", err) } return nil } -func (p *awsProvider) getSecurityGroup(_ context.Context, firewallRuleID string) (*ec2.SecurityGroup, error) { - res, err := p.ec2.DescribeSecurityGroups(&ec2.DescribeSecurityGroupsInput{ - GroupIds: aws.StringSlice([]string{firewallRuleID}), - }) +func (p *awsProvider) ReconcileFirewallRules(ctx context.Context, instanceID string, firewallRules []v1alpha1.FirewallRule) error { + // Get the instance + instance, err := p.getInstance(ctx, instanceID) if err != nil { - return nil, converter.DecodeEC2Error("failed to get security group", err) + return err } - if len(res.SecurityGroups) == 0 { - return nil, &provider.Error{ - Code: provider.NotFoundError, - Msg: fmt.Sprintf("failed to get security group: security group with group-id %s not found", firewallRuleID), - } + // Get the security group associated with the instance + securityGroup, err := p.getSecurityGroup(ctx, aws.StringValue(instance.VpcId), instanceID, "") + if err != nil && err.(*provider.Error).Code != provider.NotFoundError { + return err } - return res.SecurityGroups[0], nil -} + if securityGroup == nil { + securityGroupID, err := p.createSecurityGroup(ctx, aws.StringValue(instance.VpcId), instanceID) + if err != nil { + return converter.DecodeEC2Error("failed to create security group", err) + } -func (p *awsProvider) FetchFirewallRule(ctx context.Context, firewallRuleGroupID string) error { - _, err := p.getSecurityGroup(ctx, firewallRuleGroupID) - if err != nil { - return converter.DecodeEC2Error("failed to fetch security group", err) + securityGroup, err = p.getSecurityGroup(ctx, aws.StringValue(instance.VpcId), instanceID, securityGroupID) + if err != nil { + return err + } } - return nil -} + securityGroupID := aws.StringValue(securityGroup.GroupId) -func (p *awsProvider) CreateFirewallRule(ctx context.Context, req provider.CreateFirewallRuleRequest) (string, error) { - panic("unimplemented method for AWS: CreateFirewallRule, use CreateFirewallRuleGroup instead") -} + // Get the first network interface with a public IP address + networkInterface, err := eniWithPublicIP(instance) + if err != nil { + return err + } -func (p *awsProvider) CreateFirewallRuleGroup(ctx context.Context, req provider.CreateFirewallRuleGroupRequest) (string, error) { - res, err := p.ec2.CreateSecurityGroup(&ec2.CreateSecurityGroupInput{ - Description: aws.String(req.Description), - GroupName: aws.String(req.Name), - VpcId: aws.String(vpcID), - }) + // Get all network interfaces associated with the security group + networkInterfaces, err := p.getNetworkInterfaces(ctx, securityGroupID) if err != nil { - return "", converter.DecodeEC2Error("failed to create security group", err) + return converter.DecodeEC2Error("failed to list network interfaces", err) } - return p.UpdateFirewallRuleGroup(ctx, provider.UpdateFirewallRuleGroupRequest{ - FirewallRuleGroupID: *res.GroupId, - FirewallRules: req.FirewallRules, - }) -} + isAssociated := false + for _, ni := range networkInterfaces { + if aws.StringValue(ni.NetworkInterfaceId) == aws.StringValue(networkInterface.NetworkInterfaceId) { + isAssociated = true + continue + } -func (p *awsProvider) UpdateFirewallRule(ctx context.Context, req provider.UpdateFirewallRuleRequest) (*provider.FirewallRule, error) { - panic("unimplemented method for AWS: UpdateFirewallRule, use UpdateFirewallRuleGroup instead") -} + // Disassociate the security group from other network interfaces + groups := []string{} + for _, group := range ni.Groups { + if aws.StringValue(group.GroupId) != securityGroupID { + groups = append(groups, aws.StringValue(group.GroupId)) + } + } + _, err = p.ec2.ModifyNetworkInterfaceAttribute(ctx, &ec2.ModifyNetworkInterfaceAttributeInput{ + NetworkInterfaceId: ni.NetworkInterfaceId, + Groups: groups, + }) + if err != nil { + return converter.DecodeEC2Error("failed to modify network interface attribute", err) + } + } -func (p *awsProvider) UpdateFirewallRuleGroup(ctx context.Context, req provider.UpdateFirewallRuleGroupRequest) (string, error) { - sg, err := p.getSecurityGroup(ctx, req.FirewallRuleGroupID) - if err != nil { - return "", converter.DecodeEC2Error("failed to get security group", err) + if !isAssociated { + // Associate the security group with the network interface + // Disassociate the security group from other network interfaces + groups := []string{} + for _, group := range networkInterface.Groups { + if aws.StringValue(group.GroupId) != securityGroupID { + groups = append(groups, aws.StringValue(group.GroupId)) + } + } + groups = append(groups, securityGroupID) + _, err = p.ec2.ModifyNetworkInterfaceAttribute(ctx, &ec2.ModifyNetworkInterfaceAttributeInput{ + NetworkInterfaceId: networkInterface.NetworkInterfaceId, + Groups: groups, + }) + if err != nil { + return converter.DecodeEC2Error("failed to modify network interface attribute", err) + } } + req := encodeUpdateFirewallRuleGroupRequest(securityGroupID, firewallRules) + // Apply Ingress permissions reconciliation if err := provider.ReconcilePermissions( ctx, - req.FirewallRuleGroupID, + securityGroupID, p.authorizeSecurityGroupIngress, p.revokeSecurityGroupIngress, provider.GetIngressIPPermissions(req.FirewallRules), - converter.DecodeIpPermissions(sg.IpPermissions), + converter.DecodeNewIpPermissions(securityGroup.IpPermissions), ); err != nil { - return "", converter.DecodeEC2Error("failed to apply security group ingress permissions", err) + return converter.DecodeEC2Error("failed to apply security group ingress permissions", err) } // Apply Egress permissions reconciliation if err := provider.ReconcilePermissions( ctx, - req.FirewallRuleGroupID, + securityGroupID, p.authorizeSecurityGroupEgress, p.revokeSecurityGroupEgress, provider.GetEgressIPPermissions(req.FirewallRules), - converter.DecodeIpPermissions(sg.IpPermissionsEgress), + converter.DecodeNewIpPermissions(securityGroup.IpPermissionsEgress), ); err != nil { - return "", converter.DecodeEC2Error("failed to apply security group egress permissions", err) + return converter.DecodeEC2Error("failed to apply security group egress permissions", err) } - return req.FirewallRuleGroupID, nil + return nil } -func (p *awsProvider) DeleteFirewallRule(ctx context.Context, firewallRuleID string) error { - _, err := p.ec2.DeleteSecurityGroup(&ec2.DeleteSecurityGroupInput{ - GroupId: aws.String(firewallRuleID), - }) +func (p *awsProvider) ReconcileFirewallRulesDeletion(ctx context.Context, instanceID string) error { + // Get the instance + instance, err := p.getInstance(ctx, instanceID) if err != nil { - return converter.DecodeEC2Error("failed to delete security group", err) + return err } - return nil -} + // Get the security group associated with the instance + securityGroup, err := p.getSecurityGroup(ctx, aws.StringValue(instance.VpcId), instanceID, "") + if err != nil { + // The security group does not exist, end of reconciliation + if err.(*provider.Error).Code != provider.NotFoundError { + return nil + } + return err + } + securityGroupID := aws.StringValue(securityGroup.GroupId) -func (p *awsProvider) authorizeSecurityGroupIngress(ctx context.Context, firewallRuleID string, req provider.IPPermission) error { - _, err := p.ec2.AuthorizeSecurityGroupIngress(&ec2.AuthorizeSecurityGroupIngressInput{ - GroupId: aws.String(firewallRuleID), - IpPermissions: []*ec2.IpPermission{ - converter.EncodeIPPermission(req), - }, - }) + // Get all network interfaces associated with the security group + networkInterfaces, err := p.getNetworkInterfaces(ctx, securityGroupID) if err != nil { - return converter.DecodeEC2Error("failed to authorize security group ingress permission", err) + return err } - return nil -} + for _, ni := range networkInterfaces { + // Disassociate the security group from all network interfaces + groups := []string{} + for _, group := range ni.Groups { + if aws.StringValue(group.GroupId) != securityGroupID { + groups = append(groups, aws.StringValue(group.GroupId)) + } + } + _, err = p.ec2.ModifyNetworkInterfaceAttribute(ctx, &ec2.ModifyNetworkInterfaceAttributeInput{ + NetworkInterfaceId: ni.NetworkInterfaceId, + Groups: groups, + }) + if err != nil { + return converter.DecodeEC2Error("failed to modify network interface attribute", err) + } + } -func (p *awsProvider) revokeSecurityGroupIngress(ctx context.Context, firewallRuleID string, req provider.IPPermission) error { - _, err := p.ec2.RevokeSecurityGroupIngress(&ec2.RevokeSecurityGroupIngressInput{ - GroupId: aws.String(firewallRuleID), - IpPermissions: []*ec2.IpPermission{ - converter.EncodeIPPermission(req), - }, - }) - if err != nil { - return converter.DecodeEC2Error("failed to revoke security group ingress permission", err) + if err := p.deleteSecurityGroup(ctx, securityGroupID); err != nil { + return err } return nil } -func (p *awsProvider) authorizeSecurityGroupEgress(ctx context.Context, firewallRuleID string, req provider.IPPermission) error { - _, err := p.ec2.AuthorizeSecurityGroupEgress(&ec2.AuthorizeSecurityGroupEgressInput{ - GroupId: aws.String(firewallRuleID), - IpPermissions: []*ec2.IpPermission{ - converter.EncodeIPPermission(req), - }, - }) - if err != nil { - return converter.DecodeEC2Error("failed to authorize security group egress permission", err) +func (p *awsProvider) ReconcileExternalIP( + ctx context.Context, + instanceID string, + externalIP *v1alpha1.ExternalIP, +) (v1alpha1.ExternalIPStatus, error) { + status := externalIP.Status + + // Get the address associated with the instance + address, err := p.getAddress(ctx, externalIP.Name, "") + if err != nil && err.(*provider.Error).Code != provider.NotFoundError { + return status, err } - return nil -} + if address == nil { + addressID, err := p.createAddress(ctx, externalIP.Name, instanceID) + if err != nil { + return status, err + } -func (p *awsProvider) revokeSecurityGroupEgress(ctx context.Context, firewallRuleID string, req provider.IPPermission) error { - _, err := p.ec2.RevokeSecurityGroupEgress(&ec2.RevokeSecurityGroupEgressInput{ - GroupId: aws.String(firewallRuleID), - IpPermissions: []*ec2.IpPermission{ - converter.EncodeIPPermission(req), - }, - }) + address, err = p.getAddress(ctx, externalIP.Name, addressID) + if err != nil { + return status, err + } + } + status.State = v1alpha1.ExternalIPStateReserved + status.AddressID = address.AllocationId + status.PublicIPAddress = address.PublicIp + + if instanceID == "" { + if address.AssociationId == nil { + return status, nil + } + + // Disassociate the address from the current network interface + if err := p.disassociateAddress(ctx, *address.AssociationId); err != nil { + return status, err + } + status.InstanceID = nil + status.State = v1alpha1.ExternalIPStateReserved + return status, nil + } + + // Get the instance + instance, err := p.getInstance(ctx, instanceID) if err != nil { - return converter.DecodeEC2Error("failed to revoke security group egress permission", err) + return status, err } - return nil + // Get the first network interface with a public IP address + networkInterface, err := eniWithPublicIP(instance) + if err != nil { + return status, err + } + + if address.AssociationId != nil { + // Address is already associated with the instance + if *address.AssociationId == *networkInterface.NetworkInterfaceId { + return status, nil + } + + // Disassociate the address from the current network interface + if err := p.disassociateAddress(ctx, *address.AssociationId); err != nil { + return status, err + } + status.InstanceID = nil + status.State = v1alpha1.ExternalIPStateReserved + } + + // Associate the address with the network interface + if err := p.associateAddress(ctx, aws.StringValue(address.AllocationId), *networkInterface.NetworkInterfaceId); err != nil { + return status, err + } + status.InstanceID = &instanceID + status.State = v1alpha1.ExternalIPStateAssociated + + return status, nil } -func (p *awsProvider) AssociateFirewallRule(ctx context.Context, req provider.AssociateFirewallRuleRequest) error { - res, err := p.ec2.DescribeNetworkInterfaces(&ec2.DescribeNetworkInterfacesInput{ - NetworkInterfaceIds: aws.StringSlice([]string{req.NetworkInterfaceID}), - }) +func (p *awsProvider) ReconcileExternalIPDeletion(ctx context.Context, externalIP *v1alpha1.ExternalIP) error { + // Get the address associated with the instance + address, err := p.getAddress(ctx, externalIP.Name, "") if err != nil { + // The address does not exist, end of reconciliation + if err.(*provider.Error).Code != provider.NotFoundError { + return nil + } return err } - if len(res.NetworkInterfaces) == 0 { - return &provider.Error{ - Code: provider.NotFoundError, - Msg: fmt.Sprintf("failed to associate security group: network interface with id %s not found", req.NetworkInterfaceID), + if address.AssociationId != nil { + // Disassociate the address from the network interface + if err := p.disassociateAddress(ctx, *address.AssociationId); err != nil { + return err } } - networkInterface := res.NetworkInterfaces[0] - groups := []*string{} - for _, e := range networkInterface.Groups { - if req.FirewallRuleID != *e.GroupId { - groups = append(groups, e.GroupId) - } + return p.deleteAddress(ctx, aws.StringValue(address.AllocationId)) +} + +// encodeFirewallRuleSpecs converts an api FirewallRule slice to a FirewallRuleSpec slice. +func encodeFirewallRuleSpecs(data []v1alpha1.FirewallRule) []provider.FirewallRuleSpec { + if data == nil { + return make([]provider.FirewallRuleSpec, 0) } - groups = append(groups, aws.String(req.FirewallRuleID)) - _, err = p.ec2.ModifyNetworkInterfaceAttribute(&ec2.ModifyNetworkInterfaceAttributeInput{ - Groups: groups, - NetworkInterfaceId: aws.String(req.NetworkInterfaceID), - }) + res := make([]provider.FirewallRuleSpec, len(data)) + for i, e := range data { + res[i] = encodeFirewallRuleSpec(&e) + } + return res +} - return err +// encodeFirewallRuleSpec converts an api FirewallRule to a FirewallRuleSpec. +func encodeFirewallRuleSpec(data *v1alpha1.FirewallRule) provider.FirewallRuleSpec { + return provider.FirewallRuleSpec{ + Name: data.Name, + Description: data.Spec.Description, + Direction: encodeDirection(data.Spec.Direction), + IPPermission: &provider.IPPermission{ + FromPort: data.Spec.FromPort, + Protocol: data.Spec.Protocol, + IPRanges: encodeIPRanges(data.Spec.IPRanges), + ToPort: data.Spec.ToPort, + }, + } } -func (p *awsProvider) DisassociateFirewallRule(ctx context.Context, req provider.AssociateFirewallRuleRequest) error { - res, err := p.ec2.DescribeNetworkInterfaces(&ec2.DescribeNetworkInterfacesInput{ - NetworkInterfaceIds: aws.StringSlice([]string{req.NetworkInterfaceID}), - }) - if err != nil { - return converter.DecodeEC2Error("failed to disassociate security group", err) +// encodeIPRange converts an api IPRange to an IPRange. +func encodeIPRange(data *v1alpha1.IPRange) *provider.IPRange { + if data == nil { + return nil } - if len(res.NetworkInterfaces) == 0 { - return &provider.Error{ - Code: provider.NotFoundError, - Msg: fmt.Sprintf("failed to disassociate security group: network interface with id %s not found", req.NetworkInterfaceID), - } + return &provider.IPRange{ + CIDR: data.CIDR, + Description: data.Description, } +} - networkInterface := res.NetworkInterfaces[0] - groups := []*string{} - for _, e := range networkInterface.Groups { - if req.FirewallRuleID != aws.StringValue(e.GroupId) { - groups = append(groups, e.GroupId) - } +// encodeIPRange converts an api IPRange slice to an IPRange slice. +func encodeIPRanges(data []*v1alpha1.IPRange) []*provider.IPRange { + if data == nil { + return make([]*provider.IPRange, 0) } - _, err = p.ec2.ModifyNetworkInterfaceAttribute(&ec2.ModifyNetworkInterfaceAttributeInput{ - Groups: groups, - NetworkInterfaceId: aws.String(req.NetworkInterfaceID), - }) + res := make([]*provider.IPRange, len(data)) + for i, e := range data { + res[i] = encodeIPRange(e) + } + return res +} + +// encodeDirection converts an api Direction to a Direction. +func encodeDirection(data v1alpha1.Direction) provider.Direction { + switch data { + case v1alpha1.DirectionEgress: + return provider.DirectionEgress + case v1alpha1.DirectionIngress: + return provider.DirectionIngress + } + return provider.Direction("") +} + +const charset = "abcdefghijklmnopqrstuvwxyz" + +func randomString(length int) string { + seededRand := rand.New(rand.NewSource(time.Now().UnixNano())) + b := make([]byte, length) + for i := range b { + b[i] = charset[seededRand.Intn(len(charset))] + } + return string(b) +} - return err +// encodeUpdateFirewallRuleGroupRequest converts an api FirewallRule slice to a UpdateFirewallRuleGroupRequest slice. +func encodeUpdateFirewallRuleGroupRequest(id string, data []v1alpha1.FirewallRule) provider.UpdateFirewallRuleGroupRequest { + return provider.UpdateFirewallRuleGroupRequest{ + FirewallRuleGroupID: id, + FirewallRules: encodeFirewallRuleSpecs(data), + } +} + +func eniWithPublicIP(instance *types.Instance) (*types.InstanceNetworkInterface, error) { + idx := slices.IndexFunc(instance.NetworkInterfaces, func(ni types.InstanceNetworkInterface) bool { + return ni.Association != nil && ni.Association.PublicIp != nil + }) + if idx == -1 { + return nil, fmt.Errorf("no network interface with public IP found for instance %s", aws.StringValue(instance.InstanceId)) + } + return &instance.NetworkInterfaces[idx], nil } diff --git a/internal/provider/provider.go b/internal/provider/provider.go index 3f9d06f..eeda5f5 100644 --- a/internal/provider/provider.go +++ b/internal/provider/provider.go @@ -5,34 +5,20 @@ import ( "context" corev1 "k8s.io/api/core/v1" + + "github.com/quortex/kubestatic/api/v1alpha1" ) // Provider describes a cloud provider type Provider interface { Client GetInstanceID(corev1.Node) string - // HasGroupedFirewallRules describes wether firewall rule groups are - // supported by the provider or not (e.g. AWS SecurityGroups). - HasGroupedFirewallRules() bool } // The necessary methods for a provider client are described here. -// According of the Provider.HasGroupedFirewallRules implementation, -// one of the CreateFirewallRule / CreateFirewallRuleGroup and -// UpdateFirewallRule / UpdateFirewallRuleGroup methods must be implemented. type Client interface { - GetInstance(ctx context.Context, instanceID string) (*Instance, error) - GetAddress(ctx context.Context, addressID string) (*Address, error) - CreateAddress(ctx context.Context) (*Address, error) - DeleteAddress(ctx context.Context, addressID string) error - AssociateAddress(ctx context.Context, req AssociateAddressRequest) error - DisassociateAddress(ctx context.Context, req DisassociateAddressRequest) error - FetchFirewallRule(ctx context.Context, firewallRuleGroupID string) error - CreateFirewallRule(ctx context.Context, req CreateFirewallRuleRequest) (string, error) - CreateFirewallRuleGroup(ctx context.Context, req CreateFirewallRuleGroupRequest) (string, error) - UpdateFirewallRule(ctx context.Context, req UpdateFirewallRuleRequest) (*FirewallRule, error) - UpdateFirewallRuleGroup(ctx context.Context, req UpdateFirewallRuleGroupRequest) (string, error) - DeleteFirewallRule(ctx context.Context, firewallRuleID string) error - AssociateFirewallRule(ctx context.Context, req AssociateFirewallRuleRequest) error - DisassociateFirewallRule(ctx context.Context, req AssociateFirewallRuleRequest) error + ReconcileFirewallRules(ctx context.Context, instanceID string, firewallRules []v1alpha1.FirewallRule) error + ReconcileFirewallRulesDeletion(ctx context.Context, instanceID string) error + ReconcileExternalIP(ctx context.Context, instanceID string, externalIP *v1alpha1.ExternalIP) (v1alpha1.ExternalIPStatus, error) + ReconcileExternalIPDeletion(ctx context.Context, externalIP *v1alpha1.ExternalIP) error } diff --git a/internal/provider/utils.go b/internal/provider/utils.go index 2295d41..bdd228c 100644 --- a/internal/provider/utils.go +++ b/internal/provider/utils.go @@ -8,7 +8,7 @@ import ( ) // ReconcilePermissions perform create / delete on given permissions -// to to reach the desired state of firewall rules. +// to reach the desired state of firewall rules. func ReconcilePermissions( ctx context.Context, firewallRuleID string,