diff --git a/test/e2e/common/common.go b/test/e2e/common/common.go index 4bb1336b55..9772320685 100644 --- a/test/e2e/common/common.go +++ b/test/e2e/common/common.go @@ -26,6 +26,7 @@ const ( var ( AzureLocations = []string{"eastus2", "northeurope", "uksouth", "centralindia", "westus2"} + Architectures = []string{"amd64", "arm64"} CreateInfra = flag.Bool("create-infra", true, "create a Resource group, vNET and AKS cluster for testing") DeleteInfra = flag.Bool("delete-infra", true, "delete a Resource group, vNET and AKS cluster for testing") ) diff --git a/test/e2e/framework/kubernetes/create-agnhost-statefulset.go b/test/e2e/framework/kubernetes/create-agnhost-statefulset.go index 6783d7aa66..f9689d9f1d 100644 --- a/test/e2e/framework/kubernetes/create-agnhost-statefulset.go +++ b/test/e2e/framework/kubernetes/create-agnhost-statefulset.go @@ -17,8 +17,10 @@ import ( var ErrLabelMissingFromPod = fmt.Errorf("label missing from pod") const ( - AgnhostHTTPPort = 80 - AgnhostReplicas = 1 + AgnhostHTTPPort = 80 + AgnhostReplicas = 1 + AgnhostArchAmd64 = "amd64" + AgnhostArchArm64 = "arm64" ) type CreateAgnhostStatefulSet struct { @@ -26,6 +28,7 @@ type CreateAgnhostStatefulSet struct { AgnhostNamespace string ScheduleOnSameNode bool KubeConfigFilePath string + AgnhostArch string } func (c *CreateAgnhostStatefulSet) Run() error { @@ -42,14 +45,19 @@ func (c *CreateAgnhostStatefulSet) Run() error { ctx, cancel := context.WithTimeout(context.Background(), defaultTimeoutSeconds*time.Second) defer cancel() - agnhostStatefulest := c.getAgnhostDeployment() + // set default arch to amd64 + if c.AgnhostArch == "" { + c.AgnhostArch = AgnhostArchAmd64 + } + + agnhostStatefulSet := c.getAgnhostDeployment(c.AgnhostArch) - err = CreateResource(ctx, agnhostStatefulest, clientset) + err = CreateResource(ctx, agnhostStatefulSet, clientset) if err != nil { return fmt.Errorf("error agnhost component: %w", err) } - selector, exists := agnhostStatefulest.Spec.Selector.MatchLabels["app"] + selector, exists := agnhostStatefulSet.Spec.Selector.MatchLabels["app"] if !exists { return fmt.Errorf("missing label \"app=%s\" from agnhost statefulset: %w", c.AgnhostName, ErrLabelMissingFromPod) } @@ -71,7 +79,7 @@ func (c *CreateAgnhostStatefulSet) Stop() error { return nil } -func (c *CreateAgnhostStatefulSet) getAgnhostDeployment() *appsv1.StatefulSet { +func (c *CreateAgnhostStatefulSet) getAgnhostDeployment(arch string) *appsv1.StatefulSet { reps := int32(AgnhostReplicas) var affinity *v1.Affinity @@ -90,7 +98,6 @@ func (c *CreateAgnhostStatefulSet) getAgnhostDeployment() *appsv1.StatefulSet { }, }, } - } else { affinity = &v1.Affinity{ PodAntiAffinity: &v1.PodAntiAffinity{ @@ -141,12 +148,13 @@ func (c *CreateAgnhostStatefulSet) getAgnhostDeployment() *appsv1.StatefulSet { Spec: v1.PodSpec{ Affinity: affinity, NodeSelector: map[string]string{ - "kubernetes.io/os": "linux", + "kubernetes.io/os": "linux", + "kubernetes.io/arch": arch, }, Containers: []v1.Container{ { Name: c.AgnhostName, - Image: "acnpublic.azurecr.io/agnhost:2.40", + Image: "registry.k8s.io/e2e-test-images/agnhost:2.40", Resources: v1.ResourceRequirements{ Requests: v1.ResourceList{ "memory": resource.MustParse("20Mi"), diff --git a/test/e2e/framework/kubernetes/create-network-policy.go b/test/e2e/framework/kubernetes/create-network-policy.go index 6c0cf41709..996e232215 100644 --- a/test/e2e/framework/kubernetes/create-network-policy.go +++ b/test/e2e/framework/kubernetes/create-network-policy.go @@ -36,8 +36,8 @@ func (c *CreateDenyAllNetworkPolicy) Run() error { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - agnhostStatefulest := getNetworkPolicy(c.NetworkPolicyNamespace, c.DenyAllLabelSelector) - err = CreateResource(ctx, agnhostStatefulest, clientset) + agnhostStatefulSet := getNetworkPolicy(c.NetworkPolicyNamespace, c.DenyAllLabelSelector) + err = CreateResource(ctx, agnhostStatefulSet, clientset) if err != nil { return fmt.Errorf("error creating simple deny-all network policy: %w", err) } @@ -96,8 +96,8 @@ func (d *DeleteDenyAllNetworkPolicy) Run() error { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - agnhostStatefulest := getNetworkPolicy(d.NetworkPolicyNamespace, d.DenyAllLabelSelector) - err = DeleteResource(ctx, agnhostStatefulest, clientset) + agnhostStatefulSet := getNetworkPolicy(d.NetworkPolicyNamespace, d.DenyAllLabelSelector) + err = DeleteResource(ctx, agnhostStatefulSet, clientset) if err != nil { return fmt.Errorf("error creating simple deny-all network policy: %w", err) } diff --git a/test/e2e/jobs/jobs.go b/test/e2e/jobs/jobs.go index ddc58b5daf..375f69aa4e 100644 --- a/test/e2e/jobs/jobs.go +++ b/test/e2e/jobs/jobs.go @@ -117,12 +117,6 @@ func InstallAndTestRetinaBasicMetrics(kubeConfigFilePath, chartPath string, test TagEnv: generic.DefaultTagEnv, }, nil) - job.AddScenario(drop.ValidateDropMetric(testPodNamespace)) - - job.AddScenario(tcp.ValidateTCPMetrics(testPodNamespace)) - - job.AddScenario(windows.ValidateWindowsBasicMetric()) - dnsScenarios := []struct { name string req *dns.RequestValidationParams @@ -164,8 +158,16 @@ func InstallAndTestRetinaBasicMetrics(kubeConfigFilePath, chartPath string, test }, } - for _, scenario := range dnsScenarios { - job.AddScenario(dns.ValidateBasicDNSMetrics(scenario.name, scenario.req, scenario.resp, testPodNamespace)) + for _, arch := range common.Architectures { + job.AddScenario(drop.ValidateDropMetric(testPodNamespace, arch)) + job.AddScenario(tcp.ValidateTCPMetrics(testPodNamespace, arch)) + + for _, scenario := range dnsScenarios { + name := scenario.name + " - Arch: " + arch + job.AddScenario(dns.ValidateBasicDNSMetrics(name, scenario.req, scenario.resp, testPodNamespace, arch)) + } + + job.AddScenario(windows.ValidateWindowsBasicMetric()) } job.AddStep(&kubernetes.EnsureStableComponent{ @@ -230,8 +232,11 @@ func UpgradeAndTestRetinaAdvancedMetrics(kubeConfigFilePath, chartPath, valuesFi }, } - for _, scenario := range dnsScenarios { - job.AddScenario(dns.ValidateAdvancedDNSMetrics(scenario.name, scenario.req, scenario.resp, kubeConfigFilePath, testPodNamespace)) + for _, arch := range common.Architectures { + for _, scenario := range dnsScenarios { + name := scenario.name + " - Arch: " + arch + job.AddScenario(dns.ValidateAdvancedDNSMetrics(name, scenario.req, scenario.resp, kubeConfigFilePath, testPodNamespace, arch)) + } } job.AddScenario(latency.ValidateLatencyMetric(testPodNamespace)) @@ -260,55 +265,6 @@ func ValidateHubble(kubeConfigFilePath, chartPath string, testPodNamespace strin job.AddScenario(hubble.ValidateHubbleUIService(kubeConfigFilePath)) - job.AddScenario(drop.ValidateDropMetric(testPodNamespace)) - - job.AddScenario(tcp.ValidateTCPMetrics(testPodNamespace)) - - dnsScenarios := []struct { - name string - req *dns.RequestValidationParams - resp *dns.ResponseValidationParams - }{ - { - name: "Validate basic DNS request and response metrics for a valid domain", - req: &dns.RequestValidationParams{ - NumResponse: "0", - Query: "kubernetes.default.svc.cluster.local.", - QueryType: "A", - Command: "nslookup kubernetes.default", - ExpectError: false, - }, - resp: &dns.ResponseValidationParams{ - NumResponse: "1", - Query: "kubernetes.default.svc.cluster.local.", - QueryType: "A", - ReturnCode: "No Error", - Response: "10.0.0.1", - }, - }, - { - name: "Validate basic DNS request and response metrics for a non-existent domain", - req: &dns.RequestValidationParams{ - NumResponse: "0", - Query: "some.non.existent.domain.", - QueryType: "A", - Command: "nslookup some.non.existent.domain", - ExpectError: true, - }, - resp: &dns.ResponseValidationParams{ - NumResponse: "0", - Query: "some.non.existent.domain.", - QueryType: "A", - Response: dns.EmptyResponse, // hacky way to bypass the framework for now - ReturnCode: "Non-Existent Domain", - }, - }, - } - - for _, scenario := range dnsScenarios { - job.AddScenario(dns.ValidateBasicDNSMetrics(scenario.name, scenario.req, scenario.resp, testPodNamespace)) - } - job.AddStep(&kubernetes.EnsureStableComponent{ PodNamespace: common.KubeSystemNamespace, LabelSelector: "k8s-app=retina", @@ -365,4 +321,3 @@ func RunPerfTest(kubeConfigFilePath string, chartPath string) *types.Job { return job } - diff --git a/test/e2e/scenarios/dns/scenarios.go b/test/e2e/scenarios/dns/scenarios.go index 2a766ce8c5..4c5c4b9804 100644 --- a/test/e2e/scenarios/dns/scenarios.go +++ b/test/e2e/scenarios/dns/scenarios.go @@ -36,7 +36,7 @@ type ResponseValidationParams struct { } // ValidateBasicDNSMetrics validates basic DNS metrics present in the metrics endpoint -func ValidateBasicDNSMetrics(scenarioName string, req *RequestValidationParams, resp *ResponseValidationParams, namespace string) *types.Scenario { +func ValidateBasicDNSMetrics(scenarioName string, req *RequestValidationParams, resp *ResponseValidationParams, namespace, arch string) *types.Scenario { // generate a random ID using rand id := fmt.Sprintf("basic-dns-port-forward-%d", rand.Int()) // nolint:gosec // fine to use math/rand here agnhostName := "agnhost-" + id @@ -46,6 +46,7 @@ func ValidateBasicDNSMetrics(scenarioName string, req *RequestValidationParams, Step: &kubernetes.CreateAgnhostStatefulSet{ AgnhostName: agnhostName, AgnhostNamespace: namespace, + AgnhostArch: arch, }, }, // Need this delay to guarantee that the pods will have bpf program attached @@ -149,7 +150,7 @@ func ValidateBasicDNSMetrics(scenarioName string, req *RequestValidationParams, } // ValidateAdvancedDNSMetrics validates the advanced DNS metrics present in the metrics endpoint -func ValidateAdvancedDNSMetrics(scenarioName string, req *RequestValidationParams, resp *ResponseValidationParams, kubeConfigFilePath string, namespace string) *types.Scenario { +func ValidateAdvancedDNSMetrics(scenarioName string, req *RequestValidationParams, resp *ResponseValidationParams, kubeConfigFilePath, namespace, arch string) *types.Scenario { // random ID id := fmt.Sprintf("adv-dns-port-forward-%d", rand.Int()) // nolint:gosec // fine to use math/rand here agnhostName := "agnhost-" + id @@ -159,6 +160,7 @@ func ValidateAdvancedDNSMetrics(scenarioName string, req *RequestValidationParam Step: &kubernetes.CreateAgnhostStatefulSet{ AgnhostName: agnhostName, AgnhostNamespace: namespace, + AgnhostArch: arch, }, }, // Need this delay to guarantee that the pods will have bpf program attached diff --git a/test/e2e/scenarios/drop/scenario.go b/test/e2e/scenarios/drop/scenario.go index 17e9a59fac..014d067113 100644 --- a/test/e2e/scenarios/drop/scenario.go +++ b/test/e2e/scenarios/drop/scenario.go @@ -16,8 +16,11 @@ const ( IPTableRuleDrop = "IPTABLE_RULE_DROP" ) -func ValidateDropMetric(namespace string) *types.Scenario { - name := "Drop Metrics" +func ValidateDropMetric(namespace, arch string) *types.Scenario { + id := "drop-port-forward-" + arch + agnhostName := "agnhost-drop" + podName := agnhostName + "-0" + name := "Drop Metrics - Arch: " + arch steps := []*types.StepWrapper{ { Step: &kubernetes.CreateDenyAllNetworkPolicy{ @@ -28,7 +31,8 @@ func ValidateDropMetric(namespace string) *types.Scenario { { Step: &kubernetes.CreateAgnhostStatefulSet{ AgnhostNamespace: namespace, - AgnhostName: "agnhost-drop", + AgnhostName: agnhostName, + AgnhostArch: arch, }, }, // Need this delay to guarantee that the pods will have bpf program attached @@ -43,7 +47,7 @@ func ValidateDropMetric(namespace string) *types.Scenario { { Step: &kubernetes.ExecInPod{ PodNamespace: namespace, - PodName: "agnhost-drop-0", + PodName: podName, Command: "curl -s -m 5 bing.com", }, Opts: &types.StepOptions{ @@ -59,7 +63,7 @@ func ValidateDropMetric(namespace string) *types.Scenario { { Step: &kubernetes.ExecInPod{ PodNamespace: namespace, - PodName: "agnhost-drop-0", + PodName: podName, Command: "curl -s -m 5 bing.com", }, Opts: &types.StepOptions{ @@ -74,16 +78,16 @@ func ValidateDropMetric(namespace string) *types.Scenario { LocalPort: "10093", RemotePort: "10093", Endpoint: "metrics", - OptionalLabelAffinity: "app=agnhost-drop", // port forward to a pod on a node that also has this pod with this label, assuming same namespace + OptionalLabelAffinity: "app=" + agnhostName, // port forward to a pod on a node that also has this pod with this label, assuming same namespace }, Opts: &types.StepOptions{ - RunInBackgroundWithID: "drop-port-forward", + RunInBackgroundWithID: id, }, }, { Step: &ValidateRetinaDropMetric{ PortForwardedRetinaPort: "10093", - Source: "agnhost-drop", + Source: agnhostName, Reason: IPTableRuleDrop, Direction: "unknown", Protocol: UDP, @@ -91,7 +95,7 @@ func ValidateDropMetric(namespace string) *types.Scenario { }, { Step: &types.Stop{ - BackgroundID: "drop-port-forward", + BackgroundID: id, }, }, @@ -108,7 +112,7 @@ func ValidateDropMetric(namespace string) *types.Scenario { Step: &kubernetes.DeleteKubernetesResource{ ResourceType: kubernetes.TypeString(kubernetes.StatefulSet), ResourceNamespace: namespace, - ResourceName: "agnhost-drop", + ResourceName: agnhostName, }, Opts: &types.StepOptions{ SkipSavingParametersToJob: true, }, diff --git a/test/e2e/scenarios/tcp/scenario.go b/test/e2e/scenarios/tcp/scenario.go index 0afa0c5013..c68b1268b7 100644 --- a/test/e2e/scenarios/tcp/scenario.go +++ b/test/e2e/scenarios/tcp/scenario.go @@ -16,8 +16,11 @@ const ( IPTableRuleDrop = "IPTABLE_RULE_DROP" ) -func ValidateTCPMetrics(namespace string) *types.Scenario { - Name := "Flow Metrics" +func ValidateTCPMetrics(namespace, arch string) *types.Scenario { + id := "flow-port-forward-" + arch + agnhostName := "agnhost-tcp" + podName := agnhostName + "-0" + Name := "Flow Metrics - Arch: " + arch Steps := []*types.StepWrapper{ { Step: &kubernetes.CreateKapingerDeployment{ @@ -27,13 +30,14 @@ func ValidateTCPMetrics(namespace string) *types.Scenario { }, { Step: &kubernetes.CreateAgnhostStatefulSet{ - AgnhostName: "agnhost-a", + AgnhostName: agnhostName, AgnhostNamespace: namespace, + AgnhostArch: arch, }, }, { Step: &kubernetes.ExecInPod{ - PodName: "agnhost-a-0", + PodName: podName, PodNamespace: namespace, Command: "curl -s -m 5 bing.com", }, Opts: &types.StepOptions{ @@ -47,7 +51,7 @@ func ValidateTCPMetrics(namespace string) *types.Scenario { }, { Step: &kubernetes.ExecInPod{ - PodName: "agnhost-a-0", + PodName: podName, PodNamespace: namespace, Command: "curl -s -m 5 bing.com", }, Opts: &types.StepOptions{ @@ -61,11 +65,11 @@ func ValidateTCPMetrics(namespace string) *types.Scenario { LocalPort: "10093", RemotePort: "10093", Endpoint: "metrics", - OptionalLabelAffinity: "app=agnhost-a", // port forward to a pod on a node that also has this pod with this label, assuming same namespace + OptionalLabelAffinity: "app=" + agnhostName, // port forward to a pod on a node that also has this pod with this label, assuming same namespace }, Opts: &types.StepOptions{ SkipSavingParametersToJob: true, - RunInBackgroundWithID: "drop-flow-forward", + RunInBackgroundWithID: id, }, }, { @@ -84,13 +88,13 @@ func ValidateTCPMetrics(namespace string) *types.Scenario { }, { Step: &types.Stop{ - BackgroundID: "drop-flow-forward", + BackgroundID: id, }, }, { Step: &kubernetes.DeleteKubernetesResource{ ResourceType: kubernetes.TypeString(kubernetes.StatefulSet), - ResourceName: "agnhost-a", + ResourceName: agnhostName, ResourceNamespace: namespace, }, Opts: &types.StepOptions{ SkipSavingParametersToJob: true,