Skip to content

Commit

Permalink
[ATMOSPHERE-263] feat: Support Kubernetes 1.29.x and 1.30.x (#334)
Browse files Browse the repository at this point in the history 
* update patch versions and add zuul CI jobs for new versions

* Use cloud images as base

Signed-off-by: Mohammed Naser <[email protected]>

* Update versions to build cleanly

* use kustomize to enable keystone webhook after kubeadm init

it does not add the --authorization-* arguments until after kubeadm init run.
Once kubeadm init has finished, run kustomize to add the arguments to the kube-apiserver.yaml manifest.

ref: kubernetes/cloud-provider-openstack#2575

* fix lint error and add 1.29 and 1.30 jobs

* append webhook authz mode only to avoid duplication with defaults

api-server sets Node and RBAC as default authz modes in its command args.
And does not allow the mode specified more than once.

* fix typo

* fix lint error

* make a workaround for cilium conformance test failures

cilium/cilium#29913
kubernetes/kubernetes#120069
cilium/cilium#9207

* fix flake8 errors

---------

Signed-off-by: Mohammed Naser <[email protected]>
Co-authored-by: okozachenko1203 <[email protected]>
Co-authored-by: Mohammed Naser <[email protected]>
Co-authored-by: Oleksandr K. <[email protected]>
  • Loading branch information
@0x00ace @okozachenko1203 @mnaser
4 people authored Aug 6, 2024
1 parent 72aaa6c commit 761b592
Show file tree
Hide file tree
Showing 8 changed files with 259 additions and 112 deletions.
92 changes: 28 additions & 64 deletions magnum_cluster_api/cmd/image_builder.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
QEMU_PACKAGES = [
"qemu-kvm",
"qemu-utils",
"mkisofs",
]


Expand All @@ -59,14 +60,14 @@ def validate_version(_, __, value):
@click.option(
"--version",
show_default=True,
default="v1.27.8",
default="v1.29.5",
callback=validate_version,
help="Kubernetes version",
)
@click.option(
"--image-builder-version",
show_default=True,
default="v0.1.30",
default="v0.1.31",
help="Image builder tag (or commit) to use for building image",
)
@click.option(
Expand Down Expand Up @@ -164,67 +165,30 @@ def main(
"node_custom_roles_pre": f"{node_custom_roles_pre}",
}

# NOTE(mnaser): We use the latest tested daily ISO for Ubuntu 22.04 in order
# to avoid a lengthy upgrade process.
if operating_system == "ubuntu-2204":
iso = "jammy-live-server-amd64.iso"

customization["iso_url"] = (
f"http://cdimage.ubuntu.com/ubuntu-server/jammy/daily-live/current/{iso}"
)

# Get the SHA256 sum for the ISO
r = requests.get(
"http://cdimage.ubuntu.com/ubuntu-server/jammy/daily-live/current/SHA256SUMS"
)
r.raise_for_status()
for line in r.text.splitlines():
if iso in line:
customization["iso_checksum"] = line.split()[0]
break

# Assert that we have the checksum
assert "iso_checksum" in customization

if operating_system == "rockylinux-8":
iso = "Rocky-x86_64-minimal.iso"

customization["iso_url"] = (
f"https://download.rockylinux.org/pub/rocky/8/isos/x86_64/{iso}"
)

# Get the SHA256 sum for the ISO
r = requests.get(
"https://download.rockylinux.org/pub/rocky/8/isos/x86_64/Rocky-x86_64-minimal.iso.CHECKSUM"
)
r.raise_for_status()
for line in r.text.splitlines():
if iso in line and "SHA256" in line:
customization["iso_checksum"] = line.split()[3]
break

# Assert that we have the checksum
assert "iso_checksum" in customization

if operating_system == "rockylinux-9":
iso = "Rocky-x86_64-minimal.iso"

customization["iso_url"] = (
f"https://download.rockylinux.org/pub/rocky/9/isos/x86_64/{iso}"
)

# Get the SHA256 sum for the ISO
r = requests.get(
"https://download.rockylinux.org/pub/rocky/9/isos/x86_64/Rocky-x86_64-minimal.iso.CHECKSUM"
)
r.raise_for_status()
for line in r.text.splitlines():
if iso in line and "SHA256" in line:
customization["iso_checksum"] = line.split()[3]
break

# Assert that we have the checksum
assert "iso_checksum" in customization
# NOTE(mnaser): Inside our CI, we use a local image in order speed up the
# process, so we will not download the image from the internet.
if os.environ.get("CI") == "true":
if operating_system == "ubuntu-2204":
customization["iso_checksum"] = (
"https://static.atmosphere.dev/ubuntu/jammy/20240605.1/SHA256SUMS"
)
customization["iso_url"] = (
"https://static.atmosphere.dev/ubuntu/jammy/20240605.1/jammy-server-cloudimg-amd64.img"
)
elif operating_system == "rockylinux-8":
customization["iso_checksum"] = (
"https://static.atmosphere.dev/rocky/8/images/x86_64/CHECKSUM"
)
customization["iso_url"] = (
"https://static.atmosphere.dev/rocky/8/images/x86_64/Rocky-8-GenericCloud-Base.latest.x86_64.qcow2"
)
elif operating_system == "rockylinux-9":
customization["iso_checksum"] = (
"https://static.atmosphere.dev/rocky/9/images/x86_64/CHECKSUM"
)
customization["iso_url"] = (
"https://static.atmosphere.dev/rocky/9/images/x86_64/Rocky-9-GenericCloud-Base.latest.x86_64.qcow2"
)

# NOTE(mnaser): Let's set number of CPUs to equal the number of CPUs on the
# host to speed up the build process.
Expand Down Expand Up @@ -255,7 +219,7 @@ def main(
/usr/bin/make \
-C \
{ib_path}/images/capi \
build-qemu-{operating_system}
build-qemu-{operating_system}-cloudimg
"""
).encode("utf-8"),
env={
Expand Down
10 changes: 10 additions & 0 deletions magnum_cluster_api/conf.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,16 @@
default="$image_repository/cluster-autoscaler:v1.28.5",
help="Image for the cluster auto-scaler for Kubernetes v1.28.",
),
cfg.StrOpt(
"v1_29_image",
default="$image_repository/cluster-autoscaler:v1.29.3",
help="Image for the cluster auto-scaler for Kubernetes v1.29.",
),
cfg.StrOpt(
"v1_30_image",
default="$image_repository/cluster-autoscaler:v1.30.1",
help="Image for the cluster auto-scaler for Kubernetes v1.30.",
),
]


Expand Down
63 changes: 51 additions & 12 deletions magnum_cluster_api/resources.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -214,17 +214,22 @@ def get_object(self) -> pykube.ConfigMap:
"image": {
"tag": cilium_version,
},
# NOTE(okozachenko): cilium has a limitation https://github.com/cilium/cilium/issues/9207
# Because of that, it fails on the test
# `Services should serve endpoints on same port and different protocols`.
# https://github.com/kubernetes/kubernetes/pull/120069#issuecomment-2111252221
"k8s": {"serviceProxyName": "cilium"},
"operator": {
"image": {
"tag": cilium_version,
},
},
# NOTE(okozachenko1203): For users who run with kube-proxy (i.e. with Cilium's kube-proxy
# replacement disabled), the ClusterIP service loadbalancing when a
# request is sent from a pod running in a non-host network namespace
# is still performed at the pod network interface (until
# https://github.com/cilium/cilium/issues/16197 is fixed). For this
# case the session affinity support is disabled by default.
# NOTE(okozachenko): For users who run with kube-proxy (i.e. with Cilium's kube-proxy
# replacement disabled), the ClusterIP service loadbalancing when a
# request is sent from a pod running in a non-host network namespace
# is still performed at the pod network interface (until
# https://github.com/cilium/cilium/issues/16197 is fixed). For this
# case the session affinity support is disabled by default.
"sessionAffinity": "true",
"ipam": {
"operator": {
Expand Down Expand Up @@ -736,6 +741,9 @@ def get_object(self) -> objects.KubeadmControlPlaneTemplate:
"rm /var/lib/etcd/lost+found -rf",
"bash /run/kubeadm/configure-kube-proxy.sh",
],
"postKubeadmCommands": [
"echo PLACEHOLDER",
],
},
},
},
Expand Down Expand Up @@ -1999,18 +2007,49 @@ def get_object(self) -> objects.ClusterClass:
"jsonPatches": [
{
"op": "add",
"path": "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraArgs/authentication-token-webhook-config-file", # noqa: E501
"value": "/etc/kubernetes/webhooks/webhookconfig.yaml",
"path": "/spec/template/spec/kubeadmConfigSpec/files/-",
"value": {
"path": "/etc/kubernetes/keystone-kustomization/kustomization.yml",
"permissions": "0644",
"owner": "root:root",
"content": textwrap.dedent(
"""\
resources:
- kube-apiserver.yaml
patches:
- target:
group: ""
version: v1
kind: Pod
name: kube-apiserver
patch: |-
- op: add
path: /spec/containers/0/command/-
value: --authentication-token-webhook-config-file=/etc/kubernetes/webhooks/webhookconfig.yaml # noqa: E501
- op: add
path: /spec/containers/0/command/-
value: --authorization-webhook-config-file=/etc/kubernetes/webhooks/webhookconfig.yaml # noqa: E501
- op: add
path: /spec/containers/0/command/-
value: --authorization-mode=Webhook
"""
),
},
},
{
"op": "add",
"path": "/spec/template/spec/kubeadmConfigSpec/preKubeadmCommands/-",
"value": "mkdir /etc/kubernetes/keystone-kustomization",
},
{
"op": "add",
"path": "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraArgs/authorization-webhook-config-file", # noqa: E501
"value": "/etc/kubernetes/webhooks/webhookconfig.yaml",
"path": "/spec/template/spec/kubeadmConfigSpec/postKubeadmCommands/-",
"value": "cp /etc/kubernetes/manifests/kube-apiserver.yaml /etc/kubernetes/keystone-kustomization/kube-apiserver.yaml", # noqa: E501
},
{
"op": "add",
"path": "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraArgs/authorization-mode", # noqa: E501
"value": "Node,RBAC,Webhook",
"path": "/spec/template/spec/kubeadmConfigSpec/postKubeadmCommands/-",
"value": "kubectl kustomize /etc/kubernetes/keystone-kustomization -o /etc/kubernetes/manifests/kube-apiserver.yaml", # noqa: E501
},
],
}
Expand Down
54 changes: 42 additions & 12 deletions zuul.d/jobs-flatcar.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,36 +15,66 @@
image_os_distro: flatcar

- job:
name: magnum-cluster-api-image-build-flatcar-v1.27.8
name: magnum-cluster-api-image-build-flatcar-v1.27.15
parent: magnum-cluster-api-image-build-flatcar
vars:
kube_tag: v1.27.8
kube_tag: v1.27.15

- job:
name: magnum-cluster-api-sonobuoy-flatcar-v1.27.8
name: magnum-cluster-api-sonobuoy-flatcar-v1.27.15
parent: magnum-cluster-api-sonobuoy-flatcar
dependencies:
- name: magnum-cluster-api-image-build-flatcar-v1.27.8
- name: magnum-cluster-api-image-build-flatcar-v1.27.15
soft: true
vars:
kube_tag: v1.27.8
kube_tag: v1.27.15

- job:
name: magnum-cluster-api-sonobuoy-flatcar-v1.27.8-calico
parent: magnum-cluster-api-sonobuoy-flatcar-v1.27.8
name: magnum-cluster-api-image-build-flatcar-v1.28.11
parent: magnum-cluster-api-image-build-flatcar
vars:
kube_tag: v1.28.11

- job:
name: magnum-cluster-api-sonobuoy-flatcar-v1.28.11
parent: magnum-cluster-api-sonobuoy-flatcar
dependencies:
- name: magnum-cluster-api-image-build-flatcar-v1.28.11
soft: true
vars:
kube_tag: v1.28.11

- job:
name: magnum-cluster-api-image-build-flatcar-v1.29.6
parent: magnum-cluster-api-image-build-flatcar
vars:
kube_tag: v1.29.6

- job:
name: magnum-cluster-api-sonobuoy-flatcar-v1.29.6
parent: magnum-cluster-api-sonobuoy-flatcar
dependencies:
- name: magnum-cluster-api-image-build-flatcar-v1.29.6
soft: true
vars:
kube_tag: v1.29.6

- job:
name: magnum-cluster-api-sonobuoy-flatcar-v1.27.15-calico
parent: magnum-cluster-api-sonobuoy-flatcar-v1.27.15
vars:
network_driver: calico

- job:
name: magnum-cluster-api-sonobuoy-flatcar-v1.27.8-cilium
parent: magnum-cluster-api-sonobuoy-flatcar-v1.27.8
name: magnum-cluster-api-sonobuoy-flatcar-v1.27.15-cilium
parent: magnum-cluster-api-sonobuoy-flatcar-v1.27.15
vars:
network_driver: cilium

- project-template:
name: magnum-cluster-api-flatcar
check:
jobs:
- magnum-cluster-api-image-build-flatcar-v1.27.8
- magnum-cluster-api-sonobuoy-flatcar-v1.27.8-calico
- magnum-cluster-api-sonobuoy-flatcar-v1.27.8-cilium
- magnum-cluster-api-image-build-flatcar-v1.27.15
- magnum-cluster-api-sonobuoy-flatcar-v1.27.15-calico
- magnum-cluster-api-sonobuoy-flatcar-v1.27.15-cilium
54 changes: 42 additions & 12 deletions zuul.d/jobs-rockylinux-8.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,36 +15,66 @@
image_os_distro: ubuntu

- job:
name: magnum-cluster-api-image-build-rockylinux-8-v1.27.8
name: magnum-cluster-api-image-build-rockylinux-8-v1.27.15
parent: magnum-cluster-api-image-build-rockylinux-8
vars:
kube_tag: v1.27.8
kube_tag: v1.27.15

- job:
name: magnum-cluster-api-sonobuoy-rockylinux-8-v1.27.8
name: magnum-cluster-api-sonobuoy-rockylinux-8-v1.27.15
parent: magnum-cluster-api-sonobuoy-rockylinux-8
dependencies:
- name: magnum-cluster-api-image-build-rockylinux-8-v1.27.8
- name: magnum-cluster-api-image-build-rockylinux-8-v1.27.15
soft: true
vars:
kube_tag: v1.27.8
kube_tag: v1.27.15

- job:
name: magnum-cluster-api-sonobuoy-rockylinux-8-v1.27.8-calico
parent: magnum-cluster-api-sonobuoy-rockylinux-8-v1.27.8
name: magnum-cluster-api-image-build-rockylinux-8-v1.28.11
parent: magnum-cluster-api-image-build-rockylinux-8
vars:
kube_tag: v1.28.11

- job:
name: magnum-cluster-api-sonobuoy-rockylinux-8-v1.28.11
parent: magnum-cluster-api-sonobuoy-rockylinux-8
dependencies:
- name: magnum-cluster-api-image-build-rockylinux-8-v1.28.11
soft: true
vars:
kube_tag: v1.28.11

- job:
name: magnum-cluster-api-image-build-rockylinux-8-v1.29.6
parent: magnum-cluster-api-image-build-rockylinux-8
vars:
kube_tag: v1.29.6

- job:
name: magnum-cluster-api-sonobuoy-rockylinux-8-v1.29.6
parent: magnum-cluster-api-sonobuoy-rockylinux-8
dependencies:
- name: magnum-cluster-api-image-build-rockylinux-8-v1.29.6
soft: true
vars:
kube_tag: v1.29.6

- job:
name: magnum-cluster-api-sonobuoy-rockylinux-8-v1.27.15-calico
parent: magnum-cluster-api-sonobuoy-rockylinux-8-v1.27.15
vars:
network_driver: calico

- job:
name: magnum-cluster-api-sonobuoy-rockylinux-8-v1.27.8-cilium
parent: magnum-cluster-api-sonobuoy-rockylinux-8-v1.27.8
name: magnum-cluster-api-sonobuoy-rockylinux-8-v1.27.15-cilium
parent: magnum-cluster-api-sonobuoy-rockylinux-8-v1.27.15
vars:
network_driver: cilium

- project-template:
name: magnum-cluster-api-rockylinux-8
check:
jobs:
- magnum-cluster-api-image-build-rockylinux-8-v1.27.8
- magnum-cluster-api-sonobuoy-rockylinux-8-v1.27.8-calico
- magnum-cluster-api-sonobuoy-rockylinux-8-v1.27.8-cilium
- magnum-cluster-api-image-build-rockylinux-8-v1.27.15
- magnum-cluster-api-sonobuoy-rockylinux-8-v1.27.15-calico
- magnum-cluster-api-sonobuoy-rockylinux-8-v1.27.15-cilium
Loading

0 comments on commit 761b592

Please sign in to comment.