Container Orchestration with Kubernetes: Production Patterns | Cloud Architecture Hub
Kubernetes has become the de facto standard for container orchestration, but running it in production requires mastering patterns that go far beyond basic deployments. This guide covers advanced production patterns: operators and custom controllers for infrastructure automation, service mesh integration for observability and security, GitOps with ArgoCD for continuous delivery, and multi-cluster management for resilience and compliance.
Pattern 1: Operators & Custom Controllers
Operators extend Kubernetes with domain-specific logic, encoding operational knowledge into software. They watch custom resources and reconcile desired state.
Building a Database Operator (PostgreSQL Example)
# CRD Definition: postgresql.cnpg.io_v1_postgresql.yaml
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
name: postgresqls.postgresql.cnpg.io
spec:
group: postgresql.cnpg.io
versions:
- name: v1
served: true
storage: true
schema:
openAPIV3Schema:
type: object
properties:
spec:
type: object
properties:
instances:
type: integer
minimum: 1
maximum: 10
storageSize:
type: string
pattern: '^[0-9]+(Gi|Mi|Ti)$'
postgresql:
type: object
properties:
parameters:
type: object
additionalProperties:
type: string
bootstrap:
type: object
properties:
initdb:
type: object
properties:
database:
type: string
owner:
type: string
secret:
type: object
properties:
name:
type: string
monitoring:
type: object
properties:
enablePodMonitor:
type: boolean
customQueriesConfigMap:
type: string
3: metadata:
name: postgresql-sample
4: namespace: databases
5: spec:
instances: 3
storageSize: 50Gi
postgresql:
parameters:
max_connections: "200"
shared_buffers: "256MB"
effective_cache_size: "1GB"
bootstrap:
initdb:
database: appdb
owner: appuser
secret:
name: pg-credentials
monitoring:
enablePodMonitor: true
Controller Implementation (Go/Kubebuilder)
// controllers/postgresql_controller.go
package controllers
import (
"context"
"time"
"github.com/go-logr/logr"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
"sigs.k8s.io/controller-runtime/pkg/log"
postgresqlv1 "github.com/cloudarchitecturehub/postgresql-operator/api/v1"
)
type PostgreSQLReconciler struct {
client.Client
Scheme *runtime.Scheme
Log logr.Logger
}
func (r *PostgreSQLReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
log := log.FromContext(ctx)
// Fetch the PostgreSQL instance
pg := &postgresqlv1.PostgreSQL{}
if err := r.Get(ctx, req.NamespacedName, pg); err != nil {
return ctrl.Result{}, client.IgnoreNotFound(err)
}
// Handle deletion
if !pg.DeletionTimestamp.IsZero() {
return r.handleDeletion(ctx, pg)
}
// Add finalizer
if !controllerutil.ContainsFinalizer(pg, "postgresql.cnpg.io/finalizer") {
controllerutil.AddFinalizer(pg, "postgresql.cnpg.io/finalizer")
if err := r.Update(ctx, pg); err != nil {
return ctrl.Result{}, err
}
}
// Reconcile components
if err := r.reconcileStatefulSet(ctx, pg); err != nil {
return ctrl.Result{}, err
}
if err := r.reconcileServices(ctx, pg); err != nil {
return ctrl.Result{}, err
}
if err := r.reconcileConfigMap(ctx, pg); err != nil {
return ctrl.Result{}, err
}
if err := r.reconcileSecrets(ctx, pg); err != nil {
return ctrl.Result{}, err
}
if err := r.reconcileMonitoring(ctx, pg); err != nil {
return ctrl.Result{}, err
}
// Update status
return r.updateStatus(ctx, pg)
}
func (r *PostgreSQLReconciler) reconcileStatefulSet(ctx context.Context, pg *postgresqlv1.PostgreSQL) error {
sts := &appsv1.StatefulSet{
ObjectMeta: metav1.ObjectMeta{
Name: pg.Name,
Namespace: pg.Namespace,
},
}
_, err := ctrl.CreateOrUpdate(ctx, r.Client, sts, func() error {
// Set owner reference
if err := controllerutil.SetControllerReference(pg, sts, r.Scheme); err != nil {
return err
}
// Configure spec
sts.Spec = appsv1.StatefulSetSpec{
ServiceName: pg.Name,
Replicas: &pg.Spec.Instances,
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{
"postgresql": pg.Name,
},
},
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: map[string]string{
"postgresql": pg.Name,
},
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "postgresql",
Image: "postgres:16",
Ports: []corev1.ContainerPort{{ContainerPort: 5432}},
Env: []corev1.EnvVar{
{Name: "POSTGRES_DB", ValueFrom: &corev1.EnvVarSource{
ConfigMapKeyRef: &corev1.ConfigMapKeySelector{
LocalObjectReference: corev1.LocalObjectReference{Name: pg.Name + "-config"},
Key: "database",
},
}},
},
VolumeMounts: []corev1.VolumeMount{
{Name: "data", MountPath: "/var/lib/postgresql/data"},
},
Resources: corev1.ResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceMemory: resource.MustParse("512Mi"),
corev1.ResourceCPU: resource.MustParse("500m"),
},
Limits: corev1.ResourceList{
corev1.ResourceMemory: resource.MustParse("2Gi"),
corev1.ResourceCPU: resource.MustParse("2000m"),
},
},
},
},
Volumes: []corev1.Volume{
{Name: "data", VolumeSource: corev1.VolumeSource{
PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
ClaimName: pg.Name + "-data",
},
}},
},
},
},
VolumeClaimTemplates: []corev1.PersistentVolumeClaim{
{
ObjectMeta: metav1.ObjectMeta{Name: "data"},
Spec: corev1.PersistentVolumeClaimSpec{
AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce},
Resources: corev1.ResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceStorage: resource.MustParse(pg.Spec.StorageSize),
},
},
},
},
},
}
return nil
})
return err
}
Pattern 2: Service Mesh Integration (Istio)
Service mesh provides traffic management, security, and observability without application changes.
Traffic Management: Canary Deployments
# Istio VirtualService for Canary Release
apiVersion: networking.istio.io/v1beta1
kind: VirtualService
metadata:
name: orders-service
namespace: production
spec:
hosts:
- orders.production.svc.cluster.local
http:
- match:
- headers:
x-canary:
exact: "true"
route:
- destination:
host: orders.production.svc.cluster.local
subset: v2
weight: 100
- route:
- destination:
host: orders.production.svc.cluster.local
subset: v1
weight: 90
- destination:
host: orders.production.svc.cluster.local
subset: v2
weight: 10
retries:
attempts: 3
perTryTimeout: 2s
retryOn: connect-failure,refused-stream,unavailable,cancelled,retriable-status-codes
timeout: 10s
---
# DestinationRule with Subsets
apiVersion: networking.istio.io/v1beta1
kind: DestinationRule
metadata:
name: orders-service
namespace: production
spec:
host: orders.production.svc.cluster.local
trafficPolicy:
connectionPool:
tcp:
maxConnections: 1000
http:
h2UpgradePolicy: UPGRADE
http1MaxPendingRequests: 1000
http2MaxRequests: 1000
loadBalancer:
simple: LEAST_REQUEST
outlierDetection:
consecutive5xxErrors: 5
interval: 30s
baseEjectionTime: 30s
maxEjectionPercent: 50
subsets:
- name: v1
labels:
version: v1
- name: v2
labels:
version: v2
Mutual TLS & Authorization
# PeerAuthentication: STRICT mTLS
apiVersion: security.istio.io/v1beta1
kind: PeerAuthentication
metadata:
name: default
namespace: production
spec:
mtls:
mode: STRICT
---
# AuthorizationPolicy: Service-to-Service
apiVersion: security.istio.io/v1beta1
kind: AuthorizationPolicy
metadata:
name: orders-service-authz
namespace: production
spec:
selector:
matchLabels:
app: orders-service
rules:
- from:
- source:
principals:
- "cluster.local/ns/production/sa/api-gateway"
- "cluster.local/ns/production/sa/payment-service"
to:
- operation:
methods: ["GET", "POST"]
paths: ["/api/orders/*"]
when:
- key: request.auth.claims[iss]
values: ["https://accounts.google.com", "https://login.microsoftonline.com/*"]
Pattern 3: GitOps with ArgoCD
GitOps uses Git as the single source of truth for declarative infrastructure and applications.
ArgoCD Application Definition
# argocd/applications/orders-service.yaml
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: orders-service
namespace: argocd
finalizers:
- resources-finalizer.argocd.argoproj.io
spec:
project: production
source:
repoURL: https://github.com/cloudarchitecturehub/infra.git
targetRevision: HEAD
path: k8s/overlays/production/orders-service
kustomize:
images:
- orders-service:ghcr.io/cloudarchitecturehub/orders-service:v1.2.3
destination:
server: https://kubernetes.default.svc
namespace: production
syncPolicy:
automated:
prune: true
selfHeal: true
allowEmpty: false
syncOptions:
- CreateNamespace=true
- PrunePropagationPolicy=foreground
- PruneLast=true
retry:
limit: 5
backoff:
duration: 5s
factor: 2
maxDuration: 3m
ignoreDifferences:
- group: apps
kind: Deployment
jsonPointers:
- /spec/replicas
- group: ""
kind: Secret
jsonPointers:
- /data
ApplicationSet for Multi-Cluster
# argocd/applicationsets/multi-cluster.yaml
apiVersion: argoproj.io/v1alpha1
kind: ApplicationSet
metadata:
name: orders-service-multicluster
namespace: argocd
spec:
generators:
- clusters:
selector:
matchLabels:
environment: production
template:
metadata:
name: 'orders-service-{{name}}'
spec:
project: production
source:
repoURL: https://github.com/cloudarchitecturehub/infra.git
targetRevision: HEAD
path: k8s/overlays/production/orders-service
destination:
server: '{{server}}'
namespace: production
syncPolicy:
preserveResourcesOnDeletion: true
template:
metadata:
labels:
app: orders-service
spec:
syncPolicy:
automated:
prune: true
selfHeal: true
Progressive Delivery with Argo Rollouts
# argo-rollouts/orders-service.yaml
apiVersion: argoproj.io/v1alpha1
kind: Rollout
metadata:
name: orders-service
namespace: production
spec:
replicas: 10
strategy:
canary:
canaryMetadata:
labels:
version: canary
stableMetadata:
labels:
version: stable
steps:
- setWeight: 5
- pause: {duration: 10m}
- setWeight: 20
- pause: {duration: 10m}
- setWeight: 50
- pause: {duration: 10m}
- setWeight: 100
analysis:
templates:
- templateName: success-rate
startingStep: 1
args:
- name: service-name
value: orders-service.canary
selector:
matchLabels:
app: orders-service
template:
metadata:
labels:
app: orders-service
annotations:
prometheus.io/scrape: "true"
prometheus.io/port: "8080"
spec:
containers:
- name: orders-service
image: ghcr.io/cloudarchitecturehub/orders-service:v1.2.3
ports:
- containerPort: 8080
---
# AnalysisTemplate for Automated Promotion
apiVersion: argoproj.io/v1alpha1
kind: AnalysisTemplate
metadata:
name: success-rate
namespace: production
spec:
args:
- name: service-name
metrics:
- name: success-rate
interval: 1m
successCondition: result[0] >= 0.99
failureLimit: 3
provider:
prometheus:
address: http://prometheus.monitoring.svc:9090
query: |
sum(rate(http_requests_total{job="{{args.service-name}}",code=~"2.."}[5m]))
/
sum(rate(http_requests_total{job="{{args.service-name}}"}[5m]))
Pattern 4: Multi-Cluster Management
Managing multiple clusters requires consistent policy, networking, and identity.
Cluster Federa
Comments
Post a Comment