Argo Workflows

1
┌─────────────────────────────────────────────────────────────────┐
2
│                   Argo Workflows Components                      │
3
│                                                                  │
4
│  ┌─────────────────┐     ┌─────────────────┐                    │
5
│  │ Workflow        │     │   Argo Server   │                    │
6
│  │ Controller      │     │   (UI + API)    │                    │
7
│  └────────┬────────┘     └─────────────────┘                    │
8
│           │                                                      │
9
│  ┌────────▼────────────────────────────────┐                    │
10
│  │              Workflow Executor           │                    │
11
│  │  ┌─────────┐ ┌─────────┐ ┌─────────┐   │                    │
12
│  │  │  Pod 1  │ │  Pod 2  │ │  Pod 3  │   │                    │
13
│  │  │ (Step)  │ │ (Step)  │ │ (Step)  │   │                    │
14
│  │  └─────────┘ └─────────┘ └─────────┘   │                    │
15
│  └──────────────────────────────────────────┘                    │
16
│                                                                  │
17
│  ┌─────────────────┐     ┌─────────────────┐                    │
18
│  │ Artifact Repo   │     │  Event Source   │                    │
19
│  │ (S3/GCS/Minio)  │     │ (Webhook/Git)   │                    │
20
│  └─────────────────┘     └─────────────────┘                    │
21
└─────────────────────────────────────────────────────────────────┘

1
#!/bin/bash
2
# Argo Workflows installation
3
4
# Create namespace
5
kubectl create namespace argo
6
7
# Install Argo Workflows
8
kubectl apply -n argo -f https://github.com/argoproj/argo-workflows/releases/download/v3.5.2/install.yaml
9
10
# Install Argo Events for triggering workflows
11
kubectl create namespace argo-events
12
kubectl apply -n argo-events -f https://raw.githubusercontent.com/argoproj/argo-events/stable/manifests/install.yaml
13
14
# Configure artifact repository
15
kubectl apply -f - <<EOF
16
apiVersion: v1
17
kind: ConfigMap
18
metadata:
19
  name: artifact-repositories
20
  namespace: argo
21
data:
22
  default-v1: |
23
    archiveLogs: true
24
    s3:
25
      bucket: argo-artifacts
26
      endpoint: s3.amazonaws.com
27
      region: us-east-1
28
      accessKeySecret:
29
        name: argo-artifacts-s3
30
        key: accessKey
31
      secretKeySecret:
32
        name: argo-artifacts-s3
33
        key: secretKey
34
EOF
35
36
# Create service account with proper RBAC
37
kubectl apply -f - <<EOF
38
apiVersion: v1
39
kind: ServiceAccount
40
metadata:
41
  name: argo-workflow
42
  namespace: argo
43
---
44
apiVersion: rbac.authorization.k8s.io/v1
45
kind: ClusterRoleBinding
46
metadata:
47
  name: argo-workflow-binding
48
roleRef:
49
  apiGroup: rbac.authorization.k8s.io
50
  kind: ClusterRole
51
  name: argo-workflow-role
52
subjects:
53
  - kind: ServiceAccount
54
    name: argo-workflow
55
    namespace: argo
56
EOF
57
58
echo "Argo Workflows installed successfully"

1
apiVersion: argoproj.io/v1alpha1
2
kind: WorkflowTemplate
3
metadata:
4
  name: ci-cd-pipeline
5
  namespace: argo
6
spec:
7
  entrypoint: ci-cd
8
9
  serviceAccountName: argo-workflow
10
11
  # Workflow-level parameters
12
  arguments:
13
    parameters:
14
      - name: repo-url
15
        value: "https://github.com/company/application.git"
16
      - name: branch
17
        value: "main"
18
      - name: image-registry
19
        value: "ghcr.io/company"
20
      - name: app-name
21
        value: "payment-service"
22
      - name: environment
23
        value: "staging"
24
25
  # Volume claim template for workspace
26
  volumeClaimTemplates:
27
    - metadata:
28
        name: workspace
29
      spec:
30
        accessModes: ["ReadWriteOnce"]
31
        resources:
32
          requests:
33
            storage: 5Gi
34
35
  templates:
36
    # Main DAG template
37
    - name: ci-cd
38
      dag:
39
        tasks:
40
          - name: checkout
41
            template: git-checkout
42
            arguments:
43
              parameters:
44
                - name: repo-url
45
                  value: "{{workflow.parameters.repo-url}}"
46
                - name: branch
47
                  value: "{{workflow.parameters.branch}}"
48
49
          - name: run-tests
50
            template: test
51
            dependencies: [checkout]
52
53
          - name: security-scan
54
            template: security-scan
55
            dependencies: [checkout]
56
57
          - name: build-image
58
            template: build-and-push
59
            dependencies: [run-tests, security-scan]
60
            arguments:
61
              parameters:
62
                - name: image-tag
63
                  value: "{{workflow.parameters.image-registry}}/{{workflow.parameters.app-name}}:{{workflow.parameters.branch}}-{{workflow.uid}}"
64
65
          - name: update-manifests
66
            template: update-gitops-repo
67
            dependencies: [build-image]
68
            arguments:
69
              parameters:
70
                - name: image-tag
71
                  value: "{{workflow.parameters.image-registry}}/{{workflow.parameters.app-name}}:{{workflow.parameters.branch}}-{{workflow.uid}}"
72
                - name: environment
73
                  value: "{{workflow.parameters.environment}}"
74
75
          - name: notify
76
            template: slack-notify
77
            dependencies: [update-manifests]
78
79
    # Git checkout template
80
    - name: git-checkout
81
      inputs:
82
        parameters:
83
          - name: repo-url
84
          - name: branch
85
      container:
86
        image: alpine/git:latest
87
        command: [sh, -c]
88
        args:
89
          - |
90
            git clone --depth 1 --branch {{inputs.parameters.branch}} \
91
              {{inputs.parameters.repo-url}} /workspace/source
92
            cd /workspace/source
93
            echo "Checked out commit: $(git rev-parse HEAD)"
94
            echo "$(git rev-parse HEAD)" > /workspace/commit-sha
95
        volumeMounts:
96
          - name: workspace
97
            mountPath: /workspace
98
99
    # Test template
100
    - name: test
101
      container:
102
        image: node:18-alpine
103
        command: [sh, -c]
104
        args:
105
          - |
106
            cd /workspace/source
107
            npm ci
108
            npm run lint
109
            npm run test:unit
110
            npm run test:integration
111
        volumeMounts:
112
          - name: workspace
113
            mountPath: /workspace
114
        resources:
115
          requests:
116
            cpu: 500m
117
            memory: 1Gi
118
          limits:
119
            cpu: 2000m
120
            memory: 4Gi
121
122
    # Security scan template
123
    - name: security-scan
124
      container:
125
        image: aquasec/trivy:latest
126
        command: [sh, -c]
127
        args:
128
          - |
129
            cd /workspace/source
130
131
            # Scan for vulnerabilities in dependencies
132
            trivy fs --security-checks vuln,secret,config \
133
              --severity HIGH,CRITICAL \
134
              --exit-code 1 \
135
              --format table \
136
              .
137
138
            # Generate SBOM
139
            trivy fs --format spdx-json \
140
              --output /workspace/sbom.json \
141
              .
142
        volumeMounts:
143
          - name: workspace
144
            mountPath: /workspace
145
      outputs:
146
        artifacts:
147
          - name: sbom
148
            path: /workspace/sbom.json
149
            s3:
150
              key: "{{workflow.uid}}/sbom.json"
151
152
    # Build and push template with Kaniko
153
    - name: build-and-push
154
      inputs:
155
        parameters:
156
          - name: image-tag
157
      container:
158
        image: gcr.io/kaniko-project/executor:latest
159
        args:
160
          - --dockerfile=/workspace/source/Dockerfile
161
          - --context=/workspace/source
162
          - --destination={{inputs.parameters.image-tag}}
163
          - --cache=true
164
          - --cache-repo={{workflow.parameters.image-registry}}/cache
165
          - --snapshot-mode=redo
166
          - --use-new-run
167
        volumeMounts:
168
          - name: workspace
169
            mountPath: /workspace
170
          - name: docker-config
171
            mountPath: /kaniko/.docker
172
        resources:
173
          requests:
174
            cpu: 1000m
175
            memory: 2Gi
176
          limits:
177
            cpu: 4000m
178
            memory: 8Gi
179
      volumes:
180
        - name: docker-config
181
          secret:
182
            secretName: docker-registry-creds
183
184
    # Update GitOps repository
185
    - name: update-gitops-repo
186
      inputs:
187
        parameters:
188
          - name: image-tag
189
          - name: environment
190
      container:
191
        image: alpine/git:latest
192
        command: [sh, -c]
193
        args:
194
          - |
195
            set -e
196
197
            # Clone GitOps repo
198
            git clone https://github.com/company/gitops-repo.git /tmp/gitops
199
            cd /tmp/gitops
200
201
            # Update image tag in Kustomization
202
            cd apps/{{workflow.parameters.app-name}}/overlays/{{inputs.parameters.environment}}
203
204
            # Update image tag
205
            kustomize edit set image \
206
              {{workflow.parameters.app-name}}={{inputs.parameters.image-tag}}
207
208
            # Commit and push
209
            git config user.email "[email protected]"
210
            git config user.name "Argo Workflows"
211
            git add .
212
            git commit -m "chore: update {{workflow.parameters.app-name}} to {{inputs.parameters.image-tag}}"
213
            git push origin main
214
215
            echo "GitOps repository updated successfully"
216
        env:
217
          - name: GIT_TOKEN
218
            valueFrom:
219
              secretKeyRef:
220
                name: git-credentials
221
                key: token
222
        volumeMounts:
223
          - name: workspace
224
            mountPath: /workspace
225
226
    # Slack notification template
227
    - name: slack-notify
228
      container:
229
        image: curlimages/curl:latest
230
        command: [sh, -c]
231
        args:
232
          - |
233
            curl -X POST $SLACK_WEBHOOK_URL \
234
              -H 'Content-type: application/json' \
235
              --data '{
236
                "text": "CI/CD Pipeline Completed",
237
                "blocks": [
238
                  {
239
                    "type": "section",
240
                    "text": {
241
                      "type": "mrkdwn",
242
                      "text": "*Pipeline Completed Successfully*\n* App: {{workflow.parameters.app-name}}\n* Environment: {{workflow.parameters.environment}}\n* Branch: {{workflow.parameters.branch}}"
243
                    }
244
                  }
245
                ]
246
              }'
247
        env:
248
          - name: SLACK_WEBHOOK_URL
249
            valueFrom:
250
              secretKeyRef:
251
                name: slack-webhook
252
                key: url

1
apiVersion: argoproj.io/v1alpha1
2
kind: EventSource
3
metadata:
4
  name: github-webhook
5
  namespace: argo-events
6
spec:
7
  github:
8
    app-push:
9
      owner: company
10
      repository: application
11
      webhook:
12
        endpoint: /push
13
        port: "12000"
14
        method: POST
15
        url: https://webhooks.example.com
16
      events:
17
        - push
18
      apiToken:
19
        name: github-token
20
        key: token
21
      webhookSecret:
22
        name: github-webhook-secret
23
        key: secret
24
      contentType: json
25
      insecure: false
26
      active: true
27
28
    app-pr:
29
      owner: company
30
      repository: application
31
      webhook:
32
        endpoint: /pr
33
        port: "12000"
34
        method: POST
35
        url: https://webhooks.example.com
36
      events:
37
        - pull_request
38
      apiToken:
39
        name: github-token
40
        key: token
41
      webhookSecret:
42
        name: github-webhook-secret
43
        key: secret

1
apiVersion: argoproj.io/v1alpha1
2
kind: Sensor
3
metadata:
4
  name: github-sensor
5
  namespace: argo-events
6
spec:
7
  dependencies:
8
    - name: push-dep
9
      eventSourceName: github-webhook
10
      eventName: app-push
11
      filters:
12
        data:
13
          - path: body.ref
14
            type: string
15
            value:
16
              - refs/heads/main
17
              - refs/heads/develop
18
19
    - name: pr-dep
20
      eventSourceName: github-webhook
21
      eventName: app-pr
22
      filters:
23
        data:
24
          - path: body.action
25
            type: string
26
            value:
27
              - opened
28
              - synchronize
29
30
  triggers:
31
    # Trigger CI/CD on push to main/develop
32
    - template:
33
        name: trigger-ci-cd
34
        argoWorkflow:
35
          group: argoproj.io
36
          version: v1alpha1
37
          resource: workflows
38
          operation: submit
39
          source:
40
            resource:
41
              apiVersion: argoproj.io/v1alpha1
42
              kind: Workflow
43
              metadata:
44
                generateName: ci-cd-pipeline-
45
              spec:
46
                workflowTemplateRef:
47
                  name: ci-cd-pipeline
48
                arguments:
49
                  parameters:
50
                    - name: branch
51
                      value: ""
52
                    - name: repo-url
53
                      value: ""
54
          parameters:
55
            - src:
56
                dependencyName: push-dep
57
                dataKey: body.ref
58
                dataTemplate: "{{ trimPrefix .Input \"refs/heads/\" }}"
59
              dest: spec.arguments.parameters.0.value
60
            - src:
61
                dependencyName: push-dep
62
                dataKey: body.repository.clone_url
63
              dest: spec.arguments.parameters.1.value
64
      retryStrategy:
65
        steps: 3
66
        duration: "10s"
67
68
    # Trigger PR preview on pull request
69
    - template:
70
        name: trigger-pr-preview
71
        argoWorkflow:
72
          group: argoproj.io
73
          version: v1alpha1
74
          resource: workflows
75
          operation: submit
76
          source:
77
            resource:
78
              apiVersion: argoproj.io/v1alpha1
79
              kind: Workflow
80
              metadata:
81
                generateName: pr-preview-
82
              spec:
83
                workflowTemplateRef:
84
                  name: pr-preview-workflow
85
                arguments:
86
                  parameters:
87
                    - name: pr-number
88
                      value: ""
89
                    - name: head-sha
90
                      value: ""
91
          parameters:
92
            - src:
93
                dependencyName: pr-dep
94
                dataKey: body.number
95
              dest: spec.arguments.parameters.0.value
96
            - src:
97
                dependencyName: pr-dep
98
                dataKey: body.pull_request.head.sha
99
              dest: spec.arguments.parameters.1.value

1
apiVersion: argoproj.io/v1alpha1
2
kind: CronWorkflow
3
metadata:
4
  name: nightly-tests
5
  namespace: argo
6
spec:
7
  schedule: "0 2 * * *"  # Every day at 2 AM
8
  timezone: "America/New_York"
9
  concurrencyPolicy: Replace
10
  startingDeadlineSeconds: 0
11
  successfulJobsHistoryLimit: 3
12
  failedJobsHistoryLimit: 3
13
14
  workflowSpec:
15
    entrypoint: run-nightly-tests
16
    serviceAccountName: argo-workflow
17
18
    templates:
19
      - name: run-nightly-tests
20
        dag:
21
          tasks:
22
            - name: integration-tests
23
              template: integration-tests
24
            - name: e2e-tests
25
              template: e2e-tests
26
            - name: performance-tests
27
              template: performance-tests
28
              dependencies: [integration-tests, e2e-tests]
29
            - name: notify
30
              template: notify-results
31
              dependencies: [performance-tests]
32
33
      - name: integration-tests
34
        container:
35
          image: company/test-runner:latest
36
          command: [npm, run, test:integration]
37
38
      - name: e2e-tests
39
        container:
40
          image: company/test-runner:latest
41
          command: [npm, run, test:e2e]
42
43
      - name: performance-tests
44
        container:
45
          image: company/test-runner:latest
46
          command: [npm, run, test:performance]
47
48
      - name: notify-results
49
        container:
50
          image: curlimages/curl:latest
51
          command: [sh, -c]
52
          args:
53
            - |
54
              curl -X POST $SLACK_WEBHOOK_URL \
55
                -H 'Content-type: application/json' \
56
                --data '{"text": "Nightly tests completed"}'

1
apiVersion: argoproj.io/v1alpha1
2
kind: WorkflowTemplate
3
metadata:
4
  name: parallel-test-suite
5
  namespace: argo
6
spec:
7
  entrypoint: run-tests
8
9
  templates:
10
    - name: run-tests
11
      dag:
12
        tasks:
13
          - name: setup
14
            template: setup-environment
15
16
          # Fan-out: Run test suites in parallel
17
          - name: unit-tests
18
            template: run-test-suite
19
            dependencies: [setup]
20
            arguments:
21
              parameters:
22
                - name: suite
23
                  value: "unit"
24
25
          - name: integration-tests
26
            template: run-test-suite
27
            dependencies: [setup]
28
            arguments:
29
              parameters:
30
                - name: suite
31
                  value: "integration"
32
33
          - name: e2e-tests
34
            template: run-test-suite
35
            dependencies: [setup]
36
            arguments:
37
              parameters:
38
                - name: suite
39
                  value: "e2e"
40
41
          - name: performance-tests
42
            template: run-test-suite
43
            dependencies: [setup]
44
            arguments:
45
              parameters:
46
                - name: suite
47
                  value: "performance"
48
49
          # Fan-in: Aggregate results
50
          - name: aggregate-results
51
            template: aggregate-test-results
52
            dependencies: [unit-tests, integration-tests, e2e-tests, performance-tests]
53
54
    - name: setup-environment
55
      container:
56
        image: node:18-alpine
57
        command: [sh, -c]
58
        args:
59
          - |
60
            npm ci --cache /tmp/.npm
61
62
    - name: run-test-suite
63
      inputs:
64
        parameters:
65
          - name: suite
66
      outputs:
67
        artifacts:
68
          - name: test-results
69
            path: /workspace/test-results
70
      container:
71
        image: node:18-alpine
72
        command: [sh, -c]
73
        args:
74
          - |
75
            npm run test:{{inputs.parameters.suite}} -- \
76
              --reporter junit \
77
              --output /workspace/test-results/{{inputs.parameters.suite}}.xml
78
79
    - name: aggregate-test-results
80
      container:
81
        image: alpine:latest
82
        command: [sh, -c]
83
        args:
84
          - |
85
            echo "All test suites completed successfully"

1
apiVersion: argoproj.io/v1alpha1
2
kind: WorkflowTemplate
3
metadata:
4
  name: matrix-build
5
  namespace: argo
6
spec:
7
  entrypoint: build-matrix
8
9
  templates:
10
    - name: build-matrix
11
      dag:
12
        tasks:
13
          - name: build
14
            template: build-for-platform
15
            arguments:
16
              parameters:
17
                - name: platform
18
                  value: "{{item.platform}}"
19
                - name: arch
20
                  value: "{{item.arch}}"
21
            withItems:
22
              - { platform: "linux", arch: "amd64" }
23
              - { platform: "linux", arch: "arm64" }
24
              - { platform: "darwin", arch: "amd64" }
25
              - { platform: "darwin", arch: "arm64" }
26
              - { platform: "windows", arch: "amd64" }
27
28
    - name: build-for-platform
29
      inputs:
30
        parameters:
31
          - name: platform
32
          - name: arch
33
      container:
34
        image: golang:1.21
35
        command: [sh, -c]
36
        args:
37
          - |
38
            GOOS={{inputs.parameters.platform}} \
39
            GOARCH={{inputs.parameters.arch}} \
40
            go build -o /output/app-{{inputs.parameters.platform}}-{{inputs.parameters.arch}} .
41
        env:
42
          - name: CGO_ENABLED
43
            value: "0"

1
apiVersion: argoproj.io/v1alpha1
2
kind: WorkflowTemplate
3
metadata:
4
  name: dynamic-parallel
5
  namespace: argo
6
spec:
7
  entrypoint: main
8
9
  templates:
10
    - name: main
11
      steps:
12
        - - name: get-services
13
            template: list-services
14
        - - name: deploy-service
15
            template: deploy
16
            arguments:
17
              parameters:
18
                - name: service-name
19
                  value: "{{item}}"
20
            withParam: "{{steps.get-services.outputs.result}}"
21
22
    - name: list-services
23
      script:
24
        image: python:3.9-alpine
25
        command: [python]
26
        source: |
27
          import json
28
          services = ["api", "worker", "scheduler", "web"]
29
          print(json.dumps(services))
30
31
    - name: deploy
32
      inputs:
33
        parameters:
34
          - name: service-name
35
      container:
36
        image: kubectl:latest
37
        command: [kubectl, apply, -f]
38
        args:
39
          - /manifests/{{inputs.parameters.service-name}}/

1
apiVersion: argoproj.io/v1alpha1
2
kind: WorkflowTemplate
3
metadata:
4
  name: retry-pattern
5
  namespace: argo
6
spec:
7
  entrypoint: main
8
9
  templates:
10
    - name: main
11
      dag:
12
        tasks:
13
          - name: deploy
14
            template: deploy-with-retry
15
            continueOn:
16
              failed: true
17
          - name: notify-failure
18
            template: notify
19
            dependencies: [deploy]
20
            when: "{{tasks.deploy.status}} == Failed"
21
          - name: notify-success
22
            template: notify-success
23
            dependencies: [deploy]
24
            when: "{{tasks.deploy.status}} == Succeeded"
25
26
    - name: deploy-with-retry
27
      retryStrategy:
28
        limit: 3
29
        backoff:
30
          duration: "30s"
31
          factor: 2
32
          maxDuration: "5m"
33
        retryPolicy: "Always"
34
      container:
35
        image: kubectl:latest
36
        command: [kubectl, apply, -f, /manifests/]
37
38
    - name: notify
39
      container:
40
        image: curlimages/curl:latest
41
        command: [sh, -c]
42
        args:
43
          - |
44
            curl -X POST $SLACK_WEBHOOK_URL \
45
              -d '{"text": "Deployment failed after retries"}'
46
47
    - name: notify-success
48
      container:
49
        image: curlimages/curl:latest
50
        command: [sh, -c]
51
        args:
52
          - |
53
            curl -X POST $SLACK_WEBHOOK_URL \
54
              -d '{"text": "Deployment succeeded"}'

1
apiVersion: argoproj.io/v1alpha1
2
kind: Workflow
3
metadata:
4
  name: artifact-passing
5
spec:
6
  entrypoint: main
7
8
  templates:
9
    - name: main
10
      dag:
11
        tasks:
12
          - name: generate
13
            template: generate-artifact
14
          - name: consume
15
            template: consume-artifact
16
            dependencies: [generate]
17
            arguments:
18
              artifacts:
19
                - name: data
20
                  from: "{{tasks.generate.outputs.artifacts.data}}"
21
22
    - name: generate-artifact
23
      container:
24
        image: alpine:latest
25
        command: [sh, -c]
26
        args:
27
          - |
28
            echo '{"key": "value", "count": 42}' > /tmp/data.json
29
      outputs:
30
        artifacts:
31
          - name: data
32
            path: /tmp/data.json
33
34
    - name: consume-artifact
35
      inputs:
36
        artifacts:
37
          - name: data
38
            path: /tmp/data.json
39
      container:
40
        image: alpine:latest
41
        command: [sh, -c]
42
        args:
43
          - |
44
            cat /tmp/data.json
45
            echo "Processing data..."

1
apiVersion: argoproj.io/v1alpha1
2
kind: Workflow
3
metadata:
4
  name: parameter-passing
5
spec:
6
  entrypoint: main
7
8
  templates:
9
    - name: main
10
      dag:
11
        tasks:
12
          - name: get-version
13
            template: determine-version
14
          - name: build
15
            template: build-app
16
            dependencies: [get-version]
17
            arguments:
18
              parameters:
19
                - name: version
20
                  value: "{{tasks.get-version.outputs.parameters.version}}"
21
22
    - name: determine-version
23
      container:
24
        image: alpine:latest
25
        command: [sh, -c]
26
        args:
27
          - |
28
            VERSION=$(date +%Y%m%d)-$(git rev-parse --short HEAD)
29
            echo $VERSION > /tmp/version.txt
30
      outputs:
31
        parameters:
32
          - name: version
33
            valueFrom:
34
              path: /tmp/version.txt
35
36
    - name: build-app
37
      inputs:
38
        parameters:
39
          - name: version
40
      container:
41
        image: docker:latest
42
        command: [sh, -c]
43
        args:
44
          - |
45
            echo "Building version: {{inputs.parameters.version}}"
46
            docker build -t app:{{inputs.parameters.version}} .