Skip to content

Commit f8a5f51

Browse files
YilingCAIYilingCAI
committed
refactor logging, devops CICD using shell scritps
1 parent 81e93c3 commit f8a5f51

63 files changed

Lines changed: 10011 additions & 1297 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/ci.yml‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -146,6 +146,38 @@ jobs:
146146
--maxfail=5 \
147147
--tb=short
148148
149+
- name: Run integration tests
150+
working-directory: backend
151+
env:
152+
DATABASE_USER: postgres
153+
DATABASE_PASSWORD: postgres
154+
DATABASE_HOST: localhost
155+
DATABASE_PORT: 5432
156+
DATABASE_NAME: testdb
157+
JWT_SECRET_KEY: test-secret-key-for-ci
158+
JWT_ALGORITHM: HS256
159+
JWT_EXPIRE_MINUTES: 60
160+
ENVIRONMENT: test
161+
run: |
162+
pytest tests/integration -v --tb=short
163+
continue-on-error: true
164+
165+
- name: Run E2E tests
166+
working-directory: backend
167+
env:
168+
DATABASE_USER: postgres
169+
DATABASE_PASSWORD: postgres
170+
DATABASE_HOST: localhost
171+
DATABASE_PORT: 5432
172+
DATABASE_NAME: testdb
173+
JWT_SECRET_KEY: test-secret-key-for-ci
174+
JWT_ALGORITHM: HS256
175+
JWT_EXPIRE_MINUTES: 60
176+
ENVIRONMENT: test
177+
run: |
178+
pytest tests/e2e -v --tb=short
179+
continue-on-error: true
180+
149181
- name: Upload coverage to Codecov
150182
uses: codecov/codecov-action@v3
151183
with:

‎.github/workflows/config.json‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,3 @@
1-
# GitHub Actions Environment Configuration
2-
# Copy this to .github/workflows/config.json and customize
3-
41
{
52
"environments": {
63
"dev": {

‎.github/workflows/deploy.yml‎

Lines changed: 150 additions & 119 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ permissions:
4141

4242
env:
4343
AWS_REGION: us-east-1
44-
TF_VERSION: 1.5.0
44+
PROJECT_ROOT: .
4545

4646
jobs:
4747
validate:
@@ -57,29 +57,32 @@ jobs:
5757
run: |
5858
echo "Validating deployment inputs..."
5959
60-
# Validate environment
6160
if [[ ! "${{ inputs.environment }}" =~ ^(dev|staging|prod)$ ]]; then
62-
echo "Error: Invalid environment. Must be dev, staging, or prod"
61+
echo "❌ Error: Invalid environment. Must be dev, staging, or prod"
6362
exit 1
6463
fi
6564
66-
echo "✓ Environment: ${{ inputs.environment }}"
67-
echo "✓ Image tag: ${{ inputs.image-tag }}"
65+
if [ -z "${{ inputs.image-tag }}" ]; then
66+
echo "❌ Error: image-tag is required"
67+
exit 1
68+
fi
69+
70+
echo "✅ Validation passed"
6871
echo "proceed=true" >> $GITHUB_OUTPUT
6972
7073
approval:
71-
name: Approval Gate
74+
name: Approval Gate - ${{ inputs.environment }}
7275
runs-on: ubuntu-latest
7376
if: inputs.require-approval
7477
environment:
7578
name: ${{ inputs.environment }}-approval
7679
needs: [validate]
7780
steps:
7881
- name: Waiting for approval
79-
run: echo "✓ Deployment approved by ${{ github.actor }}"
82+
run: echo "✅ Deployment approved by ${{ github.actor }}"
8083

81-
terraform:
82-
name: Terraform - ${{ inputs.environment }}
84+
terraform-plan:
85+
name: Terraform Plan - ${{ inputs.environment }}
8386
runs-on: ubuntu-latest
8487
if: inputs.run-terraform && needs.validate.outputs.proceed == 'true'
8588
needs: [validate, approval]
@@ -88,143 +91,171 @@ jobs:
8891
steps:
8992
- uses: actions/checkout@v4
9093

91-
- name: AWS Authentication
92-
uses: ./.github/actions/aws-auth
94+
- name: Configure AWS credentials
95+
uses: aws-actions/configure-aws-credentials@v4
9396
with:
94-
role-arn: ${{ secrets.AWS_ROLE_TO_ASSUME }}
97+
role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
9598
aws-region: ${{ env.AWS_REGION }}
9699

97-
- name: Terraform Validate
98-
uses: ./.github/actions/terraform/validate
100+
- name: Setup Terraform
101+
uses: hashicorp/setup-terraform@v2
99102
with:
100-
working-directory: ./infra
101-
terraform-version: ${{ env.TF_VERSION }}
103+
terraform_version: 1.5.0
102104

103105
- name: Terraform Plan
104-
uses: ./.github/actions/terraform/plan
105-
with:
106-
working-directory: ./infra
107-
environment: ${{ inputs.environment }}
108-
aws-region: ${{ env.AWS_REGION }}
109-
state-bucket: ${{ secrets.TERRAFORM_STATE_BUCKET }}
110-
state-lock-table: ${{ secrets.TERRAFORM_LOCK_TABLE }}
106+
env:
107+
ENV: ${{ inputs.environment }}
108+
TERRAFORM_STATE_BUCKET: ${{ secrets.TERRAFORM_STATE_BUCKET }}
109+
TERRAFORM_LOCK_TABLE: ${{ secrets.TERRAFORM_LOCK_TABLE }}
110+
run: |
111+
make tf-plan
111112
112-
- name: Terraform Apply
113-
if: ${{ inputs.run-terraform }}
114-
uses: ./.github/actions/terraform/apply
113+
- name: Upload plan artifact
114+
uses: actions/upload-artifact@v3
115115
with:
116-
working-directory: ./infra
117-
environment: ${{ inputs.environment }}
118-
aws-region: ${{ env.AWS_REGION }}
119-
state-bucket: ${{ secrets.TERRAFORM_STATE_BUCKET }}
120-
state-lock-table: ${{ secrets.TERRAFORM_LOCK_TABLE }}
121-
plan-artifact-path: ./infra/tfplan.${{ inputs.environment }}
116+
name: tfplan-${{ inputs.environment }}
117+
path: /tmp/tf-plans/tfplan.${{ inputs.environment }}
118+
retention-days: 1
122119

123-
deploy:
124-
name: ECS Deployment - ${{ inputs.environment }}
120+
terraform-apply:
121+
name: Terraform Apply - ${{ inputs.environment }}
125122
runs-on: ubuntu-latest
126-
if: |
127-
!inputs.terraform-only &&
128-
needs.validate.outputs.proceed == 'true'
129-
needs: [validate, approval, terraform]
123+
if: inputs.run-terraform && needs.validate.outputs.proceed == 'true'
124+
needs: [validate, approval, terraform-plan]
130125
environment:
131126
name: ${{ inputs.environment }}
132127
steps:
133128
- uses: actions/checkout@v4
134129

135-
- name: AWS Authentication
136-
uses: ./.github/actions/aws-auth
130+
- name: Configure AWS credentials
131+
uses: aws-actions/configure-aws-credentials@v4
137132
with:
138-
role-arn: ${{ secrets.AWS_ROLE_TO_ASSUME }}
133+
role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
139134
aws-region: ${{ env.AWS_REGION }}
140135

141-
- name: Get ECR Registry
142-
id: ecr
143-
run: |
144-
REGISTRY=$(aws ecr describe-repositories \
145-
--repository-names mypythonproject1/backend \
146-
--query 'repositories[0].repositoryUri' \
147-
--output text | cut -d'/' -f1)
148-
echo "registry=$REGISTRY" >> $GITHUB_OUTPUT
149-
150-
- name: Deploy Backend
151-
uses: ./.github/actions/ecs/deploy
136+
- name: Setup Terraform
137+
uses: hashicorp/setup-terraform@v2
152138
with:
153-
cluster: mypythonproject1-cluster-${{ inputs.environment }}
154-
service: backend-service-${{ inputs.environment }}
155-
task-definition: ./infra/modules/ecs/task-definition-backend.json
156-
container-name: backend
157-
image: ${{ steps.ecr.outputs.registry }}/mypythonproject1/backend:${{ inputs.image-tag }}
158-
wait-for-stability: "true"
159-
timeout-seconds: "600"
160-
enable-smoke-tests: "true"
161-
162-
- name: Deploy Frontend
163-
uses: ./.github/actions/ecs/deploy
139+
terraform_version: 1.5.0
140+
141+
- name: Download plan artifact
142+
uses: actions/download-artifact@v3
164143
with:
165-
cluster: mypythonproject1-cluster-${{ inputs.environment }}
166-
service: frontend-service-${{ inputs.environment }}
167-
task-definition: ./infra/modules/ecs/task-definition-frontend.json
168-
container-name: frontend
169-
image: ${{ steps.ecr.outputs.registry }}/mypythonproject1/frontend:${{ inputs.image-tag }}
170-
wait-for-stability: "true"
171-
timeout-seconds: "600"
172-
enable-smoke-tests: "true"
173-
174-
- name: Post-Deployment Report
175-
if: success()
144+
name: tfplan-${{ inputs.environment }}
145+
path: /tmp/tf-plans/
146+
147+
- name: Terraform Apply
148+
env:
149+
ENV: ${{ inputs.environment }}
150+
TERRAFORM_STATE_BUCKET: ${{ secrets.TERRAFORM_STATE_BUCKET }}
151+
TERRAFORM_LOCK_TABLE: ${{ secrets.TERRAFORM_LOCK_TABLE }}
176152
run: |
177-
echo "## ✅ Deployment Successful" >> $GITHUB_STEP_SUMMARY
178-
echo "- **Environment**: ${{ inputs.environment }}" >> $GITHUB_STEP_SUMMARY
179-
echo "- **Image Tag**: ${{ inputs.image-tag }}" >> $GITHUB_STEP_SUMMARY
180-
echo "- **Deployed by**: ${{ github.actor }}" >> $GITHUB_STEP_SUMMARY
181-
echo "- **Timestamp**: $(date -u +'%Y-%m-%d %H:%M:%S UTC')" >> $GITHUB_STEP_SUMMARY
153+
chmod +x scripts/terraform-apply.sh
154+
make tf-apply
155+
156+
docker-build:
157+
name: Docker Build - ${{ inputs.environment }}
158+
runs-on: ubuntu-latest
159+
if: needs.validate.outputs.proceed == 'true' && !inputs.terraform-only
160+
needs: [validate]
161+
permissions:
162+
contents: read
163+
packages: write
164+
steps:
165+
- uses: actions/checkout@v4
166+
167+
- name: Set up Docker Buildx
168+
uses: docker/setup-buildx-action@v2
169+
170+
- name: Configure AWS credentials
171+
uses: aws-actions/configure-aws-credentials@v4
172+
with:
173+
role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
174+
aws-region: ${{ env.AWS_REGION }}
175+
176+
- name: Get AWS Account ID
177+
id: account
178+
run: |
179+
ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text)
180+
echo "account_id=$ACCOUNT_ID" >> $GITHUB_OUTPUT
181+
182+
- name: Docker Build & Push
183+
env:
184+
ENV: ${{ inputs.environment }}
185+
IMAGE_TAG: ${{ inputs.image-tag }}
186+
AWS_ACCOUNT_ID: ${{ steps.account.outputs.account_id }}
187+
run: |
188+
chmod +x scripts/docker-build.sh
189+
make docker-push
182190
183-
rollback:
184-
name: Rollback
191+
ecs-deploy:
192+
name: ECS Deploy - ${{ inputs.environment }}
185193
runs-on: ubuntu-latest
186-
if: failure() && !inputs.terraform-only
187-
needs: [deploy]
194+
if: needs.validate.outputs.proceed == 'true' && !inputs.terraform-only
195+
needs: [validate, approval, terraform-apply, docker-build]
188196
environment:
189197
name: ${{ inputs.environment }}
190198
steps:
191-
- name: AWS Authentication
192-
uses: ./.github/actions/aws-auth
199+
- uses: actions/checkout@v4
200+
201+
- name: Configure AWS credentials
202+
uses: aws-actions/configure-aws-credentials@v4
193203
with:
194-
role-arn: ${{ secrets.AWS_ROLE_TO_ASSUME }}
204+
role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
195205
aws-region: ${{ env.AWS_REGION }}
196206

197-
- name: Automatic Rollback
207+
- name: Get AWS Account ID
208+
id: account
209+
run: |
210+
ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text)
211+
echo "account_id=$ACCOUNT_ID" >> $GITHUB_OUTPUT
212+
213+
- name: ECS Deploy
214+
env:
215+
ENV: ${{ inputs.environment }}
216+
IMAGE_TAG: ${{ inputs.image-tag }}
217+
AWS_ACCOUNT_ID: ${{ steps.account.outputs.account_id }}
218+
run: |
219+
chmod +x scripts/ecs-deploy.sh
220+
make ecs-deploy
221+
222+
post-deployment:
223+
name: Post-Deployment - ${{ inputs.environment }}
224+
runs-on: ubuntu-latest
225+
if: success()
226+
needs: [validate, ecs-deploy]
227+
steps:
228+
- uses: actions/checkout@v4
229+
230+
- name: Deployment Summary
231+
run: |
232+
echo "## ✅ Deployment Successful" >> $GITHUB_STEP_SUMMARY
233+
echo "" >> $GITHUB_STEP_SUMMARY
234+
echo "| Attribute | Value |" >> $GITHUB_STEP_SUMMARY
235+
echo "|-----------|-------|" >> $GITHUB_STEP_SUMMARY
236+
echo "| Environment | ${{ inputs.environment }} |" >> $GITHUB_STEP_SUMMARY
237+
echo "| Image Tag | ${{ inputs.image-tag }} |" >> $GITHUB_STEP_SUMMARY
238+
echo "| Branch | ${{ github.ref_name }} |" >> $GITHUB_STEP_SUMMARY
239+
echo "| Commit | ${{ github.sha }} |" >> $GITHUB_STEP_SUMMARY
240+
echo "| Deployed by | ${{ github.actor }} |" >> $GITHUB_STEP_SUMMARY
241+
echo "| Timestamp | $(date -u +'%Y-%m-%d %H:%M:%S UTC') |" >> $GITHUB_STEP_SUMMARY
242+
243+
notify-failure:
244+
name: Notify Failure
245+
runs-on: ubuntu-latest
246+
if: failure()
247+
needs:
248+
[
249+
validate,
250+
terraform-plan,
251+
terraform-apply,
252+
docker-build,
253+
ecs-deploy,
254+
]
255+
steps:
256+
- name: Deployment Failed
198257
run: |
199-
echo "⚠ Deployment failed, initiating automatic rollback..."
200-
201-
CLUSTER="mypythonproject1-cluster-${{ inputs.environment }}"
202-
SERVICES=("backend-service-${{ inputs.environment }}" "frontend-service-${{ inputs.environment }}")
203-
204-
for SERVICE in "${SERVICES[@]}"; do
205-
echo "Rolling back $SERVICE..."
206-
207-
CURRENT_TD=$(aws ecs describe-services \
208-
--cluster "$CLUSTER" \
209-
--services "$SERVICE" \
210-
--query 'services[0].taskDefinition' \
211-
--output text)
212-
213-
PREVIOUS_TD=$(aws ecs list-task-definitions \
214-
--family-prefix "${SERVICE}" \
215-
--sort DESCENDING \
216-
--query "taskDefinitionArns[1]" \
217-
--output text)
218-
219-
if [ -n "$PREVIOUS_TD" ] && [ "$PREVIOUS_TD" != "None" ]; then
220-
aws ecs update-service \
221-
--cluster "$CLUSTER" \
222-
--service "$SERVICE" \
223-
--task-definition "$PREVIOUS_TD" \
224-
--force-new-deployment
225-
226-
echo "✓ Rollback initiated for $SERVICE"
227-
else
228-
echo "⚠ No previous task definition found for $SERVICE"
229-
fi
230-
done
258+
echo "## ❌ Deployment Failed" >> $GITHUB_STEP_SUMMARY
259+
echo "Environment: ${{ inputs.environment }}" >> $GITHUB_STEP_SUMMARY
260+
echo "Image Tag: ${{ inputs.image-tag }}" >> $GITHUB_STEP_SUMMARY
261+
exit 1

0 commit comments

Comments
 (0)