Initial commit: Full Crawl API implementation
This commit is contained in:
141
.github/workflows/ci.yml
vendored
Normal file
141
.github/workflows/ci.yml
vendored
Normal file
@@ -0,0 +1,141 @@
|
||||
name: CI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, develop]
|
||||
pull_request:
|
||||
branches: [main]
|
||||
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
DATABASE_URL: postgres://crawlapi:crawlapi@localhost:5432/crawlapi
|
||||
|
||||
jobs:
|
||||
test:
|
||||
name: Test
|
||||
runs-on: ubuntu-latest
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16-alpine
|
||||
env:
|
||||
POSTGRES_USER: crawlapi
|
||||
POSTGRES_PASSWORD: crawlapi
|
||||
POSTGRES_DB: crawlapi
|
||||
options: >-
|
||||
--health-cmd pg_isready
|
||||
--health-interval 10s
|
||||
--health-timeout 5s
|
||||
--health-retries 5
|
||||
ports:
|
||||
- 5432:5432
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
options: >-
|
||||
--health-cmd "redis-cli ping"
|
||||
--health-interval 10s
|
||||
--health-timeout 5s
|
||||
--health-retries 5
|
||||
ports:
|
||||
- 6379:6379
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Rust
|
||||
uses: dtolnay/rust-action@stable
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: '20'
|
||||
|
||||
- name: Cache cargo
|
||||
uses: Swatinem/rust-cache@v2
|
||||
|
||||
- name: Install sqlx-cli
|
||||
run: cargo install sqlx-cli --no-default-features --features native-tls,postgres
|
||||
|
||||
- name: Run migrations
|
||||
run: sqlx migrate run --source crates/db/migrations
|
||||
|
||||
- name: Check formatting
|
||||
run: cargo fmt --all -- --check
|
||||
|
||||
- name: Run clippy
|
||||
run: cargo clippy --all-targets --all-features -- -D warnings
|
||||
|
||||
- name: Run tests
|
||||
run: cargo test --workspace
|
||||
|
||||
- name: Audit dependencies
|
||||
run: cargo install cargo-audit && cargo audit
|
||||
|
||||
- name: Build release
|
||||
run: cargo build --release
|
||||
|
||||
- name: Build Docker images
|
||||
run: |
|
||||
docker build -f Dockerfile.api -t crawlapi/api:test .
|
||||
docker build -f Dockerfile.worker -t crawlapi/worker:test .
|
||||
|
||||
- name: Install Playwright
|
||||
run: |
|
||||
cd playwright && npm install && npx playwright install chromium
|
||||
|
||||
- name: Run E2E tests
|
||||
run: |
|
||||
cd e2e && npm install && npx playwright test
|
||||
env:
|
||||
API_URL: http://localhost:3000
|
||||
|
||||
build-and-push:
|
||||
name: Build & Push
|
||||
needs: test
|
||||
runs-on: ubuntu-latest
|
||||
if: github.ref == 'refs/heads/main'
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Build and push API
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
context: .
|
||||
file: ./Dockerfile.api
|
||||
push: true
|
||||
tags: |
|
||||
crawlapi/api:${{ github.sha }}
|
||||
crawlapi/api:latest
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
|
||||
- name: Build and push Worker
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
context: .
|
||||
file: ./Dockerfile.worker
|
||||
push: true
|
||||
tags: |
|
||||
crawlapi/worker:${{ github.sha }}
|
||||
crawlapi/worker:latest
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
|
||||
- name: Build and push Frontend
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
context: .
|
||||
file: ./Dockerfile.frontend
|
||||
push: true
|
||||
tags: |
|
||||
crawlapi/frontend:${{ github.sha }}
|
||||
crawlapi/frontend:latest
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
88
.github/workflows/deploy.yml
vendored
Normal file
88
.github/workflows/deploy.yml
vendored
Normal file
@@ -0,0 +1,88 @@
|
||||
name: Deploy
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
tags: ['v*']
|
||||
|
||||
env:
|
||||
KUBECONFIG: ${{ github.workspace }}/kubeconfig
|
||||
|
||||
jobs:
|
||||
deploy-staging:
|
||||
name: Deploy to Staging
|
||||
runs-on: ubuntu-latest
|
||||
environment: staging
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Setup kubectl
|
||||
uses: azure/setup-kubectl@v4
|
||||
with:
|
||||
version: 'v1.29.0'
|
||||
|
||||
- name: Setup Helm
|
||||
uses: azure/setup-helm@v4
|
||||
with:
|
||||
version: '3.14.0'
|
||||
|
||||
- name: Configure kubectl
|
||||
run: |
|
||||
echo "${{ secrets.KUBE_CONFIG_STAGING }}" | base64 -d > kubeconfig
|
||||
|
||||
- name: Deploy to staging
|
||||
run: |
|
||||
kubectl set image deployment/api api=crawlapi/api:${{ github.sha }} -n crawlapi-staging
|
||||
kubectl set image deployment/worker worker=crawlapi/worker:${{ github.sha }} -n crawlapi-staging
|
||||
kubectl set image deployment/frontend frontend=crawlapi/frontend:${{ github.sha }} -n crawlapi-staging
|
||||
kubectl rollout status deployment/api -n crawlapi-staging --timeout=300s
|
||||
kubectl rollout status deployment/worker -n crawlapi-staging --timeout=300s
|
||||
|
||||
- name: Run smoke tests
|
||||
run: |
|
||||
curl -sf https://staging.crawlapi.dev/metrics || exit 1
|
||||
curl -sf -X POST https://staging.crawlapi.dev/api/content \
|
||||
-H "x-api-key: ${{ secrets.STAGING_API_KEY }}" \
|
||||
-d '{"url":"https://example.com"}' || exit 1
|
||||
|
||||
deploy-production:
|
||||
name: Deploy to Production
|
||||
needs: deploy-staging
|
||||
runs-on: ubuntu-latest
|
||||
environment: production
|
||||
if: startsWith(github.ref, 'refs/tags/v')
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Setup kubectl
|
||||
uses: azure/setup-kubectl@v4
|
||||
with:
|
||||
version: 'v1.29.0'
|
||||
|
||||
- name: Configure kubectl
|
||||
run: |
|
||||
echo "${{ secrets.KUBE_CONFIG_PRODUCTION }}" | base64 -d > kubeconfig
|
||||
|
||||
- name: Deploy to production
|
||||
run: |
|
||||
kubectl set image deployment/api api=crawlapi/api:${{ github.sha }} -n crawlapi
|
||||
kubectl set image deployment/worker worker=crawlapi/worker:${{ github.sha }} -n crawlapi
|
||||
kubectl set image deployment/frontend frontend=crawlapi/frontend:${{ github.sha }} -n crawlapi
|
||||
kubectl rollout status deployment/api -n crawlapi --timeout=300s
|
||||
kubectl rollout status deployment/worker -n crawlapi --timeout=300s
|
||||
|
||||
- name: Verify deployment
|
||||
run: |
|
||||
kubectl get pods -n crawlapi
|
||||
curl -sf https://api.crawlapi.dev/metrics || exit 1
|
||||
|
||||
- name: Notify on failure
|
||||
if: failure()
|
||||
uses: slackapi/slack-github-action@v1
|
||||
with:
|
||||
payload: |
|
||||
{
|
||||
"text": "🚨 Production deploy failed for Crawl API ${{ github.sha }}"
|
||||
}
|
||||
env:
|
||||
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
|
||||
Reference in New Issue
Block a user