Initial commit: Full Crawl API implementation

k8s/worker.yaml

@@ -0,0 +1,69 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: worker
+  namespace: crawlapi
+spec:
+  replicas: 5
+  selector:
+    matchLabels:
+      app: worker
+  template:
+    metadata:
+      labels:
+        app: worker
+    spec:
+      containers:
+      - name: worker
+        image: crawlapi/worker:latest
+        env:
+        - name: DATABASE_URL
+          value: "postgres://$(POSTGRES_USER):$(POSTGRES_PASSWORD)@postgres:5432/crawlapi"
+        - name: REDIS_URL
+          value: "redis://redis:6379"
+        - name: S3_ENDPOINT
+          value: "http://minio:9000"
+        - name: S3_BUCKET
+          value: "crawlapi"
+        - name: PLAYWRIGHT_SCRIPT_PATH
+          value: "/app/playwright/pool.js"
+        - name: BROWSER_POOL_SIZE
+          value: "5"
+        - name: MAX_PAGES_PER_BROWSER
+          value: "10"
+        envFrom:
+        - secretRef:
+            name: crawlapi-secrets
+        resources:
+          requests:
+            memory: "512Mi"
+            cpu: "500m"
+          limits:
+            memory: "2Gi"
+            cpu: "2000m"
+---
+apiVersion: autoscaling/v2
+kind: HorizontalPodAutoscaler
+metadata:
+  name: worker-hpa
+  namespace: crawlapi
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: worker
+  minReplicas: 3
+  maxReplicas: 20
+  metrics:
+  - type: Resource
+    resource:
+      name: cpu
+      target:
+        type: Utilization
+        averageUtilization: 70
+  - type: Resource
+    resource:
+      name: memory
+      target:
+        type: Utilization
+        averageUtilization: 80