Initial commit: Full Crawl API implementation

crates/db/Cargo.toml

@@ -0,0 +1,17 @@
+[package]
+name = "db"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+shared = { path = "../shared" }
+sqlx = { workspace = true }
+tokio = { workspace = true }
+uuid = { workspace = true }
+chrono = { workspace = true }
+thiserror = { workspace = true }
+anyhow = { workspace = true }
+tracing = { workspace = true }
+
+[dev-dependencies]
+serde_json = { workspace = true }

crates/db/migrations/001_init.sql

@@ -0,0 +1,38 @@
+CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
+
+CREATE TABLE users (
+    id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
+    email VARCHAR(255) UNIQUE NOT NULL,
+    password_hash VARCHAR(255),
+    google_id VARCHAR(255) UNIQUE,
+    credits BIGINT NOT NULL DEFAULT 30,
+    tier VARCHAR(50) NOT NULL DEFAULT 'free',
+    created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+    updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+CREATE TABLE api_keys (
+    id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
+    user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
+    key_hash VARCHAR(255) UNIQUE NOT NULL,
+    name VARCHAR(255) NOT NULL DEFAULT 'Default',
+    last_used_at TIMESTAMPTZ,
+    created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+CREATE TABLE usage_logs (
+    id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
+    user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
+    api_key_id UUID NOT NULL REFERENCES api_keys(id) ON DELETE CASCADE,
+    endpoint VARCHAR(100) NOT NULL,
+    url TEXT NOT NULL,
+    status VARCHAR(50) NOT NULL,
+    credits_used BIGINT NOT NULL DEFAULT 1,
+    duration_ms BIGINT NOT NULL DEFAULT 0,
+    created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+CREATE INDEX idx_api_keys_user_id ON api_keys(user_id);
+CREATE INDEX idx_api_keys_key_hash ON api_keys(key_hash);
+CREATE INDEX idx_usage_logs_user_id ON usage_logs(user_id);
+CREATE INDEX idx_usage_logs_created_at ON usage_logs(created_at);

crates/db/migrations/002_oauth_and_subscriptions.sql

@@ -0,0 +1,27 @@
+CREATE TABLE oauth_accounts (
+    id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
+    user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
+    provider VARCHAR(50) NOT NULL,
+    provider_account_id VARCHAR(255) NOT NULL,
+    created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+    UNIQUE(provider, provider_account_id)
+);
+
+CREATE INDEX idx_oauth_accounts_user_id ON oauth_accounts(user_id);
+
+CREATE TABLE subscriptions (
+    id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
+    user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
+    stripe_customer_id VARCHAR(255),
+    stripe_subscription_id VARCHAR(255),
+    stripe_price_id VARCHAR(255),
+    status VARCHAR(50) NOT NULL DEFAULT 'incomplete',
+    tier VARCHAR(50) NOT NULL DEFAULT 'free',
+    current_period_start TIMESTAMPTZ,
+    current_period_end TIMESTAMPTZ,
+    created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+    updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+CREATE INDEX idx_subscriptions_user_id ON subscriptions(user_id);
+CREATE INDEX idx_subscriptions_stripe_customer ON subscriptions(stripe_customer_id);

crates/db/migrations/003_teams.sql

@@ -0,0 +1,21 @@
+CREATE TABLE teams (
+    id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
+    name VARCHAR(255) NOT NULL,
+    slug VARCHAR(255) UNIQUE NOT NULL,
+    owner_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
+    created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+    updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+CREATE TABLE team_members (
+    id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
+    team_id UUID NOT NULL REFERENCES teams(id) ON DELETE CASCADE,
+    user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
+    role VARCHAR(50) NOT NULL DEFAULT 'member',
+    created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+    UNIQUE(team_id, user_id)
+);
+
+CREATE INDEX idx_teams_owner ON teams(owner_id);
+CREATE INDEX idx_team_members_team ON team_members(team_id);
+CREATE INDEX idx_team_members_user ON team_members(user_id);

crates/db/src/connection.rs

@@ -0,0 +1,7 @@
+use sqlx::PgPool;
+
+pub type DbPool = PgPool;
+
+pub async fn create_pool(database_url: &str) -> Result<DbPool, sqlx::Error> {
+    PgPool::connect(database_url).await
+}

crates/db/src/lib.rs

@@ -0,0 +1,4 @@
+pub mod connection;
+pub mod repos;
+
+pub use connection::DbPool;

crates/db/src/repos/api_keys.rs

@@ -0,0 +1,64 @@
+use shared::models::ApiKey;
+use sqlx::PgPool;
+use uuid::Uuid;
+
+pub async fn find_by_key_hash(pool: &PgPool, key_hash: &str) -> Result<Option<ApiKey>, sqlx::Error> {
+    sqlx::query_as::<_, ApiKey>(
+        r#"SELECT id, user_id, key_hash, name, last_used_at, created_at FROM api_keys WHERE key_hash = $1"#,
+    )
+    .bind(key_hash)
+    .fetch_optional(pool)
+    .await
+}
+
+pub async fn create(
+    pool: &PgPool,
+    user_id: Uuid,
+    key_hash: &str,
+    name: &str,
+) -> Result<ApiKey, sqlx::Error> {
+    sqlx::query_as::<_, ApiKey>(
+        r#"INSERT INTO api_keys (id, user_id, key_hash, name)
+           VALUES ($1, $2, $3, $4)
+           RETURNING id, user_id, key_hash, name, last_used_at, created_at"#,
+    )
+    .bind(Uuid::new_v4())
+    .bind(user_id)
+    .bind(key_hash)
+    .bind(name)
+    .fetch_one(pool)
+    .await
+}
+
+pub async fn list_by_user(pool: &PgPool, user_id: Uuid) -> Result<Vec<ApiKey>, sqlx::Error> {
+    sqlx::query_as::<_, ApiKey>(
+        r#"SELECT id, user_id, key_hash, name, last_used_at, created_at FROM api_keys WHERE user_id = $1 ORDER BY created_at DESC"#,
+    )
+    .bind(user_id)
+    .fetch_all(pool)
+    .await
+}
+
+pub async fn update_last_used(pool: &PgPool, id: Uuid) -> Result<(), sqlx::Error> {
+    sqlx::query(
+        r#"UPDATE api_keys SET last_used_at = $1 WHERE id = $2"#,
+    )
+    .bind(chrono::Utc::now())
+    .bind(id)
+    .execute(pool)
+    .await?;
+
+    Ok(())
+}
+
+pub async fn delete_by_id(pool: &PgPool, id: Uuid, user_id: Uuid) -> Result<bool, sqlx::Error> {
+    let result = sqlx::query(
+        r#"DELETE FROM api_keys WHERE id = $1 AND user_id = $2"#,
+    )
+    .bind(id)
+    .bind(user_id)
+    .execute(pool)
+    .await?;
+
+    Ok(result.rows_affected() > 0)
+}

crates/db/src/repos/mod.rs

@@ -0,0 +1,6 @@
+pub mod api_keys;
+pub mod oauth;
+pub mod subscriptions;
+pub mod teams;
+pub mod usage_logs;
+pub mod users;

crates/db/src/repos/oauth.rs

@@ -0,0 +1,37 @@
+use shared::models::OAuthAccount;
+use sqlx::PgPool;
+use uuid::Uuid;
+
+pub async fn find_by_provider(
+    pool: &PgPool,
+    provider: &str,
+    provider_account_id: &str,
+) -> Result<Option<OAuthAccount>, sqlx::Error> {
+    sqlx::query_as::<_, OAuthAccount>(
+        r#"SELECT id, user_id, provider, provider_account_id, created_at 
+           FROM oauth_accounts WHERE provider = $1 AND provider_account_id = $2"#,
+    )
+    .bind(provider)
+    .bind(provider_account_id)
+    .fetch_optional(pool)
+    .await
+}
+
+pub async fn create(
+    pool: &PgPool,
+    user_id: Uuid,
+    provider: &str,
+    provider_account_id: &str,
+) -> Result<OAuthAccount, sqlx::Error> {
+    sqlx::query_as::<_, OAuthAccount>(
+        r#"INSERT INTO oauth_accounts (id, user_id, provider, provider_account_id)
+           VALUES ($1, $2, $3, $4)
+           RETURNING id, user_id, provider, provider_account_id, created_at"#,
+    )
+    .bind(Uuid::new_v4())
+    .bind(user_id)
+    .bind(provider)
+    .bind(provider_account_id)
+    .fetch_one(pool)
+    .await
+}

crates/db/src/repos/subscriptions.rs

@@ -0,0 +1,76 @@
+use shared::models::Subscription;
+use sqlx::PgPool;
+use uuid::Uuid;
+
+pub async fn find_by_user(pool: &PgPool, user_id: Uuid) -> Result<Option<Subscription>, sqlx::Error> {
+    sqlx::query_as::<_, Subscription>(
+        r#"SELECT id, user_id, stripe_customer_id, stripe_subscription_id, stripe_price_id,
+           status, tier, current_period_start, current_period_end, created_at, updated_at
+           FROM subscriptions WHERE user_id = $1 ORDER BY created_at DESC LIMIT 1"#,
+    )
+    .bind(user_id)
+    .fetch_optional(pool)
+    .await
+}
+
+pub async fn find_by_stripe_subscription(
+    pool: &PgPool,
+    stripe_subscription_id: &str,
+) -> Result<Option<Subscription>, sqlx::Error> {
+    sqlx::query_as::<_, Subscription>(
+        r#"SELECT id, user_id, stripe_customer_id, stripe_subscription_id, stripe_price_id,
+           status, tier, current_period_start, current_period_end, created_at, updated_at
+           FROM subscriptions WHERE stripe_subscription_id = $1"#,
+    )
+    .bind(stripe_subscription_id)
+    .fetch_optional(pool)
+    .await
+}
+
+pub async fn create_or_update(
+    pool: &PgPool,
+    user_id: Uuid,
+    stripe_customer_id: Option<&str>,
+    stripe_subscription_id: Option<&str>,
+    stripe_price_id: Option<&str>,
+    status: &str,
+    tier: &str,
+) -> Result<Subscription, sqlx::Error> {
+    sqlx::query_as::<_, Subscription>(
+        r#"INSERT INTO subscriptions (id, user_id, stripe_customer_id, stripe_subscription_id, stripe_price_id, status, tier)
+           VALUES ($1, $2, $3, $4, $5, $6, $7)
+           ON CONFLICT (user_id) DO UPDATE SET
+             stripe_customer_id = EXCLUDED.stripe_customer_id,
+             stripe_subscription_id = EXCLUDED.stripe_subscription_id,
+             stripe_price_id = EXCLUDED.stripe_price_id,
+             status = EXCLUDED.status,
+             tier = EXCLUDED.tier,
+             updated_at = NOW()
+           RETURNING id, user_id, stripe_customer_id, stripe_subscription_id, stripe_price_id,
+                     status, tier, current_period_start, current_period_end, created_at, updated_at"#,
+    )
+    .bind(Uuid::new_v4())
+    .bind(user_id)
+    .bind(stripe_customer_id)
+    .bind(stripe_subscription_id)
+    .bind(stripe_price_id)
+    .bind(status)
+    .bind(tier)
+    .fetch_one(pool)
+    .await
+}
+
+pub async fn update_status(
+    pool: &PgPool,
+    stripe_subscription_id: &str,
+    status: &str,
+) -> Result<(), sqlx::Error> {
+    sqlx::query(
+        r#"UPDATE subscriptions SET status = $1, updated_at = NOW() WHERE stripe_subscription_id = $2"#,
+    )
+    .bind(status)
+    .bind(stripe_subscription_id)
+    .execute(pool)
+    .await?;
+    Ok(())
+}

crates/db/src/repos/teams.rs

@@ -0,0 +1,68 @@
+use shared::models::{Team, TeamMember};
+use sqlx::PgPool;
+use uuid::Uuid;
+
+pub async fn create(pool: &PgPool, name: &str, slug: &str, owner_id: Uuid) -> Result<Team, sqlx::Error> {
+    sqlx::query_as::<_, Team>(
+        r#"INSERT INTO teams (id, name, slug, owner_id)
+           VALUES ($1, $2, $3, $4)
+           RETURNING id, name, slug, owner_id, created_at, updated_at"#,
+    )
+    .bind(Uuid::new_v4())
+    .bind(name)
+    .bind(slug)
+    .bind(owner_id)
+    .fetch_one(pool)
+    .await
+}
+
+pub async fn find_by_slug(pool: &PgPool, slug: &str) -> Result<Option<Team>, sqlx::Error> {
+    sqlx::query_as::<_, Team>(
+        r#"SELECT id, name, slug, owner_id, created_at, updated_at FROM teams WHERE slug = $1"#,
+    )
+    .bind(slug)
+    .fetch_optional(pool)
+    .await
+}
+
+pub async fn find_by_id(pool: &PgPool, id: Uuid) -> Result<Option<Team>, sqlx::Error> {
+    sqlx::query_as::<_, Team>(
+        r#"SELECT id, name, slug, owner_id, created_at, updated_at FROM teams WHERE id = $1"#,
+    )
+    .bind(id)
+    .fetch_optional(pool)
+    .await
+}
+
+pub async fn add_member(pool: &PgPool, team_id: Uuid, user_id: Uuid, role: &str) -> Result<TeamMember, sqlx::Error> {
+    sqlx::query_as::<_, TeamMember>(
+        r#"INSERT INTO team_members (id, team_id, user_id, role)
+           VALUES ($1, $2, $3, $4)
+           RETURNING id, team_id, user_id, role, created_at"#,
+    )
+    .bind(Uuid::new_v4())
+    .bind(team_id)
+    .bind(user_id)
+    .bind(role)
+    .fetch_one(pool)
+    .await
+}
+
+pub async fn list_members(pool: &PgPool, team_id: Uuid) -> Result<Vec<TeamMember>, sqlx::Error> {
+    sqlx::query_as::<_, TeamMember>(
+        r#"SELECT id, team_id, user_id, role, created_at FROM team_members WHERE team_id = $1"#,
+    )
+    .bind(team_id)
+    .fetch_all(pool)
+    .await
+}
+
+pub async fn find_member(pool: &PgPool, team_id: Uuid, user_id: Uuid) -> Result<Option<TeamMember>, sqlx::Error> {
+    sqlx::query_as::<_, TeamMember>(
+        r#"SELECT id, team_id, user_id, role, created_at FROM team_members WHERE team_id = $1 AND user_id = $2"#,
+    )
+    .bind(team_id)
+    .bind(user_id)
+    .fetch_optional(pool)
+    .await
+}

crates/db/src/repos/usage_logs.rs

@@ -0,0 +1,47 @@
+use shared::models::UsageLog;
+use sqlx::PgPool;
+use uuid::Uuid;
+
+pub async fn create(
+    pool: &PgPool,
+    user_id: Uuid,
+    api_key_id: Uuid,
+    endpoint: &str,
+    url: &str,
+    status: &str,
+    credits_used: i64,
+    duration_ms: i64,
+) -> Result<UsageLog, sqlx::Error> {
+    sqlx::query_as::<_, UsageLog>(
+        r#"INSERT INTO usage_logs (id, user_id, api_key_id, endpoint, url, status, credits_used, duration_ms)
+           VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
+           RETURNING id, user_id, api_key_id, endpoint, url, status, credits_used, duration_ms, created_at"#,
+    )
+    .bind(Uuid::new_v4())
+    .bind(user_id)
+    .bind(api_key_id)
+    .bind(endpoint)
+    .bind(url)
+    .bind(status)
+    .bind(credits_used)
+    .bind(duration_ms)
+    .fetch_one(pool)
+    .await
+}
+
+pub async fn list_by_user(
+    pool: &PgPool,
+    user_id: Uuid,
+    limit: i64,
+    offset: i64,
+) -> Result<Vec<UsageLog>, sqlx::Error> {
+    sqlx::query_as::<_, UsageLog>(
+        r#"SELECT id, user_id, api_key_id, endpoint, url, status, credits_used, duration_ms, created_at
+           FROM usage_logs WHERE user_id = $1 ORDER BY created_at DESC LIMIT $2 OFFSET $3"#,
+    )
+    .bind(user_id)
+    .bind(limit)
+    .bind(offset)
+    .fetch_all(pool)
+    .await
+}

crates/db/src/repos/users.rs

@@ -0,0 +1,64 @@
+use shared::models::User;
+use sqlx::PgPool;
+use uuid::Uuid;
+
+pub async fn find_by_email(pool: &PgPool, email: &str) -> Result<Option<User>, sqlx::Error> {
+    sqlx::query_as::<_, User>(
+        r#"SELECT id, email, password_hash, google_id, credits, tier, created_at, updated_at FROM users WHERE email = $1"#,
+    )
+    .bind(email)
+    .fetch_optional(pool)
+    .await
+}
+
+pub async fn find_by_id(pool: &PgPool, id: Uuid) -> Result<Option<User>, sqlx::Error> {
+    sqlx::query_as::<_, User>(
+        r#"SELECT id, email, password_hash, google_id, credits, tier, created_at, updated_at FROM users WHERE id = $1"#,
+    )
+    .bind(id)
+    .fetch_optional(pool)
+    .await
+}
+
+pub async fn create(
+    pool: &PgPool,
+    email: &str,
+    password_hash: Option<&str>,
+    google_id: Option<&str>,
+) -> Result<User, sqlx::Error> {
+    sqlx::query_as::<_, User>(
+        r#"INSERT INTO users (id, email, password_hash, google_id, credits, tier)
+           VALUES ($1, $2, $3, $4, 30, 'free')
+           RETURNING id, email, password_hash, google_id, credits, tier, created_at, updated_at"#,
+    )
+    .bind(Uuid::new_v4())
+    .bind(email)
+    .bind(password_hash)
+    .bind(google_id)
+    .fetch_one(pool)
+    .await
+}
+
+pub async fn deduct_credits(pool: &PgPool, user_id: Uuid, amount: i64) -> Result<bool, sqlx::Error> {
+    let result = sqlx::query(
+        r#"UPDATE users SET credits = credits - $1 WHERE id = $2 AND credits >= $1"#,
+    )
+    .bind(amount)
+    .bind(user_id)
+    .execute(pool)
+    .await?;
+
+    Ok(result.rows_affected() > 0)
+}
+
+pub async fn add_credits(pool: &PgPool, user_id: Uuid, amount: i64) -> Result<(), sqlx::Error> {
+    sqlx::query(
+        r#"UPDATE users SET credits = credits + $1 WHERE id = $2"#,
+    )
+    .bind(amount)
+    .bind(user_id)
+    .execute(pool)
+    .await?;
+
+    Ok(())
+}

crates/db/tests/db_test.rs

@@ -0,0 +1,18 @@
+use shared::models::User;
+
+#[test]
+fn test_user_model_serialization() {
+    let user = User {
+        id: uuid::Uuid::new_v4(),
+        email: "test@example.com".to_string(),
+        password_hash: Some("hash".to_string()),
+        google_id: None,
+        credits: 30,
+        tier: "free".to_string(),
+        created_at: chrono::Utc::now(),
+        updated_at: chrono::Utc::now(),
+    };
+
+    let json = serde_json::to_string(&user).unwrap();
+    assert!(json.contains("test@example.com"));
+}