

A production-ready implementation for fetching, analyzing, and publishing security-focused tech content like "Where was Nvidia's security team?"
| Service | Purpose | Free Tier Available |
|---|---|---|
| NewsAPI | Fetch tech/security news articles | Yes (100 req/day) |
| OpenAI API | AI-powered article analysis & summarization | Pay-per-use |
| PostgreSQL | Article storage & metadata | Self-hosted / Supabase / Neon |
| Redis | Caching & rate limiting | Self-hosted / Upstash |
| Vercel/Netlify | Frontend deployment | Yes |
| Railway/Render | Backend deployment | Yes |
# Required versions
node >= 20.0.0
python >= 3.11
postgresql >= 15
redis >= 7.0
# Verify installations
node --version && python --version && psql --version && redis-cli --version
icarax-blog/
├── backend/ # FastAPI Python backend
│ ├── app/
│ │ ├── api/ # API routes
│ │ ├── core/ # Config, security, database
│ │ ├── models/ # SQLAlchemy models
│ │ ├── schemas/ # Pydantic schemas
│ │ ├── services/ # Business logic
│ │ └── tasks/ # Background jobs
│ ├── tests/
│ ├── requirements.txt
│ └── Dockerfile
├── frontend/ # Next.js TypeScript frontend
│ ├── src/
│ │ ├── app/ # App Router pages
│ │ ├── components/ # React components
│ │ ├── lib/ # Utilities, API client
│ │ └── types/ # TypeScript types
│ ├── package.json
│ └── Dockerfile
├── docker-compose.yml
└── .env.example
cd backend
# Create virtual environment
python -m venv venv
source venv/bin/activate # Windows: venv\Scripts\activate
# Install dependencies
pip install --upgrade pip
pip install -r requirements.txt
requirements.txt
# Core framework
fastapi==0.109.0
uvicorn[standard]==0.27.0
gunicorn==21.2.0
# Database
sqlalchemy==2.0.25
asyncpg==0.29.0
alembic==1.13.1
psycopg2-binary==2.9.9
# Redis & Caching
redis==5.0.1
aioredis==2.0.1
# HTTP & External APIs
httpx==0.26.0
openai==1.12.0
newsapi-python==0.2.7
# Validation & Serialization
pydantic==2.5.3
pydantic-settings==2.1.0
pydantic-extra-types==2.4.3
# Security
python-jose[cryptography]==3.3.0
passlib[bcrypt]==1.7.4
python-multipart==0.0.6
# Background Tasks
celery==5.3.6
flower==2.0.1
# Monitoring & Logging
structlog==24.1.0
prometheus-client==0.19.0
sentry-sdk==1.39.1
# Utilities
python-dotenv==1.0.0
email-validator==2.1.0.post1
tenacity==8.2.3
dateutil==2.8.2
beautifulsoup4==4.12.3
readability-lxml==0.8.1
# Testing
pytest==7.4.4
pytest-asyncio==0.23.3
pytest-cov==4.1.0
httpx==0.26.0
faker==22.0.0
cd frontend
# Install dependencies
npm install
# Or with yarn/pnpm
# yarn install
# pnpm install
package.json
{
"name": "icarax-frontend",
"version": "1.0.0",
"private": true,
"scripts": {
"dev": "next dev",
"build": "next build",
"start": "next start",
"lint": "next lint",
"test": "jest",
"test:watch": "jest --watch",
"type-check": "tsc --noEmit"
},
"dependencies": {
"next": "14.2.0",
"react": "^18.3.0",
"react-dom": "^18.3.0",
"swr": "^2.2.0",
"zod": "^3.22.0",
"date-fns": "^3.6.0",
"lucide-react": "^0.378.0",
"clsx": "^2.1.0",
"tailwind-merge": "^2.2.0",
"@tanstack/react-query": "^5.28.0",
"axios": "^1.6.0",
"react-markdown": "^9.0.0",
"rehype-sanitize": "^6.0.0",
"rehype-raw": "^7.0.0"
},
"devDependencies": {
"@types/node": "^20.12.0",
"@types/react": "^18.3.0",
"@types/react-dom": "^18.3.0",
"typescript": "^5.4.0",
"tailwindcss": "^3.4.0",
"postcss": "^8.4.0",
"autoprefixer": "^10.4.0",
"eslint": "^8.57.0",
"eslint-config-next": "14.2.0",
"jest": "^29.7.0",
"@testing-library/react": "^14.2.0",
"@testing-library/jest-dom": "^6.4.0"
}
}
docker-compose.yml
version: '3.8'
services:
postgres:
image: postgres:15-alpine
environment:
POSTGRES_DB: icarax
POSTGRES_USER: icarax
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-dev_password}
volumes:
- postgres_data:/var/lib/postgresql/data
ports:
- "5432:5432"
healthcheck:
test: ["CMD-SHELL", "pg_isready -U icarax"]
interval: 10s
timeout: 5s
retries: 5
redis:
image: redis:7-alpine
ports:
- "6379:6379"
volumes:
- redis_data:/data
healthcheck:
test: ["CMD", "redis-cli", "ping"]
interval: 10s
timeout: 5s
retries: 5
backend:
build:
context: ./backend
dockerfile: Dockerfile.dev
ports:
- "8000:8000"
environment:
- DATABASE_URL=postgresql+asyncpg://icarax:dev_password@postgres:5432/icarax
- REDIS_URL=redis://redis:6379/0
- ENVIRONMENT=development
env_file:
- ./backend/.env
volumes:
- ./backend:/app
depends_on:
postgres:
condition: service_healthy
redis:
condition: service_healthy
command: uvicorn app.main:app --host 0.0.0.0 --port 8000 --reload
frontend:
build:
context: ./frontend
dockerfile: Dockerfile.dev
ports:
- "3000:3000"
environment:
- NEXT_PUBLIC_API_URL=http://localhost:8000
env_file:
- ./frontend/.env.local
volumes:
- ./frontend:/app
- /app/node_modules
- /app/.next
depends_on:
- backend
command: npm run dev
celery-worker:
build:
context: ./backend
dockerfile: Dockerfile.dev
environment:
- DATABASE_URL=postgresql+asyncpg://icarax:dev_password@postgres:5432/icarax
- REDIS_URL=redis://redis:6379/0
env_file:
- ./backend/.env
volumes:
- ./backend:/app
depends_on:
postgres:
condition: service_healthy
redis:
condition: service_healthy
command: celery -A app.tasks.celery_app worker --loglevel=info
celery-beat:
build:
context: ./backend
dockerfile: Dockerfile.dev
environment:
- DATABASE_URL=postgresql+asyncpg://icarax:dev_password@postgres:5432/icarax
- REDIS_URL=redis://redis:6379/0
env_file:
- ./backend/.env
volumes:
- ./backend:/app
depends_on:
postgres:
condition: service_healthy
redis:
condition: service_healthy
command: celery -A app.tasks.celery_app beat --loglevel=info
volumes:
postgres_data:
redis_data:
backend/app/core/config.py
"""
Application configuration using Pydantic Settings.
Loads from environment variables with validation.
"""
from functools import lru_cache
from typing import List, Optional
from pydantic import Field, field_validator
from pydantic_settings import BaseSettings, SettingsConfigDict
class Settings(BaseSettings):
"""Application settings with environment variable mapping."""
model_config = SettingsConfigDict(
env_file=".env",
env_file_encoding="utf-8",
case_sensitive=True,
extra="ignore"
)
# Application
APP_NAME: str = "ICARAX Tech Blog"
APP_VERSION: str = "1.0.0"
ENVIRONMENT: str = Field(default="development", pattern="^(development|staging|production)$")
DEBUG: bool = False
API_V1_PREFIX: str = "/api/v1"
# Server
HOST: str = "0.0.0.0"
PORT: int = 8000
WORKERS: int = 4
# Database
DATABASE_URL: str = Field(..., description="PostgreSQL async connection string")
DATABASE_POOL_SIZE: int = 20
DATABASE_MAX_OVERFLOW: int = 10
DATABASE_POOL_TIMEOUT: int = 30
# Redis
REDIS_URL: str = Field(default="redis://localhost:6379/0")
REDIS_MAX_CONNECTIONS: int = 50
# Security
SECRET_KEY: str = Field(..., min_length=32, description="JWT signing key")
ALGORITHM: str = "HS256"
ACCESS_TOKEN_EXPIRE_MINUTES: int = 30
REFRESH_TOKEN_EXPIRE_DAYS: int = 7
# CORS
ALLOWED_ORIGINS: List[str] = Field(default=["http://localhost:3000"])
@field_validator("ALLOWED_ORIGINS", mode="before")
@classmethod
def parse_origins(cls, v: str | List[str]) -> List[str]:
if isinstance(v, str):
return [origin.strip() for origin in v.split(",")]
return v
# External APIs
NEWSAPI_KEY: Optional[str] = None
OPENAI_API_KEY: Optional[str] = None
OPENAI_MODEL: str = "gpt-4-turbo-preview"
OPENAI_MAX_TOKENS: int = 4000
# Rate Limiting
RATE_LIMIT_REQUESTS: int = 100
RATE_LIMIT_WINDOW_SECONDS: int = 60
# Monitoring
SENTRY_DSN: Optional[str] = None
LOG_LEVEL: str = "INFO"
# Email (for notifications)
SMTP_HOST: Optional[str] = None
SMTP_PORT: int = 587
SMTP_USER: Optional[str] = None
SMTP_PASSWORD: Optional[str] = None
EMAIL_FROM: str = "noreply@icarax.tech"
@lru_cache
def get_settings() -> Settings:
"""Cached settings instance for dependency injection."""
return Settings()
settings = get_settings()
backend/app/models/article.py
"""
SQLAlchemy models for articles, authors, and categories.
Uses async SQLAlchemy 2.0 with proper indexing.
"""
from datetime import datetime
from enum import Enum as PyEnum
from typing import Optional, List
from uuid import uuid4
from sqlalchemy import (
String, Text, DateTime, ForeignKey, Index, Enum, Boolean,
Integer, func, UniqueConstraint
)
from sqlalchemy.dialects.postgresql import UUID, JSONB
from sqlalchemy.orm import Mapped, mapped_column, relationship, declarative_base
Base = declarative_base()
class ArticleStatus(PyEnum):
"""Article publication status."""
DRAFT = "draft"
PENDING_REVIEW = "pending_review"
PUBLISHED = "published"
ARCHIVED = "archived"
class ArticleSource(PyEnum):
"""Source of the article."""
MANUAL = "manual"
NEWSAPI = "newsapi"
RSS = "rss"
WEBHOOK = "webhook"
class Category(Base):
"""Article categories/tags."""
__tablename__ = "categories"
id: Mapped[UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid4)
name: Mapped[str] = mapped_column(String(100), unique=True, nullable=False, index=True)
slug: Mapped[str] = mapped_column(String(100), unique=True, nullable=False, index=True)
description: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
color: Mapped[str] = mapped_column(String(7), default="#6366f1") # Hex color
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
updated_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True),
server_default=func.now(),
onupdate=func.now()
)
# Relationships
articles: Mapped[List["ArticleCategory"]] = relationship(back_populates="category")
class Author(Base):
"""Article authors."""
__tablename__ = "authors"
id: Mapped[UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid4)
email: Mapped[str] = mapped_column(String(255), unique=True, nullable=False, index=True)
name: Mapped[str] = mapped_column(String(200), nullable=False)
bio: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
avatar_url: Mapped[Optional[str]] = mapped_column(String(500), nullable=True)
twitter_handle: Mapped[Optional[str]] = mapped_column(String(100), nullable=True)
linkedin_url: Mapped[Optional[str]] = mapped_column(String(500), nullable=True)
is_active: Mapped[bool] = mapped_column(Boolean, default=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
updated_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True),
server_default=func.now(),
onupdate=func.now()
)
# Relationships
articles: Mapped[List["Article"]] = relationship(back_populates="author")
class Article(Base):
"""Main article model with full-text search support."""
__tablename__ = "articles"
id: Mapped[UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid4)
# Core content
title: Mapped[str] = mapped_column(String(500), nullable=False, index=True)
slug: Mapped[str] = mapped_column(String(500), unique=True, nullable=False, index=True)
excerpt: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
content: Mapped[str] = mapped_column(Text, nullable=False)
content_html: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
# SEO
meta_title: Mapped[Optional[str]] = mapped_column(String(500), nullable=True)
meta_description: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
og_image_url: Mapped[Optional[str]] = mapped_column(String(500), nullable=True)
# Publication
status: Mapped[ArticleStatus] = mapped_column(
Enum(ArticleStatus),
default=ArticleStatus.DRAFT,
nullable=False,
index=True
)
source: Mapped[ArticleSource] = mapped_column(
Enum(ArticleSource),
default=ArticleSource.MANUAL,
nullable=False
)
source_url: Mapped[Optional[str]] = mapped_column(String(1000), nullable=True)
source_metadata: Mapped[Optional[dict]] = mapped_column(JSONB, nullable=True)
# Author
author_id: Mapped[UUID] = mapped_column(
UUID(as_uuid=True),
ForeignKey("authors.id", ondelete="SET NULL"),
nullable=True,
index=True
)
# Timestamps
published_at: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True), nullable=True, index=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
updated_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True),
server_default=func.now(),
onupdate=func.now()
)
# Analytics
view_count: Mapped[int] = mapped_column(Integer, default=0)
read_time_minutes: Mapped[int] = mapped_column(Integer, default=0)
# AI Analysis
ai_summary: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
ai_key_points: Mapped[Optional[List[str]]] = mapped_column(JSONB, nullable=True)
ai_sentiment: Mapped[Optional[str]] = mapped_column(String(50), nullable=True)
ai_topics: Mapped[Optional[List[str]]] = mapped_column(JSONB, nullable=True)
ai_analyzed_at: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True), nullable=True)
# Relationships
author: Mapped[Optional["Author"]] = relationship(back_populates="articles")
categories: Mapped[List["ArticleCategory"]] = relationship(back_populates="article", cascade="all, delete-orphan")
# Table constraints and indexes
__table_args__ = (
Index("ix_articles_status_published_at", "status", "published_at"),
Index("ix_articles_source_created_at", "source", "created_at"),
Index("ix_articles_title_trgm", "title", postgresql_using="gin", postgresql_ops={"title": "gin_trgm_ops"}),
Index("ix_articles_content_trgm", "content", postgresql_using="gin", postgresql_ops={"content": "gin_trgm_ops"}),
)
class ArticleCategory(Base):
"""Many-to-many relationship between articles and categories."""
__tablename__ = "article_categories"
article_id: Mapped[UUID] = mapped_column(
UUID(as_uuid=True),
ForeignKey("articles.id", ondelete="CASCADE"),
primary_key=True
)
category_id: Mapped[UUID] = mapped_column(
UUID(as_uuid=True),
ForeignKey("categories.id", ondelete="CASCADE"),
primary_key=True
)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
# Relationships
article: Mapped["Article"] = relationship(back_populates="categories")
category: Mapped["Category"] = relationship(back_populates="articles")
__table_args__ = (
Index("ix_article_categories_category_id", "category_id"),
)
backend/app/core/database.py
"""
Async database configuration with connection pooling and health checks.
"""
from contextlib import asynccontextmanager
from typing import AsyncGenerator
from sqlalchemy.ext.asyncio import (
AsyncSession,
async_sessionmaker,
create_async_engine,
AsyncEngine
)
from sqlalchemy.pool import NullPool
from sqlalchemy import text
from app.core.config import get_settings
settings = get_settings()
class DatabaseManager:
"""Manages database connections and sessions."""
def __init__(self):
self._engine: AsyncEngine | None = None
self._session_factory: async_sessionmaker[AsyncSession] | None = None
def initialize(self) -> None:
"""Initialize the database engine and session factory."""
if settings.ENVIRONMENT == "testing":
# Use NullPool for testing to avoid connection issues
poolclass = NullPool
pool_size = 0
max_overflow = 0
else:
poolclass = None
pool_size = settings.DATABASE_POOL_SIZE
max_overflow = settings.DATABASE_MAX_OVERFLOW
self._engine = create_async_engine(
settings.DATABASE_URL,
echo=settings.DEBUG,
poolclass=poolclass,
pool_size=pool_size,
max_overflow=max_overflow,
pool_timeout=settings.DATABASE_POOL_TIMEOUT,
pool_pre_ping=True, # Verify connections before use
)
self._session_factory = async_sessionmaker(
self._engine,
class_=AsyncSession,
expire_on_commit=False,
autoflush=False,
)
async def close(self) -> None:
"""Close all database connections."""
if self._engine:
await self._engine.dispose()
self._engine = None
self._session_factory = None
@asynccontextmanager
async def session(self) -> AsyncGenerator[AsyncSession, None]:
"""Get a database session with automatic cleanup."""
if not self._session_factory:
raise RuntimeError("Database not initialized. Call initialize() first.")
async with self._session_factory() as session:
try:
yield session
await session.commit()
except Exception:
await session.rollback()
raise
finally:
await session.close()
async def health_check(self) -> bool:
"""Check database connectivity."""
if not self._engine:
return False
try:
async with self._engine.connect() as conn:
await conn.execute(text("SELECT 1"))
return True
except Exception:
return False
@property
def engine(self) -> AsyncEngine:
if not self._engine:
raise RuntimeError("Database not initialized")
return self._engine
# Global database manager instance
db_manager = DatabaseManager()
async def get_db() -> AsyncGenerator[AsyncSession, None]:
"""FastAPI dependency for database sessions."""
async with db_manager.session() as session:
yield session
async def init_db() -> None:
"""Initialize database tables."""
from app.models.article import Base
async with db_manager.engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
backend/app/services/news_fetcher.py
"""
Service for fetching security/tech news from multiple sources.
Includes NewsAPI, RSS feeds, and webhook ingestion.
"""
import asyncio
import hashlib
import logging
from datetime import datetime, timedelta
from typing import List, Optional, Dict, Any
from urllib.parse import urlparse
import httpx
from bs4 import BeautifulSoup
from readability import Document
from newsapi import NewsApiClient
from sqlalchemy import select, and_
from sqlalchemy.ext.asyncio import AsyncSession
from app.core.config import get_settings
from app.models.article import Article, ArticleStatus, ArticleSource, Category, ArticleCategory
from app.core.database import get_db
logger = logging.getLogger(__name__)
settings = get_settings()
class NewsFetcherService:
"""Fetches and processes news articles from various sources."""
def __init__(self):
self.newsapi_client: Optional[NewsApiClient] = None
self.http_client = httpx.AsyncClient(
timeout=30.0,
follow_redirects=True,
headers={"User-Agent": "ICARAX-Bot/1.0 (+https://icarax.tech/bot)"}
)
self._initialize_clients()
def _initialize_clients(self) -> None:
"""Initialize API clients."""
if settings.NEWSAPI_KEY:
self.newsapi_client = NewsApiClient(api_key=settings.NEWSAPI_KEY)
async def close(self) -> None:
"""Close HTTP client."""
await self.http_client.aclose()
# --- NewsAPI Integration ---
async def fetch_security_news_newsapi(
self,
days_back: int = 1,
page_size: int = 50
) -> List[Dict[str, Any]]:
"""
Fetch security-related news from NewsAPI.
Args:
days_back: How many days back to fetch
page_size: Number of articles per request (max 100)
Returns:
List of normalized article dictionaries
"""
if not self.newsapi_client:
logger.warning("NewsAPI key not configured")
return []
from_date = (datetime.utcnow() - timedelta(days=days_back)).strftime("%Y-%m-%d")
# Security-focused query
query = (
"security OR cybersecurity OR breach OR vulnerability OR hack OR "
"ransomware OR malware OR phishing OR exploit OR CVE OR "
"data breach OR security team OR CISO OR infosec"
)
try:
response = self.newsapi_client.get_everything(
q=query,
language="en",
sort_by="publishedAt",
from_param=from_date,
page_size=min(page_size, 100),
domains="techcrunch.com,theverge.com,wired.com,arstechnica.com,"
"theregister.com,zdnet.com,computerweekly.com,"
"darkreading.com,threatpost.com,bleepingcomputer.com"
)
articles = response.get("articles", [])
logger.info(f"Fetched {len(articles)} articles from NewsAPI")
return [self._normalize_newsapi_article(a) for a in articles if a.get("content")]
except Exception as e:
logger.error(f"NewsAPI fetch failed: {e}")
return []
def _normalize_newsapi_article(self, article: Dict) -> Dict[str, Any]:
"""Normalize NewsAPI article to our schema."""
return {
"title": article.get("title", "").strip(),
"excerpt": article.get("description", "").strip() if article.get("description") else None,
"source_url": article.get("url", ""),
"source_metadata": {
"source_name": article.get("source", {}).get("name"),
"author": article.get("author"),
"published_at": article.get("publishedAt"),
"url_to_image": article.get("urlToImage"),
},
"content": article.get("content", ""),
}
# --- RSS Feed Integration ---
SECURITY_RSS_FEEDS = [
"https://feeds.feedburner.com/TheHackersNews",
"https://krebsonsecurity.com/feed/",
"https://www.schneier.com/feed/atom/",
"https://blog.trailofbits.com/feed.xml",
"https://googleprojectzero.blogspot.com/feeds/posts/default",
"https://blog.malwarebytes.com/feed/",
]
async def fetch_rss_feeds(self) -> List[Dict[str, Any]]:
"""Fetch articles from security RSS feeds."""
all_articles = []
for feed_url in self.SECURITY_RSS_FEEDS:
try:
articles = await self._fetch_single_rss(feed_url)
all_articles.extend(articles)
except Exception as e:
logger.error(f"Failed to fetch RSS {feed_url}: {e}")
return all_articles
async def _fetch_single_rss(self, feed_url: str) -> List[Dict[str, Any]]:
"""Fetch and parse a single RSS/Atom feed."""
response = await self.http_client.get(feed_url)
response.raise_for_status()
soup = BeautifulSoup(response.content, "xml")
items = soup.find_all(["item", "entry"])
articles = []
for item in items[:20]: # Limit per feed
article = self._parse_rss_item(item, feed_url)
if article:
articles.append(article)
return articles
def _parse_rss_item(self, item, feed_url: str) -> Optional[Dict[str, Any]]:
"""Parse RSS/Atom item to article dict."""
# Handle both RSS and Atom formats
title = item.find(["title", "title"])
link = item.find(["link", "link"])
content = item.find(["content:encoded", "content", "description", "summary"])
pub_date = item.find(["pubDate", "published", "updated"])
author = item.find(["author", "dc:creator"])
if not title or not title.get_text(strip=True):
return None
# Get full content via readability if we have a link
article_url = link.get("href") if link and link.get("href") else link.get_text(strip=True) if link else ""
return {
"title": title.get_text(strip=True),
"excerpt": content.get_text(strip=True)[:500] if content else None,
"source_url": article_url,
"source_metadata": {
"feed_url": feed_url,
"author": author.get_text(strip=True) if author else None,
"published_at": pub_date.get_text(strip=True) if pub_date else None,
},
"content": content.get_text(strip=True) if content else "",
}
# --- Content Extraction ---
async def extract_full_content(self, url: str) -> Optional[str]:
"""
Extract clean article content from URL using readability.
Args:
url: Article URL to extract content from
Returns:
Clean HTML content or None if failed
"""
try:
response = await self.http_client.get(url)
response.raise_for_status()
doc = Document(response.text)
return doc.summary(html_partial=True)
except Exception as e:
logger.error(f"Content extraction failed for {url}: {e}")
return None
# --- Deduplication ---
def _generate_content_hash(self, title: str, source_url: str) -> str:
"""Generate hash for deduplication."""
content = f"{title.lower().strip()}|{source_url}"
return hashlib.sha256(content.encode()).hexdigest()[:32]
async def is_duplicate(self, db: AsyncSession, title: str, source_url: str) -> bool:
"""Check if article already exists."""
content_hash = self._generate_content_hash(title, source_url)
result = await db.execute(
select(Article.id).where(
Article.source_metadata.op("->>")("content_hash") == content_hash
)
)
return result.scalar_one_or_none() is not None
# --- Article Creation ---
async def create_article_from_fetch(
self,
db: AsyncSession,
article_data: Dict[str, Any],
category_slugs: List[str] = None
) -> Optional[Article]:
"""Create article from fetched data with deduplication."""
title = article_data.get("title", "").strip()
source_url = article_data.get("source_url", "").strip()
if not title or not source_url:
return None
# Check duplicate
if await self.is_duplicate(db, title, source_url):
logger.debug(f"Duplicate article skipped: {title[:50]}")
return None
# Generate slug
slug = self._generate_slug(title)
# Ensure unique slug
counter = 1
original_slug = slug
while True:
result = await db.execute(select(Article.id).where(Article.slug == slug))
if not result.scalar_one_or_none():
break
slug = f"{original_slug}-{counter}"
counter += 1
# Extract full content if needed
content = article_data.get("content", "")
if len(content) < 500 and source_url:
full_content = await self.extract_full_content(source_url)
if full_content:
content = full_content
# Calculate read time (avg 200 words/min)
word_count = len(content.split())
read_time = max(1, round(word_count / 200))
# Create article
article = Article(
title=title,
slug=slug,
excerpt=article_data.get("excerpt"),
content=content,
source=ArticleSource.NEWSAPI,
source_url=source_url,
source_metadata={
**article_data.get("source_metadata", {}),
"content_hash": self._generate_content_hash(title, source_url),
"fetched_at": datetime.utcnow().isoformat(),
},
status=ArticleStatus.PENDING_REVIEW,
read_time_minutes=read_time,
)
db.add(article)
await db.flush()
---
## Next Steps
1. **Get API Access** - Sign up at the official website
2. **Try the Examples** - Run the code snippets above
3. **Read the Docs** - Check official documentation
4. **Join Communities** - Discord, Reddit, GitHub discussions
5. **Experiment** - Build something cool!
## Further Reading
- [TechCrunch AI](https://techcrunch.com/category/artificial-intelligence/)
- [The Verge](https://www.theverge.com/technology)
- [Wired AI](https://www.wired.com/tag/artificial-intelligence/)
- [Medium AI](https://medium.com/topic/artificial-intelligence)
**Source:** [TechCrunch AI](https://techcrunch.com/2026/11/14/where-was-nvidias-security-team/)
---
*Follow ICARAX for more AI insights and tutorials.*
