From 29a7f12abee66ba238383b6242e3c81fde6d10b2 Mon Sep 17 00:00:00 2001
From: Rune Olsen <rune@rune.pm>
Date: Mon, 26 Jan 2026 13:09:32 +0100
Subject: [PATCH] bug fixing

---
 .gitignore                         |   3 +-
 QUICK_START.md                     | 187 ++++++++++++++++++
 README.md                          |  27 ++-
 TROUBLESHOOTING.md                 | 307 +++++++++++++++++++++++++++++
 config.yaml => config.yaml.example |  15 +-
 setup.sh                           |  63 ++++++
 src/config.py                      |  12 ++
 7 files changed, 597 insertions(+), 17 deletions(-)
 create mode 100644 QUICK_START.md
 create mode 100644 TROUBLESHOOTING.md
 rename config.yaml => config.yaml.example (83%)
 create mode 100755 setup.sh

diff --git a/.gitignore b/.gitignore
index 4d9d2e3..3bc6766 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,6 @@
-# Environment variables
+# Environment variables and local config
 .env
+config.yaml
 
 # Python
 __pycache__/
diff --git a/QUICK_START.md b/QUICK_START.md
new file mode 100644
index 0000000..d46fa2f
--- /dev/null
+++ b/QUICK_START.md
@@ -0,0 +1,187 @@
+# Quick Start Guide
+
+Get your News Agent running in 5 minutes!
+
+## 1. Initial Setup
+
+```bash
+# Navigate to project directory
+cd ~/news-agent
+
+# Run setup script
+./setup.sh
+
+# This creates config.yaml and .env from templates
+```
+
+## 2. Configure Credentials
+
+**Edit `.env` file** (no quotes needed):
+```bash
+nano .env
+```
+
+```env
+OPENROUTER_API_KEY=sk-or-v1-your-actual-key-here
+SMTP_USERNAME=your-email@yourdomain.com
+SMTP_PASSWORD=your-smtp-password
+```
+
+Get OpenRouter API key: https://openrouter.ai/keys
+
+## 3. Configure Email Settings
+
+```bash
+nano config.yaml
+```
+
+**Update these sections:**
+```yaml
+email:
+  to: "your-email@example.com"
+  from: "news-agent@yourdomain.com"
+  smtp:
+    host: "mail.yourdomain.com"
+    port: 587
+    use_tls: true
+```
+
+## 4. Install Dependencies
+
+```bash
+python3.11 -m venv .venv
+source .venv/bin/activate
+pip install feedparser httpx openai pydantic pydantic-settings jinja2 premailer python-dotenv pyyaml aiosqlite
+```
+
+## 5. Test Run
+
+```bash
+python -m src.main
+```
+
+**Expected output:**
+```
+INFO - News Agent starting...
+INFO - Fetching from 14 RSS sources...
+INFO - Fetched 152 articles from all sources
+INFO - Processing XX new articles with AI...
+INFO - Filtering articles by relevance...
+INFO - Selected 10 relevant articles
+INFO - Generating AI summaries...
+INFO - Sending email...
+INFO - Email sent successfully to your-email@example.com
+INFO - Daily digest sent successfully with 10 articles!
+```
+
+**Check your email inbox!**
+
+## 6. Set Up Daily Schedule
+
+```bash
+# Copy systemd files
+mkdir -p ~/.config/systemd/user
+cp systemd/news-agent.service ~/.config/systemd/user/
+cp systemd/news-agent.timer ~/.config/systemd/user/
+
+# Enable timer
+systemctl --user daemon-reload
+systemctl --user enable news-agent.timer
+systemctl --user start news-agent.timer
+
+# Enable lingering (run when not logged in)
+sudo loginctl enable-linger $USER
+
+# Check status
+systemctl --user list-timers
+```
+
+Done! You'll receive daily digests at 07:00 Europe/Oslo time.
+
+## Common Adjustments
+
+### Get More Articles
+
+Edit `config.yaml`:
+```yaml
+ai:
+  filtering:
+    min_score: 5.0  # Lower = more articles (default: 5.5)
+```
+
+### Change AI Model
+
+Free (with rate limits):
+```yaml
+ai:
+  model: "google/gemini-2.0-flash-exp:free"
+```
+
+Recommended (paid, ~$0.05/day):
+```yaml
+ai:
+  model: "openai/gpt-4o-mini"
+```
+
+Best quality (~$0.10/day):
+```yaml
+ai:
+  model: "anthropic/claude-3.5-haiku"
+```
+
+### Change Schedule Time
+
+Edit `~/.config/systemd/user/news-agent.timer`:
+```ini
+[Timer]
+OnCalendar=08:30  # Change to your preferred time
+```
+
+Then reload:
+```bash
+systemctl --user daemon-reload
+systemctl --user restart news-agent.timer
+```
+
+## Troubleshooting
+
+### Only 1-2 Articles in Email
+
+Lower the filtering threshold in `config.yaml`:
+```yaml
+ai:
+  filtering:
+    min_score: 5.0  # Was 5.5 or 6.5
+```
+
+### Hit Rate Limit
+
+Switch to paid model (see "Change AI Model" above)
+
+### No Email Received
+
+1. Check logs: `tail -f data/logs/news-agent.log`
+2. Test email: `python -m src.test_email_simple`
+3. See `SMTP_CONFIG.md` for detailed troubleshooting
+
+### Model Not Found Error
+
+Update model name in `config.yaml` - see `MODELS.md` for current names
+
+## Next Steps
+
+- Read `TROUBLESHOOTING.md` for solutions to common issues
+- Read `MODELS.md` to choose the best AI model for your needs
+- Read `SMTP_CONFIG.md` for email server configuration help
+- Customize RSS feeds in `config.yaml`
+- Adjust your interests in `config.yaml` for better filtering
+
+## Cost Estimate
+
+- **Free models**: $0/month (with rate limits)
+- **GPT-4o-mini**: ~$1.50-3.00/month
+- **Claude Haiku**: ~$3.00-7.50/month
+
+Monitor costs: https://openrouter.ai/activity
+
+Enjoy your daily tech news digests!
diff --git a/README.md b/README.md
index 78ffa0f..5cd15c1 100644
--- a/README.md
+++ b/README.md
@@ -61,14 +61,20 @@ pip install -e .
 ### 3. Configure News Agent
 
 ```bash
-# Copy environment template
-cp .env.example .env
+# Run setup script to copy config files
+./setup.sh
 
-# Edit .env and add your credentials
+# OR manually copy files:
+cp .env.example .env
+cp config.yaml.example config.yaml
+```
+
+**Edit `.env` file:**
+```bash
 nano .env
 ```
 
-**Required in `.env`:**
+Add your credentials (NO quotes needed):
 ```bash
 # OpenRouter API Key
 OPENROUTER_API_KEY=sk-or-v1-...your-key-here...
@@ -78,7 +84,7 @@ SMTP_USERNAME=your-email@yourdomain.com
 SMTP_PASSWORD=your-smtp-password
 ```
 
-**Edit `config.yaml`:**
+**Edit `config.yaml` file:**
 ```bash
 nano config.yaml
 ```
@@ -101,10 +107,13 @@ email:
 - **Outlook/Office365**: `smtp.office365.com:587`
 - **SendGrid**: `smtp.sendgrid.net:587`, use API key as password
 
-Optionally adjust:
-- AI model (default: `google/gemini-flash-1.5-8b` - fast and cheap)
-- Filtering threshold (default: 6.5/10)
-- Max articles per digest (default: 15)
+**Important settings to adjust:**
+- `ai.model`: Use `openai/gpt-4o-mini` (recommended, ~$0.05/day) or see MODELS.md
+- `ai.filtering.min_score`: Lower = more articles (5.5 recommended, was 6.5)
+- `ai.filtering.max_articles`: Maximum articles per digest (default: 15)
+- `email.to`: Your email address
+- `email.from`: Sender address
+- `smtp`: Your mail server settings
 - RSS sources (add/remove feeds)
 - Your interests for AI filtering
 
diff --git a/TROUBLESHOOTING.md b/TROUBLESHOOTING.md
new file mode 100644
index 0000000..96d3409
--- /dev/null
+++ b/TROUBLESHOOTING.md
@@ -0,0 +1,307 @@
+# Troubleshooting Guide
+
+## Common Issues and Solutions
+
+### Issue: Only 1-2 Articles in Digest (Expected 10-15)
+
+**Cause:** Filtering score is too strict or AI model is rating articles too harshly.
+
+**Solutions:**
+
+1. **Lower the filtering threshold** in `config.yaml`:
+   ```yaml
+   ai:
+     filtering:
+       min_score: 5.0  # Lower = more articles (default was 6.5)
+   ```
+
+2. **Check what scores articles are getting:**
+   ```bash
+   # View recent article scores in database
+   sqlite3 data/articles.db "SELECT title, relevance_score FROM articles WHERE processed=1 ORDER BY fetched_at DESC LIMIT 20;"
+   ```
+   
+   If most scores are between 4-6, lower your `min_score` to 4.5 or 5.0.
+
+3. **Adjust your interests** in `config.yaml` to be broader:
+   ```yaml
+   ai:
+     interests:
+       - "Technology news"  # Broader
+       - "Software development"  # Broader
+       # Instead of very specific interests
+   ```
+
+4. **Temporarily disable filtering** to see all articles:
+   ```yaml
+   ai:
+     filtering:
+       enabled: false  # Get ALL articles (not recommended long-term)
+   ```
+
+### Issue: Hit Rate Limit on Free Model
+
+**Cause:** Free models have strict rate limits (e.g., 10 requests/minute).
+
+**Solution:** Switch to a paid model in `config.yaml`:
+
+```yaml
+ai:
+  model: "openai/gpt-4o-mini"  # ~$0.05-0.10/day, no rate limits
+  # OR
+  model: "anthropic/claude-3.5-haiku"  # ~$0.10/day, excellent quality
+```
+
+Add credits to your OpenRouter account: https://openrouter.ai/credits
+
+### Issue: No Email Received
+
+**Check these in order:**
+
+1. **Look for error in logs:**
+   ```bash
+   tail -n 50 data/logs/news-agent.log | grep -i email
+   ```
+
+2. **Verify SMTP settings:**
+   ```bash
+   # Check config
+   grep -A 6 "smtp:" config.yaml
+   
+   # Check credentials
+   grep SMTP .env
+   ```
+
+3. **Test email independently:**
+   ```bash
+   python -m src.test_email_simple
+   ```
+
+4. **Common fixes:**
+   - Wrong port (use 587 for TLS, 465 for SSL)
+   - Missing credentials in `.env`
+   - Firewall blocking SMTP port
+   - Wrong hostname
+   - See `SMTP_CONFIG.md` for detailed troubleshooting
+
+### Issue: Articles Are All Duplicates
+
+**Cause:** Database hasn't been cleared and articles were already fetched.
+
+**Solution:**
+
+```bash
+# Option 1: Clear the database completely
+rm data/articles.db
+python -m src.main
+
+# Option 2: Just clear processed status to re-process
+sqlite3 data/articles.db "UPDATE articles SET processed=0, included_in_digest=0;"
+python -m src.main
+```
+
+### Issue: Email Not Nicely Formatted
+
+**Check:**
+1. Are you viewing in a web-based email client? (Gmail, Outlook web, etc.)
+   - HTML emails look best in web clients
+   
+2. Plain text client?
+   - The email includes both HTML and plain text versions
+   - Plain text is readable but less styled
+
+3. Want to customize?
+   - Edit `src/email/templates/daily_digest.html`
+   - CSS is inline for maximum email client compatibility
+
+### Issue: Model Name Error (404)
+
+```
+ERROR - No endpoints found for [model-name]
+```
+
+**Cause:** Model name is wrong or model is no longer available.
+
+**Solution:**
+
+1. Check current models at: https://openrouter.ai/models
+
+2. Update `config.yaml` with correct name:
+   ```yaml
+   ai:
+     model: "openai/gpt-4o-mini"  # Copy exact name from OpenRouter
+   ```
+
+3. See `MODELS.md` for recommended models
+
+### Issue: Too Expensive
+
+**Current cost too high?**
+
+Check usage: https://openrouter.ai/activity
+
+**Solutions:**
+
+1. **Use cheaper model:**
+   ```yaml
+   ai:
+     model: "openai/gpt-4o-mini"  # Cheapest reliable option (~$0.05/day)
+   ```
+
+2. **Process fewer articles:**
+   ```yaml
+   ai:
+     filtering:
+       max_articles: 10  # Reduce from 15
+       min_score: 6.0  # Higher score = fewer articles
+   ```
+
+3. **Remove some RSS sources** from `config.yaml`
+
+4. **Use free model** (with rate limits):
+   ```yaml
+   ai:
+     model: "google/gemini-2.0-flash-exp:free"
+   ```
+   Note: May fail during runs due to rate limits
+
+### Issue: Summaries Are Poor Quality
+
+**Solutions:**
+
+1. **Use better model:**
+   ```yaml
+   ai:
+     model: "anthropic/claude-3.5-haiku"  # Best summarization
+   ```
+
+2. **Adjust prompts** in `src/ai/prompts.py`:
+   - Edit `SUMMARIZATION_SYSTEM_PROMPT`
+   - Make it more specific to your needs
+
+3. **Check article content:**
+   - Some RSS feeds have limited content
+   - AI can only summarize what's in the feed
+
+### Issue: Articles Not Relevant
+
+**Solutions:**
+
+1. **Update your interests** in `config.yaml`:
+   ```yaml
+   ai:
+     interests:
+       - "Specific topic you care about"
+       - "Another specific interest"
+   ```
+
+2. **Check article scores:**
+   ```bash
+   sqlite3 data/articles.db "SELECT title, relevance_score, category FROM articles WHERE included_in_digest=1 ORDER BY relevance_score DESC LIMIT 10;"
+   ```
+
+3. **Increase filtering threshold:**
+   ```yaml
+   ai:
+     filtering:
+       min_score: 7.0  # Only very relevant articles
+   ```
+
+### Issue: SystemD Timer Not Running
+
+**Check timer status:**
+```bash
+systemctl --user status news-agent.timer
+systemctl --user list-timers
+```
+
+**Common fixes:**
+
+1. **Enable lingering:**
+   ```bash
+   sudo loginctl enable-linger $USER
+   ```
+
+2. **Reload systemd:**
+   ```bash
+   systemctl --user daemon-reload
+   systemctl --user restart news-agent.timer
+   ```
+
+3. **Check service logs:**
+   ```bash
+   journalctl --user -u news-agent.service -f
+   ```
+
+4. **Verify paths in service file:**
+   ```bash
+   nano ~/.config/systemd/user/news-agent.service
+   # Make sure WorkingDirectory and ExecStart paths are correct
+   ```
+
+### Issue: Permission Denied
+
+**Running as root?**
+- Don't run as root - use your normal user account
+- systemd user services run as your user
+
+**File permissions:**
+```bash
+chmod 600 .env
+chmod 644 config.yaml
+chmod +x setup.sh
+```
+
+### Issue: Import Errors
+
+```
+ModuleNotFoundError: No module named 'feedparser'
+```
+
+**Solution:**
+
+1. **Activate virtual environment:**
+   ```bash
+   source .venv/bin/activate
+   ```
+
+2. **Install dependencies:**
+   ```bash
+   pip install feedparser httpx openai pydantic pydantic-settings jinja2 premailer python-dotenv pyyaml aiosqlite
+   ```
+
+## Getting More Help
+
+1. **Check logs:**
+   ```bash
+   cat data/logs/news-agent.log
+   ```
+
+2. **Run in verbose mode** (shows all output):
+   ```bash
+   python -m src.main
+   ```
+
+3. **Test components individually:**
+   ```bash
+   # Test email
+   python -m src.test_email_simple
+   
+   # Test database
+   sqlite3 data/articles.db ".tables"
+   
+   # Test config
+   python -c "from src.config import get_config; c = get_config(); print(f'Model: {c.ai.model}')"
+   ```
+
+4. **Review documentation:**
+   - `README.md` - Overview
+   - `SETUP.md` - Installation
+   - `SMTP_CONFIG.md` - Email configuration
+   - `MODELS.md` - AI model selection
+   - `CHANGES.md` - Recent changes
+
+5. **OpenRouter support:**
+   - Dashboard: https://openrouter.ai/activity
+   - Docs: https://openrouter.ai/docs
+   - Models: https://openrouter.ai/models
diff --git a/config.yaml b/config.yaml.example
similarity index 83%
rename from config.yaml
rename to config.yaml.example
index 6cba7a0..69b6d50 100644
--- a/config.yaml
+++ b/config.yaml.example
@@ -72,16 +72,17 @@ sources:
 ai:
   provider: "openrouter"
   base_url: "https://openrouter.ai/api/v1"
-  model: "google/gemini-flash-1.5-8b"
-  # Alternative models (see https://openrouter.ai/models for full list):
-  # - "google/gemini-2.0-flash-exp:free" (free, experimental)
-  # - "anthropic/claude-3.5-haiku" (better quality, slightly more expensive)
-  # - "meta-llama/llama-3.1-8b-instruct:free" (free, good quality)
-  # - "openai/gpt-4o-mini" (good quality, moderate price)
+  model: "openai/gpt-4o-mini"
+  # Alternative models (always check https://openrouter.ai/models for current names):
+  # - "openai/gpt-4o-mini" (RECOMMENDED - good quality, ~$0.05-0.10/day, no rate limits)
+  # - "anthropic/claude-3.5-haiku" (excellent quality, ~$0.10/day)
+  # - "google/gemini-2.0-flash-exp:free" (FREE but has rate limits - not good for production)
+  # - "meta-llama/llama-3.1-8b-instruct:free" (FREE but has rate limits)
+  # NOTE: Free models have strict rate limits - use paid models for daily production use
   
   filtering:
     enabled: true
-    min_score: 6.5  # Out of 10 - articles below this score are filtered out
+    min_score: 5.5  # Out of 10 - lower score = more articles (was 6.5, now more permissive)
     max_articles: 15  # Maximum articles to include in daily digest
     
   interests:
diff --git a/setup.sh b/setup.sh
new file mode 100755
index 0000000..97c1655
--- /dev/null
+++ b/setup.sh
@@ -0,0 +1,63 @@
+#!/bin/bash
+# Setup script for News Agent
+
+set -e
+
+echo "========================================="
+echo "News Agent Setup"
+echo "========================================="
+echo ""
+
+# Check if config.yaml exists
+if [ -f "config.yaml" ]; then
+    echo "✓ config.yaml already exists - skipping"
+else
+    echo "Creating config.yaml from config.yaml.example..."
+    cp config.yaml.example config.yaml
+    echo "✓ Created config.yaml"
+    echo ""
+    echo "IMPORTANT: Edit config.yaml and update:"
+    echo "  - email.to: your email address"
+    echo "  - email.from: sender address"
+    echo "  - smtp settings: your mail server"
+    echo ""
+fi
+
+# Check if .env exists
+if [ -f ".env" ]; then
+    echo "✓ .env already exists - skipping"
+else
+    echo "Creating .env from .env.example..."
+    cp .env.example .env
+    echo "✓ Created .env"
+    echo ""
+    echo "IMPORTANT: Edit .env and add:"
+    echo "  - OPENROUTER_API_KEY=your-api-key"
+    echo "  - SMTP_USERNAME=your-email@domain.com"
+    echo "  - SMTP_PASSWORD=your-password"
+    echo ""
+fi
+
+echo "========================================="
+echo "Next Steps:"
+echo "========================================="
+echo "1. Edit .env file:"
+echo "   nano .env"
+echo ""
+echo "2. Edit config.yaml file:"
+echo "   nano config.yaml"
+echo ""
+echo "3. Create virtual environment:"
+echo "   python3.11 -m venv .venv"
+echo "   source .venv/bin/activate"
+echo ""
+echo "4. Install dependencies:"
+echo "   pip install feedparser httpx openai pydantic pydantic-settings jinja2 premailer python-dotenv pyyaml aiosqlite"
+echo ""
+echo "5. Test run:"
+echo "   python -m src.main"
+echo ""
+echo "6. Set up systemd timer:"
+echo "   See SETUP.md for instructions"
+echo ""
+echo "========================================="
diff --git a/src/config.py b/src/config.py
index 8623b21..76a1131 100644
--- a/src/config.py
+++ b/src/config.py
@@ -98,6 +98,18 @@ class Config:
         """Load configuration from YAML file and environment variables"""
         self.config_path = Path(config_path)
 
+        # If config.yaml doesn't exist, try config.yaml.example
+        if not self.config_path.exists():
+            example_path = Path(str(config_path) + ".example")
+            if example_path.exists():
+                raise FileNotFoundError(
+                    f"Config file '{config_path}' not found. "
+                    f"Copy '{example_path}' to '{config_path}' and edit it with your settings.\n"
+                    f"Run: cp {example_path} {config_path}"
+                )
+            else:
+                raise FileNotFoundError(f"Config file '{config_path}' not found")
+
         # Load YAML configuration
         with open(self.config_path) as f:
             self._config: dict[str, Any] = yaml.safe_load(f)