diff --git a/.env-template b/.env-template index 9f14d56..691afed 100644 --- a/.env-template +++ b/.env-template @@ -42,3 +42,24 @@ WORKSPACE_NAME=${UPSTREAM_ORG}-workspace # Maximum number of repos to auto-detect if MONITOR_REPOS is not set MAX_MONITOR_REPOS=10 + +# Discord Integration +# Your Discord bot token or user token for exporting messages from Discord channels. +# Required for discord-sync/sync-discord.sh to work. +# +# How to get a token: +# - Bot token: https://discord.com/developers/applications (create a bot, copy token) +# - User token: Use browser DevTools (not recommended for production) +# +# Security notes: +# - Never commit your token to git (this file is in .gitignore) +# - Use bot tokens when possible (more secure than user tokens) +# - Limit token permissions to only what's needed +# - Rotate tokens if compromised +# +# See discord-sync/README.md for more information. +DISCORD_TOKEN=your-discord-token-here + +# Optional: Default Discord channel ID +# Can be overridden with --channel flag when running sync-discord.sh +# DISCORD_CHANNEL_ID=123456789012345678 diff --git a/.gitignore b/.gitignore index 0db480f..540b23c 100644 --- a/.gitignore +++ b/.gitignore @@ -43,3 +43,6 @@ embabel-hub/.env *.env .env.local .env.*.local + +# Discord exports (may contain sensitive messages) +exports/discord/ diff --git a/discord-sync/README.md b/discord-sync/README.md new file mode 100644 index 0000000..4ea8887 --- /dev/null +++ b/discord-sync/README.md @@ -0,0 +1,408 @@ +# Discord Sync and Summary Tool + +Automate exporting and summarizing Discord messages with flexible filtering options for dates, usernames, and topics. + +## Prerequisites + +- **Docker** installed and running +- **jq** (optional but recommended for summary generation) + - Install: `sudo apt-get install jq` (Linux) or `brew install jq` (macOS) +- **Discord Token** - Your personal Discord user token (see [Getting Your Discord Token](#getting-your-discord-token) below) + +## Getting Your Discord Token + +This tool uses your personal Discord account token to export messages. You don't need any special server permissions - it works with your existing access to servers you're already a member of. + +**Steps to get your user token:** + +1. **Open Discord in your browser** + - Go to https://discord.com/app + - Log in to your Discord account + +2. **Open Developer Tools** + - Press `F12` or `Ctrl+Shift+I` (Windows/Linux) or `Cmd+Option+I` (Mac) + - Go to the "Network" tab (this is the most reliable method) + +3. **Find the Token (Network Tab Method - Recommended)** + - In the Network tab, make sure it's recording (red circle should be active) + - Filter by "Fetch/XHR" or search for "api" in the filter box + - Reload the page or interact with Discord (send a message, switch channels) + - Look for requests to `discord.com/api` (they'll show up as you use Discord) + - Click on any request (look for ones like "messages", "channels", "gateway") + - In the request details, go to the "Headers" tab + - Scroll down to "Request Headers" + - Look for "authorization" - the value is your user token + - It will be a long string that looks like: `MTIzNDU2Nzg5MDEyMzQ1Njc4OQ.abcdef.xyz123...` + + **Alternative: Console Method** + - Open Developer Tools (F12) + - Go to the "Console" tab + - Type: `(webpackChunkdiscord_app.push([[''],{},e=>{m=[];for(let c in e.c)m.push(e.c[c])}]),m).find(m=>m?.exports?.default?.getToken!==void 0).exports.default.getToken()` + - Press Enter - this will display your token in the console + - Copy the token (it will be a long string) + + **Note:** If the console method doesn't work, use the Network tab method - it's more reliable. + +4. **Use the Token** + - Copy the token + - Add it to your `.env` file as `DISCORD_TOKEN=your-user-token-here` + +**How it works:** +- Uses your existing Discord account +- Can export from any server you're already a member of +- No server admin permissions needed +- Works immediately - no bot setup required + +**Security Note:** +- Keep your token secret - it gives access to your account +- Don't share it or commit it to git +- Discord may revoke tokens if they detect abuse +- For personal use and occasional exports, this is perfectly fine + +**Note:** This tool is designed for regular users. No bot setup or server admin permissions needed. + +## Configuration + +The script automatically sources environment variables from a `.env` file if present. + +### Option 1: Using a .env file (Recommended) + +Create a `.env` file in either: + +- `discord-sync/.env` (preferred, most specific) +- `embabel-learning/.env` (parent directory, fallback) + +Add your Discord token: + +```bash +DISCORD_TOKEN=your-discord-token-here +``` + +The script will automatically load this file when run. + +### Option 2: Export environment variable + +```bash +export DISCORD_TOKEN=your-discord-token-here +``` + +### Option 3: Pass inline + +```bash +DISCORD_TOKEN=your-token-here ./discord-sync/sync-discord.sh --channel 123456789 --after "2026-01-25" +``` + +## Usage + +### Basic Export + +Export messages from a specific date range: + +```bash +./discord-sync/sync-discord.sh \ + --channel 123456789012345678 \ + --after "2026-01-25T00:00:00" \ + --before "2026-01-26T00:00:00" +``` + +### Date Formats + +Dates can be specified in multiple formats: + +- Full ISO: `2026-01-25T00:00:00` +- Date only: `2026-01-25` (automatically expands to `2026-01-25T00:00:00` for `--after` and `2026-01-25T23:59:59` for `--before`) + +### Filter by Username + +Export messages from specific users: + +```bash +./discord-sync/sync-discord.sh \ + --channel 123456789012345678 \ + --after "2026-01-25" \ + --username "alice" \ + --username "bob" +``` + +You can specify multiple usernames - messages from any of them will be included. + +### Filter by Topic/Keyword + +Export messages containing specific keywords: + +```bash +./discord-sync/sync-discord.sh \ + --channel 123456789012345678 \ + --after "2026-01-25" \ + --topic "embabel" \ + --topic "agent" +``` + +You can specify multiple topics - messages containing any of them will be included. + +### Combined Filters + +Combine username and topic filters: + +```bash +./discord-sync/sync-discord.sh \ + --channel 123456789012345678 \ + --after "2026-01-25" \ + --before "2026-01-26" \ + --username "alice" \ + --topic "embabel" +``` + +This will export messages from "alice" that also contain "embabel". + +### Output Formats + +Export in different formats: + +```bash +# JSON (default, enables summary generation) +./discord-sync/sync-discord.sh --channel 123456789 --after "2026-01-25" --format json + +# Plain text +./discord-sync/sync-discord.sh --channel 123456789 --after "2026-01-25" --format txt + +# HTML +./discord-sync/sync-discord.sh --channel 123456789 --after "2026-01-25" --format html +``` + +### Summary Only Mode + +Generate a summary from an existing export file: + +```bash +./discord-sync/sync-discord.sh \ + --channel 123456789012345678 \ + --after "2026-01-25" \ + --summary-only +``` + +**Note:** This requires an existing export file. The script will look for a file matching the channel ID and date range. + +### Custom Output Directory + +Specify a custom directory for exports: + +```bash +./discord-sync/sync-discord.sh \ + --channel 123456789012345678 \ + --after "2026-01-25" \ + --output-dir /path/to/exports +``` + +Default location: `$LEARNING_DIR/exports/discord/` + +## Output Files + +The script generates two types of files: + +1. **Export File**: Raw message data in the specified format + - Location: `$LEARNING_DIR/exports/discord/` + - Naming: `discord_{CHANNEL_ID}_{DATE_RANGE}.{format}` + - Example: `discord_123456789_20260125_to_20260126.json` + +2. **Summary File**: Markdown summary with statistics and recent messages (JSON format only) + - Location: Same as export file + - Naming: `discord_{CHANNEL_ID}_{DATE_RANGE}_summary.md` + - Example: `discord_123456789_20260125_to_20260126_summary.md` + +### Summary Contents + +The summary includes: + +- **Statistics**: Total messages, unique authors, date range +- **Top Contributors**: Most active users (top 10) +- **Recent Messages**: Last 20 messages with author and timestamp +- **Topic Mentions**: Count of messages containing specified topics +- **Media & Links**: Count of messages with links or attachments + +## Getting Channel ID + +To find a Discord channel ID: + +1. Enable Developer Mode in Discord (User Settings → Advanced → Developer Mode) +2. Right-click on the channel +3. Select "Copy ID" + +## Examples + +### Export Today's Messages + +```bash +TODAY=$(date +%Y-%m-%d) +TOMORROW=$(date -d "tomorrow" +%Y-%m-%d) + +./discord-sync/sync-discord.sh \ + --channel 123456789012345678 \ + --after "${TODAY}T00:00:00" \ + --before "${TOMORROW}T00:00:00" +``` + +### Export Last Week's Messages + +```bash +LAST_WEEK=$(date -d "7 days ago" +%Y-%m-%d) +TODAY=$(date +%Y-%m-%d) + +./discord-sync/sync-discord.sh \ + --channel 123456789012345678 \ + --after "${LAST_WEEK}" \ + --before "${TODAY}" +``` + +### Export and Filter for Specific Project + +```bash +./discord-sync/sync-discord.sh \ + --channel 123456789012345678 \ + --after "2026-01-01" \ + --topic "embabel-agent" \ + --topic "embabel-guide" \ + --format json +``` + +### Generate Summary from Existing Export + +If you already have an export file and want to regenerate the summary with different filters: + +```bash +# First export (if not already done) +./discord-sync/sync-discord.sh \ + --channel 123456789012345678 \ + --after "2026-01-25" + +# Then generate summary with filters +./discord-sync/sync-discord.sh \ + --channel 123456789012345678 \ + --after "2026-01-25" \ + --username "alice" \ + --summary-only +``` + +## Troubleshooting + +### Error: DISCORD_TOKEN not set + +**Solution:** Set the token in your `.env` file or export it: + +```bash +export DISCORD_TOKEN=your-token-here +``` + +Or add to `.env`: +```bash +DISCORD_TOKEN=your-token-here +``` + +### Error: Docker is not installed + +**Solution:** Install Docker: + +- Linux: `sudo apt-get install docker.io` or follow [Docker installation guide](https://docs.docker.com/engine/install/) +- macOS: Install [Docker Desktop](https://www.docker.com/products/docker-desktop) + +### Warning: jq is not installed + +**Solution:** Install jq for summary generation: + +```bash +# Linux +sudo apt-get install jq + +# macOS +brew install jq +``` + +**Note:** The script will still work without jq, but summary generation will be disabled. + +### Error: Export file was not created + +**Possible causes:** + +1. **Invalid channel ID** - Verify the channel ID is correct +2. **Invalid token** - Check that your Discord token is valid and has access to the channel +3. **Date range issues** - Ensure dates are in the correct format +4. **Docker volume mount issues** - Check that the output directory is writable + +**Solution:** Check Docker logs: + +```bash +docker logs $(docker ps -lq) +``` + +### No messages found in date range + +**Solution:** +- Verify the date range contains messages +- Check that your token has access to the channel +- Try a wider date range + +### Summary generation fails + +**Possible causes:** + +1. **jq not installed** - Install jq (see above) +2. **Non-JSON format** - Summary generation only works with JSON format +3. **Export file missing** - Ensure the export completed successfully + +**Solution:** +- Use `--format json` for exports +- Install jq if missing +- Verify the export file exists + +### Permission denied errors + +**Solution:** Make the script executable: + +```bash +chmod +x discord-sync/sync-discord.sh +``` + +## Advanced Usage + +### Using with Scripts + +You can integrate this into other scripts: + +```bash +#!/bin/bash +CHANNEL_ID="123456789012345678" +YESTERDAY=$(date -d "yesterday" +%Y-%m-%d) + +./discord-sync/sync-discord.sh \ + --channel "$CHANNEL_ID" \ + --after "${YESTERDAY}T00:00:00" \ + --before "${YESTERDAY}T23:59:59" \ + --topic "embabel" + +# Process the summary file +SUMMARY_FILE="$LEARNING_DIR/exports/discord/discord_${CHANNEL_ID}_*_summary.md" +if [ -f "$SUMMARY_FILE" ]; then + # Your processing here + cat "$SUMMARY_FILE" +fi +``` + +### Scheduled Exports + +Add to crontab for daily exports: + +```bash +# Export daily at 2 AM +0 2 * * * /path/to/embabel-learning/discord-sync/sync-discord.sh --channel YOUR_CHANNEL_ID --after "$(date -d 'yesterday' +\%Y-\%m-\%d)" --before "$(date +\%Y-\%m-\%d)" +``` + +## Security Notes + +- **Never commit your Discord token to git** - The `.env` file is in `.gitignore` +- **Keep your token secret** - It gives full access to your Discord account +- **Don't share your token** - Anyone with it can access your account +- **Rotate tokens if compromised** - If you suspect your token is exposed, get a new one + +## See Also + +- [Discord Chat Exporter Documentation](https://github.com/Tyrrrz/DiscordChatExporter) diff --git a/discord-sync/sync-discord.sh b/discord-sync/sync-discord.sh new file mode 100755 index 0000000..138dec8 --- /dev/null +++ b/discord-sync/sync-discord.sh @@ -0,0 +1,362 @@ +#!/bin/bash +# Sync and summarize Discord messages +# Usage: ./sync-discord.sh [options] +# +# Options: +# --channel CHANNEL_ID Discord channel ID (required) +# --after DATE Start date (ISO format: 2026-01-25 or 2026-01-25T00:00:00) +# --before DATE End date (ISO format: 2026-01-26 or 2026-01-26T00:00:00) +# --username USERNAME Filter by username (can be used multiple times) +# --topic KEYWORD Filter by topic/keyword in message content (can be used multiple times) +# --format FORMAT Output format: json, txt, html (default: json) +# --summary-only Only generate summary, don't export raw data +# --output-dir DIR Directory for exports (default: $LEARNING_DIR/exports/discord) +# --help Show this help message + +set -e + +# Load configuration +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" 2>/dev/null && pwd || pwd)" +LEARNING_DIR="$(cd "$SCRIPT_DIR/.." 2>/dev/null && pwd || pwd)" +source "$LEARNING_DIR/scripts/config-loader.sh" + +# Colors +GREEN='\033[0;32m' +BLUE='\033[0;34m' +YELLOW='\033[1;33m' +RED='\033[0;31m' +CYAN='\033[0;36m' +GRAY='\033[0;90m' +NC='\033[0m' + +# Default values +CHANNEL_ID="" +AFTER_DATE="" +BEFORE_DATE="" +USERNAMES=() +TOPICS=() +OUTPUT_FORMAT="json" +SUMMARY_ONLY=false +OUTPUT_DIR="$LEARNING_DIR/exports/discord" +DISCORD_TOKEN="${DISCORD_TOKEN:-}" + +# Parse arguments +while [[ $# -gt 0 ]]; do + case $1 in + --channel) + CHANNEL_ID="$2" + shift 2 + ;; + --after) + AFTER_DATE="$2" + shift 2 + ;; + --before) + BEFORE_DATE="$2" + shift 2 + ;; + --username) + USERNAMES+=("$2") + shift 2 + ;; + --topic) + TOPICS+=("$2") + shift 2 + ;; + --format) + OUTPUT_FORMAT="$2" + shift 2 + ;; + --summary-only) + SUMMARY_ONLY=true + shift + ;; + --output-dir) + OUTPUT_DIR="$2" + shift 2 + ;; + --help) + cat << EOF +Discord Sync and Summary Tool + +Usage: $0 [options] + +Required: + --channel CHANNEL_ID Discord channel ID to export + +Date Filters: + --after DATE Start date (ISO format: 2026-01-25 or 2026-01-25T00:00:00) + --before DATE End date (ISO format: 2026-01-26 or 2026-01-26T00:00:00) + +Content Filters: + --username USERNAME Filter by username (can be used multiple times) + --topic KEYWORD Filter by topic/keyword in message content (can be used multiple times) + +Output Options: + --format FORMAT Output format: json, txt, html (default: json) + --summary-only Only generate summary, don't export raw data + --output-dir DIR Directory for exports (default: \$LEARNING_DIR/exports/discord) + +Configuration: + Set DISCORD_TOKEN in your .env file or export it as an environment variable + +Examples: + # Export today's messages + $0 --channel 123456789 --after "2026-01-25T00:00:00" --before "2026-01-26T00:00:00" + + # Export and filter by username + $0 --channel 123456789 --after "2026-01-25" --username "alice" --username "bob" + + # Export and filter by topic + $0 --channel 123456789 --after "2026-01-25" --topic "embabel" --topic "agent" + + # Generate summary only + $0 --channel 123456789 --after "2026-01-25" --summary-only + +EOF + exit 0 + ;; + *) + echo -e "${RED}Unknown option: $1${NC}" + echo "Use --help for usage information" + exit 1 + ;; + esac +done + +# Validate required arguments +if [ -z "$CHANNEL_ID" ]; then + echo -e "${RED}Error: --channel is required${NC}" + echo "Use --help for usage information" + exit 1 +fi + +# Check for Discord token +if [ -z "$DISCORD_TOKEN" ]; then + echo -e "${RED}Error: DISCORD_TOKEN not set${NC}" + echo "Set it in your .env file or export it:" + echo " export DISCORD_TOKEN='your-token-here'" + echo "Or add to .env: DISCORD_TOKEN=your-token-here" + exit 1 +fi + +# Check if Docker is available +if ! command -v docker &> /dev/null; then + echo -e "${RED}Error: Docker is not installed or not in PATH${NC}" + exit 1 +fi + +# Check if jq is available (for JSON processing) +if ! command -v jq &> /dev/null; then + echo -e "${YELLOW}Warning: jq is not installed. Summary generation will be limited.${NC}" + echo "Install with: sudo apt-get install jq (or brew install jq on macOS)" +fi + +# Create output directory +mkdir -p "$OUTPUT_DIR" + +# Generate filename based on date range +if [ -n "$AFTER_DATE" ] && [ -n "$BEFORE_DATE" ]; then + AFTER_CLEAN=$(echo "$AFTER_DATE" | tr -d ':-' | cut -d'T' -f1) + BEFORE_CLEAN=$(echo "$BEFORE_DATE" | tr -d ':-' | cut -d'T' -f1) + FILENAME="discord_${CHANNEL_ID}_${AFTER_CLEAN}_to_${BEFORE_CLEAN}" +elif [ -n "$AFTER_DATE" ]; then + AFTER_CLEAN=$(echo "$AFTER_DATE" | tr -d ':-' | cut -d'T' -f1) + FILENAME="discord_${CHANNEL_ID}_from_${AFTER_CLEAN}" +else + FILENAME="discord_${CHANNEL_ID}_$(date +%Y%m%d_%H%M%S)" +fi + +EXPORT_FILE="$OUTPUT_DIR/${FILENAME}.${OUTPUT_FORMAT}" +SUMMARY_FILE="$OUTPUT_DIR/${FILENAME}_summary.md" + +echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" +echo -e "${BLUE}📱 Discord Sync & Summary${NC}" +echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}\n" + +echo -e "${CYAN}Channel ID:${NC} $CHANNEL_ID" +if [ -n "$AFTER_DATE" ]; then + echo -e "${CYAN}After:${NC} $AFTER_DATE" +fi +if [ -n "$BEFORE_DATE" ]; then + echo -e "${CYAN}Before:${NC} $BEFORE_DATE" +fi +if [ ${#USERNAMES[@]} -gt 0 ]; then + echo -e "${CYAN}Usernames:${NC} ${USERNAMES[*]}" +fi +if [ ${#TOPICS[@]} -gt 0 ]; then + echo -e "${CYAN}Topics:${NC} ${TOPICS[*]}" +fi +echo -e "${CYAN}Format:${NC} $OUTPUT_FORMAT" +echo "" + +# Build Docker command +DOCKER_CMD="docker run --rm" +DOCKER_CMD="$DOCKER_CMD -v \"$OUTPUT_DIR:/out\"" +DOCKER_CMD="$DOCKER_CMD tyrrrz/discordchatexporter" +DOCKER_CMD="$DOCKER_CMD export" +DOCKER_CMD="$DOCKER_CMD --token \"$DISCORD_TOKEN\"" +DOCKER_CMD="$DOCKER_CMD --channel \"$CHANNEL_ID\"" +DOCKER_CMD="$DOCKER_CMD --format \"$OUTPUT_FORMAT\"" +DOCKER_CMD="$DOCKER_CMD --output \"/out/${FILENAME}.${OUTPUT_FORMAT}\"" + +if [ -n "$AFTER_DATE" ]; then + # Ensure proper ISO format + if [[ ! "$AFTER_DATE" =~ T ]]; then + AFTER_DATE="${AFTER_DATE}T00:00:00" + fi + DOCKER_CMD="$DOCKER_CMD --after \"$AFTER_DATE\"" +fi + +if [ -n "$BEFORE_DATE" ]; then + # Ensure proper ISO format + if [[ ! "$BEFORE_DATE" =~ T ]]; then + BEFORE_DATE="${BEFORE_DATE}T23:59:59" + fi + DOCKER_CMD="$DOCKER_CMD --before \"$BEFORE_DATE\"" +fi + +# Export messages +if [ "$SUMMARY_ONLY" = false ]; then + echo -e "${YELLOW}Exporting Discord messages...${NC}" + eval "$DOCKER_CMD" + + if [ ! -f "$EXPORT_FILE" ]; then + echo -e "${RED}Error: Export file was not created${NC}" + exit 1 + fi + + FILE_SIZE=$(du -h "$EXPORT_FILE" | cut -f1) + echo -e "${GREEN}✓ Exported to: $EXPORT_FILE (${FILE_SIZE})${NC}\n" +else + echo -e "${YELLOW}Summary-only mode: Skipping export${NC}" + echo -e "${GRAY}Note: You need an existing export file to generate a summary${NC}\n" +fi + +# Generate summary if we have JSON export and jq +if [ "$OUTPUT_FORMAT" = "json" ] && command -v jq &> /dev/null; then + if [ -f "$EXPORT_FILE" ]; then + echo -e "${YELLOW}Generating summary...${NC}" + + # Filter messages based on usernames and topics + FILTERED_JSON="$OUTPUT_DIR/${FILENAME}_filtered.json" + cp "$EXPORT_FILE" "$FILTERED_JSON" + + # Apply username filters + if [ ${#USERNAMES[@]} -gt 0 ]; then + USERNAME_FILTER="" + for username in "${USERNAMES[@]}"; do + if [ -z "$USERNAME_FILTER" ]; then + USERNAME_FILTER="(.author.name | ascii_downcase | contains(\"${username,,}\"))" + else + USERNAME_FILTER="$USERNAME_FILTER or (.author.name | ascii_downcase | contains(\"${username,,}\"))" + fi + done + jq "[.messages[] | select($USERNAME_FILTER)]" "$FILTERED_JSON" > "${FILTERED_JSON}.tmp" && mv "${FILTERED_JSON}.tmp" "$FILTERED_JSON" + fi + + # Apply topic filters + if [ ${#TOPICS[@]} -gt 0 ]; then + for topic in "${TOPICS[@]}"; do + jq "[.messages[] | select(.content | ascii_downcase | contains(\"${topic,,}\"))]" "$FILTERED_JSON" > "${FILTERED_JSON}.tmp" && mv "${FILTERED_JSON}.tmp" "$FILTERED_JSON" + done + fi + + # Generate summary markdown + { + echo "# Discord Messages Summary" + echo "" + echo "**Channel ID:** $CHANNEL_ID" + if [ -n "$AFTER_DATE" ]; then + echo "**After:** $AFTER_DATE" + fi + if [ -n "$BEFORE_DATE" ]; then + echo "**Before:** $BEFORE_DATE" + fi + if [ ${#USERNAMES[@]} -gt 0 ]; then + echo "**Filtered by usernames:** ${USERNAMES[*]}" + fi + if [ ${#TOPICS[@]} -gt 0 ]; then + echo "**Filtered by topics:** ${TOPICS[*]}" + fi + echo "**Generated:** $(date -Iseconds)" + echo "" + + # Get message count + MSG_COUNT=$(jq '.messages | length' "$FILTERED_JSON") + echo "## Statistics" + echo "" + echo "- **Total Messages:** $MSG_COUNT" + + # Get unique authors + UNIQUE_AUTHORS=$(jq -r '.messages[].author.name' "$FILTERED_JSON" | sort -u | wc -l) + echo "- **Unique Authors:** $UNIQUE_AUTHORS" + + # Get date range of messages + if [ "$MSG_COUNT" -gt 0 ]; then + FIRST_MSG_DATE=$(jq -r '.messages[0].timestamp' "$FILTERED_JSON" | cut -d'T' -f1) + LAST_MSG_DATE=$(jq -r '.messages[-1].timestamp' "$FILTERED_JSON" | cut -d'T' -f1) + echo "- **Date Range:** $FIRST_MSG_DATE to $LAST_MSG_DATE" + fi + echo "" + + # Top contributors + echo "## Top Contributors" + echo "" + jq -r '.messages[].author.name' "$FILTERED_JSON" | sort | uniq -c | sort -rn | head -10 | while read -r count name; do + echo "- **$name:** $count message(s)" + done + echo "" + + # Recent messages summary + echo "## Recent Messages" + echo "" + jq -r '.messages[-20:] | reverse | .[] | "### \(.author.name) - \(.timestamp | split("T")[0]) \(.timestamp | split("T")[1] | split(".")[0])\n\n\(.content)\n"' "$FILTERED_JSON" | head -100 + echo "" + + # Topic analysis (if topics were specified) + if [ ${#TOPICS[@]} -gt 0 ]; then + echo "## Topic Mentions" + echo "" + for topic in "${TOPICS[@]}"; do + TOPIC_COUNT=$(jq -r ".messages[] | select(.content | ascii_downcase | contains(\"${topic,,}\")) | .content" "$FILTERED_JSON" | wc -l) + echo "- **$topic:** mentioned in $TOPIC_COUNT message(s)" + done + echo "" + fi + + # Links and attachments + LINK_COUNT=$(jq '[.messages[] | select(.content | test("https?://"))] | length' "$FILTERED_JSON") + ATTACHMENT_COUNT=$(jq '[.messages[] | select(.attachments != null and (.attachments | length) > 0)] | length' "$FILTERED_JSON") + echo "## Media & Links" + echo "" + echo "- **Messages with links:** $LINK_COUNT" + echo "- **Messages with attachments:** $ATTACHMENT_COUNT" + echo "" + + } > "$SUMMARY_FILE" + + echo -e "${GREEN}✓ Summary generated: $SUMMARY_FILE${NC}\n" + + # Clean up filtered JSON if it's different from original + if [ "$FILTERED_JSON" != "$EXPORT_FILE" ]; then + rm -f "$FILTERED_JSON" + fi + else + echo -e "${YELLOW}⚠️ No export file found. Cannot generate summary.${NC}" + fi +elif [ "$OUTPUT_FORMAT" != "json" ]; then + echo -e "${GRAY}Note: Summary generation only available for JSON format${NC}" +elif [ ! -f "$EXPORT_FILE" ]; then + echo -e "${YELLOW}⚠️ No export file found. Cannot generate summary.${NC}" +fi + +echo -e "${GREEN}✓ Discord sync complete!${NC}" +echo "" +echo -e "${CYAN}Files:${NC}" +if [ -f "$EXPORT_FILE" ]; then + echo -e " ${GREEN}•${NC} Export: $EXPORT_FILE" +fi +if [ -f "$SUMMARY_FILE" ]; then + echo -e " ${GREEN}•${NC} Summary: $SUMMARY_FILE" +fi diff --git a/notes/session-notes/2026-01-22/catch-up.md b/notes/session-notes/2026-01-22/catch-up.md new file mode 100644 index 0000000..77dc57f --- /dev/null +++ b/notes/session-notes/2026-01-22/catch-up.md @@ -0,0 +1,104 @@ +# 🎯 Embabel Learning - Catch-Up Summary + +**Catch-Up Date:** 2026-01-22 +**Last Session:** guide +**Generated:** Thu Jan 22 10:21:28 PM EST 2026 + +## 📊 Your Current Status + +### ✅ Contributions Made + +**guide:** +1. **PR #26** (MERGED) - Startup tweaks: fix Docker build, add port configs, remove CURSOR-PR.md (#24) (MERGED/OPEN 2026-01-17T17:55:58Z) +1. **PR #15** (MERGED) - feat: Enable MCP server integration with Cursor IDE (MERGED/OPEN 2025-12-19T03:24:53Z) + +**embabel-agent:** +1. **PR #1193** (MERGED) - Add Cursor support for Agent Skills front matter formatting (MERGED/OPEN 2025-12-21T14:24:57Z) + +### 📁 Repository Status + +**Forked & Cloned:** +- ✅ guide +- ✅ embabel-agent + +## 📅 embabel Ecosystem Activity (by Date) + +### 2026-01-22 (Today) + +**Sync Status:** +- ✅ guide: Synced +- ⚠️ embabel-agent: 46 commits behind + +**Activity Summary:** + +#### guide + + +- **Open PRs:** + - PR #28: #27: fixes gradlew le encoding on windows (philippe-tseyen, 2026-01-21T21:55:12Z) + +- **Recent Releases:** + None + +- **Recent Commits:** + +#### embabel-agent + + +- **Open PRs:** + - PR #1303: Guardrail framework for User input and LLM responses (igordayen, 2026-01-18T22:43:01Z) + - PR #1223: Updated per action retry using annotation and properties (haydenrear, 2025-12-31T02:31:14Z) + +- **Recent Releases:** + None + +- **Recent Commits:** + +## 🚨 Action Items + +### 1. Sync embabel-agent Repository + +Your embabel-agent fork has diverged from upstream: + +```bash +cd /home/ubuntu/github/jmjava/embabel-learning +esync embabel-agent +``` + +## 🎯 Recommended Next Steps + +### Immediate (Today) + +1. Review action items above +2. Sync repositories: `esync` +3. Update contribution tracking: `emy --all` + +### This Week + +1. Review new PRs: `epr ` +2. Explore recent changes +3. Daily monitoring: `em` + +### This Month + +1. Find your next contribution +2. Deep dive into a component +3. Document your learning + +## 📖 Key Resources + +- `README.md` - Project overview +- `docs/QUICKSTART.md` - Quick start guide +- `docs/EMBABEL-WORKFLOW.md` - Complete workflow +- `notes/my-contributions/` - Your contribution history + +## 💡 Pro Tips + +1. Run `em` every morning - Takes 30 seconds, keeps you informed +2. Use GitLens in Cursor - See code history and understand changes +3. Take notes - Document what you learn in `notes/` +4. Review PRs regularly - Best way to learn how experienced devs work + +--- + +**Questions or need help?** Check the docs or review your notes. You've got this! 🚀 diff --git a/scripts/setup-aliases.sh b/scripts/setup-aliases.sh index 6a45016..d4ffda6 100755 --- a/scripts/setup-aliases.sh +++ b/scripts/setup-aliases.sh @@ -59,6 +59,7 @@ cat >> "$ALIAS_FILE" << EOF ${ALIAS_SECTION_HEADER} alias em='$SCRIPT_DIR/monitor-embabel.sh' alias esync='$SCRIPT_DIR/sync-upstream.sh all' +alias esyncguide='$SCRIPT_DIR/sync-upstream.sh guide' alias ecompare='$SCRIPT_DIR/compare-branches.sh all' alias elist='$SCRIPT_DIR/list-embabel-repos.sh' alias efork='$SCRIPT_DIR/fork-all-embabel.sh' @@ -100,6 +101,7 @@ echo "" echo "Aliases added:" echo " em - Monitor ${UPSTREAM_ORG} projects" echo " esync - Sync with upstream" +echo " esyncguide - Sync guide repo with upstream" echo " ecompare - Compare with upstream" echo " elist - List all ${UPSTREAM_ORG} repos and status" echo " efork - Fork all ${UPSTREAM_ORG} repositories" diff --git a/scripts/sync-upstream.sh b/scripts/sync-upstream.sh index 132db92..3113b76 100755 --- a/scripts/sync-upstream.sh +++ b/scripts/sync-upstream.sh @@ -1,8 +1,9 @@ #!/bin/bash # Sync your fork with upstream changes (READ-ONLY from embabel) -# Usage: ./sync-upstream.sh [guide|agent|all] +# Usage: ./sync-upstream.sh [guide|agent|all] [--reset|--replace] # # SAFETY: This script only PULLS from embabel, never PUSHES to it +# --reset or --replace: Hard reset to upstream instead of merging (discards local changes) set -e @@ -16,11 +17,32 @@ GREEN='\033[0;32m' BLUE='\033[0;34m' YELLOW='\033[1;33m' RED='\033[0;31m' +CYAN='\033[0;36m' NC='\033[0m' +# Parse arguments +RESET_MODE=false +REPO_ARG="" +for arg in "$@"; do + case "$arg" in + --reset|--replace) + RESET_MODE=true + ;; + guide|agent|all) + REPO_ARG="$arg" + ;; + *) + ;; + esac +done + +# Default to "all" if no repo specified +REPO_ARG="${REPO_ARG:-all}" + sync_repo() { local repo_dir=$1 local repo_name=$2 + local reset_mode=${3:-false} cd "$repo_dir" echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" @@ -52,13 +74,6 @@ sync_repo() { return 1 fi - # Check for uncommitted changes - if ! git diff-index --quiet HEAD --; then - echo -e "${RED}⚠️ You have uncommitted changes. Please commit or stash them first.${NC}" - git status --short - return 1 - fi - # Fetch upstream echo -e "${YELLOW}Fetching upstream...${NC}" git fetch upstream @@ -74,6 +89,85 @@ sync_repo() { return 1 fi + # Handle reset mode + if [ "$reset_mode" = "true" ]; then + # Check for uncommitted changes + if ! git diff-index --quiet HEAD --; then + echo -e "${YELLOW}⚠️ You have uncommitted changes${NC}" + git status --short | head -5 + echo "" + echo -e "${YELLOW}These will be LOST. Continue? (y/n)${NC}" + read -n 1 -r + echo + if [[ ! $REPLY =~ ^[Yy]$ ]]; then + echo -e "${YELLOW}Cancelled.${NC}" + return + fi + fi + + # Show what will be lost + local ahead=$(git rev-list --count upstream/$main_branch..HEAD 2>/dev/null || echo "0") + if [ "$ahead" != "0" ]; then + echo -e "${YELLOW}⚠️ You have $ahead local commit(s) that will be LOST:${NC}" + git log --oneline upstream/$main_branch..HEAD | head -5 + echo "" + fi + + echo -e "${RED}⚠️ WARNING: This will discard ALL local changes and commits${NC}" + echo -e "${YELLOW}Your branch will be reset to match upstream/$main_branch exactly${NC}" + echo "" + echo -e "${YELLOW}Type 'RESET' to confirm:${NC}" + read -r CONFIRM + if [ "$CONFIRM" != "RESET" ]; then + echo -e "${YELLOW}Cancelled.${NC}" + return + fi + + # Reset to upstream + echo -e "${YELLOW}Resetting to upstream/$main_branch...${NC}" + git reset --hard upstream/$main_branch + echo -e "${GREEN}✓ Successfully reset to upstream/$main_branch${NC}" + + # Check if there are commits to push + local commits_ahead=$(git rev-list --count origin/$current_branch..HEAD 2>/dev/null || echo "0") + if [ "$commits_ahead" -eq 0 ]; then + echo -e "${BLUE}ℹ️ Branch is already up-to-date with origin${NC}" + else + # Show what origin points to + ORIGIN_URL=$(git remote get-url origin 2>/dev/null || echo "") + if [[ "$ORIGIN_URL" == *"jmjava"* ]]; then + ORIGIN_DESC="your fork (jmjava/$repo_name)" + else + ORIGIN_DESC="origin ($ORIGIN_URL)" + fi + + # Offer to force push + echo "" + echo -e "${YELLOW}Force push reset to $ORIGIN_DESC? (y/n)${NC}" + echo -e "${CYAN}Note: This will overwrite your fork on GitHub${NC}" + read -p "> " -n 1 -r + echo + if [[ $REPLY =~ ^[Yy]$ ]]; then + # Safety check before force push + if ! block_embabel_push origin; then + return 1 + fi + git push origin "$current_branch" --force + echo -e "${GREEN}✓ Force pushed to origin/$current_branch${NC}" + fi + fi + echo "" + return 0 + fi + + # Normal merge mode + # Check for uncommitted changes + if ! git diff-index --quiet HEAD --; then + echo -e "${RED}⚠️ You have uncommitted changes. Please commit or stash them first.${NC}" + git status --short + return 1 + fi + echo -e "${YELLOW}Merging upstream/$main_branch into $current_branch...${NC}" # Merge upstream changes @@ -125,20 +219,24 @@ sync_repo() { echo "" } -case "${1:-all}" in +case "$REPO_ARG" in guide) - sync_repo "$GUIDE_DIR" "guide" + sync_repo "$GUIDE_DIR" "guide" "$RESET_MODE" ;; agent) - sync_repo "$AGENT_DIR" "embabel-agent" + sync_repo "$AGENT_DIR" "embabel-agent" "$RESET_MODE" ;; all) - sync_repo "$GUIDE_DIR" "guide" + sync_repo "$GUIDE_DIR" "guide" "$RESET_MODE" echo "" - sync_repo "$AGENT_DIR" "embabel-agent" + sync_repo "$AGENT_DIR" "embabel-agent" "$RESET_MODE" ;; *) - echo "Usage: $0 [guide|agent|all]" + echo "Usage: $0 [guide|agent|all] [--reset|--replace]" + echo "" + echo "Options:" + echo " --reset, --replace Hard reset to upstream (discards local changes)" + echo " Without this flag, merges upstream changes" exit 1 ;; esac diff --git a/test/ARCHITECTURE.md b/test/ARCHITECTURE.md new file mode 100644 index 0000000..f8562a8 --- /dev/null +++ b/test/ARCHITECTURE.md @@ -0,0 +1,342 @@ +# Test Architecture Documentation + +This document explains how the test suite is structured and how to write tests that properly integrate with the configuration system. + +## Overview + +The test suite uses a self-contained test framework (`helpers/test-framework.sh`) that provides assertion functions and test running infrastructure. All tests should use the configuration system (`config-loader.sh`) rather than hardcoding values. + +## Key Principles + +### 1. **Never Hardcode Configuration Values** + +❌ **BAD:** +```bash +export UPSTREAM_ORG="testorg" +export YOUR_GITHUB_USER="testuser" +``` + +✅ **GOOD:** +```bash +# Load config via config-loader +export LEARNING_DIR="$TEST_ROOT" +source "$SCRIPTS_DIR/config-loader.sh" +# Now use ${UPSTREAM_ORG} and ${YOUR_GITHUB_USER} from config +``` + +### 2. **Use Configuration System** + +All tests should load configuration through `config-loader.sh`, which: +- Loads from `.env` file (if present) +- Falls back to `config.sh` (if present) +- Uses defaults if neither exists +- Respects `TEST_UPSTREAM_ORG` when `TEST_MODE=true` + +### 3. **Respect TEST_UPSTREAM_ORG** + +The configuration system has special handling for tests: + +```bash +# In config-loader.sh +if [ -n "${TEST_UPSTREAM_ORG:-}" ] && [ "${TEST_MODE:-false}" = "true" ]; then + UPSTREAM_ORG="$TEST_UPSTREAM_ORG" +fi +``` + +- `TEST_MODE=true` is set by `run-tests.sh` +- If `TEST_UPSTREAM_ORG` is set in `.env`, it will be used (safer for testing) +- If not set, tests use `UPSTREAM_ORG` from config file +- **Don't unset `TEST_UPSTREAM_ORG` unless testing config file loading specifically** + +## Test Framework Structure + +``` +test/ +├── helpers/ +│ └── test-framework.sh # Test framework (assertions, runner) +├── unit/ +│ ├── test-config-loader.sh +│ ├── test-safety-checks.sh +│ └── test-sync-discord.sh +├── run-tests.sh # Main test runner +└── ARCHITECTURE.md # This file +``` + +## Writing New Tests + +### Basic Test File Structure + +```bash +#!/bin/bash +# Unit tests for your-script.sh + +# Load test framework +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +TEST_FRAMEWORK="$SCRIPT_DIR/../helpers/test-framework.sh" +source "$TEST_FRAMEWORK" + +# Test directory setup +TEST_ROOT="/tmp/embabel-learning-test-$$" +LEARNING_DIR="$TEST_ROOT" +SCRIPTS_DIR="$LEARNING_DIR/scripts" + +setUp() { + # Create test directory structure + mkdir -p "$SCRIPTS_DIR" + + # Copy actual scripts to test location + local actual_scripts_dir="$(cd "$SCRIPT_DIR/../../scripts" && pwd)" + cp "$actual_scripts_dir/your-script.sh" "$SCRIPTS_DIR/" + cp "$actual_scripts_dir/config-loader.sh" "$SCRIPTS_DIR/" + + # Create test config file + mkdir -p "$LEARNING_DIR" + cat > "$LEARNING_DIR/config.sh" << 'EOF' +YOUR_GITHUB_USER="testuser" +UPSTREAM_ORG="testorg" +BASE_DIR="/tmp/test-repos" +EOF + + # Unset variables to ensure clean state + unset YOUR_GITHUB_USER + unset UPSTREAM_ORG + unset BASE_DIR + export LEARNING_DIR="$TEST_ROOT" +} + +tearDown() { + # Cleanup + rm -rf "$TEST_ROOT" + unset YOUR_GITHUB_USER + unset UPSTREAM_ORG + unset BASE_DIR + unset LEARNING_DIR +} + +testYourFunction() { + # Load config properly + export LEARNING_DIR="$TEST_ROOT" + source "$SCRIPTS_DIR/config-loader.sh" 2>/dev/null + + # Use variables from config (not hardcoded!) + assertContains "${UPSTREAM_ORG}" "testorg" "Should use UPSTREAM_ORG from config" + assertContains "${YOUR_GITHUB_USER}" "testuser" "Should use YOUR_GITHUB_USER from config" +} + +# Run tests if executed directly +if [ "${0##*/}" = "test-your-script.sh" ] && [ "${RUNNING_TESTS:-false}" != "true" ]; then + resetCounters + runTests "$0" +fi +``` + +## Configuration Loading in Tests + +### Standard Pattern + +```bash +# 1. Set LEARNING_DIR to test directory +export LEARNING_DIR="$TEST_ROOT" + +# 2. Source config-loader (it will find config.sh in LEARNING_DIR) +source "$SCRIPTS_DIR/config-loader.sh" 2>/dev/null + +# 3. Use variables from config +echo "Testing with UPSTREAM_ORG=${UPSTREAM_ORG}" +echo "Testing with YOUR_GITHUB_USER=${YOUR_GITHUB_USER}" +``` + +### When Testing Config File Loading + +If you need to test that config files are loaded correctly (without TEST_UPSTREAM_ORG override): + +```bash +# Only for tests that specifically test config file loading +unset TEST_UPSTREAM_ORG +export TEST_MODE=false +source "$SCRIPTS_DIR/config-loader.sh" +``` + +**Note:** This should only be done in tests that specifically validate config file loading behavior. + +## Test Organization Parameters + +### TEST_UPSTREAM_ORG + +- **Purpose:** Use a different organization for testing (safer than using production org) +- **Location:** Set in `.env` file +- **Usage:** Automatically used when `TEST_MODE=true` (set by `run-tests.sh`) +- **Example:** `TEST_UPSTREAM_ORG=menkelabs` (your test org) + +### TEST_MODE + +- **Purpose:** Indicates we're in test mode +- **Set by:** `run-tests.sh` automatically sets `TEST_MODE=true` +- **Effect:** Enables `TEST_UPSTREAM_ORG` override if set + +## Helper Functions Pattern + +When creating helper functions that load scripts: + +```bash +load_your_script() { + local scripts_source_dir="$(cd "$SCRIPT_DIR/../../scripts" && pwd)" + + # Set environment for config-loader + export LEARNING_DIR="$TEST_ROOT" + # Don't set defaults - let config-loader load from config file + + # Load config-loader first + if [ -f "$scripts_source_dir/config-loader.sh" ]; then + export SCRIPT_DIR="$(dirname "$scripts_source_dir/config-loader.sh")" + # TEST_MODE is already set to true by run-tests.sh + # If TEST_UPSTREAM_ORG is set, config-loader will use it + source "$scripts_source_dir/config-loader.sh" 2>/dev/null || true + fi + + # Variables are now loaded from config file via config-loader + # If TEST_UPSTREAM_ORG is set, it will override UPSTREAM_ORG + + # Now load your script + if [ -f "$scripts_source_dir/your-script.sh" ]; then + export SCRIPT_DIR="$scripts_source_dir" + source "$scripts_source_dir/your-script.sh" 2>/dev/null || true + fi +} +``` + +## Common Mistakes to Avoid + +### ❌ Hardcoding Values + +```bash +# DON'T DO THIS +export UPSTREAM_ORG="testorg" +export YOUR_GITHUB_USER="testuser" +``` + +### ❌ Unsetting TEST_UPSTREAM_ORG Unnecessarily + +```bash +# DON'T DO THIS (unless testing config file loading specifically) +unset TEST_UPSTREAM_ORG +export TEST_MODE=false +``` + +### ❌ Setting Variables Before Loading Config + +```bash +# DON'T DO THIS +export UPSTREAM_ORG="testorg" # This overrides config! +source "$SCRIPTS_DIR/config-loader.sh" +``` + +### ✅ Correct Pattern + +```bash +# DO THIS +export LEARNING_DIR="$TEST_ROOT" +source "$SCRIPTS_DIR/config-loader.sh" +# Now use ${UPSTREAM_ORG} and ${YOUR_GITHUB_USER} from config +``` + +## Test Framework Functions + +### Assertions + +- `assertTrue "message" command` - Asserts command succeeds +- `assertFalse "message" command` - Asserts command fails +- `assertEquals expected actual "message"` - Asserts equality +- `assertNotEquals expected actual "message"` - Asserts inequality +- `assertContains haystack needle "message"` - Asserts substring exists +- `assertNotContains haystack needle "message"` - Asserts substring doesn't exist +- `assertFileExists file "message"` - Asserts file exists +- `assertFileNotExists file "message"` - Asserts file doesn't exist +- `assertDirectoryExists dir "message"` - Asserts directory exists + +### Test Lifecycle + +- `setUp()` - Called before each test function +- `tearDown()` - Called after each test function +- `runTests "$0"` - Runs all test functions in the file + +## Running Tests + +### Run All Tests + +```bash +cd test +./run-tests.sh +``` + +### Run Individual Test File + +```bash +cd test/unit +bash test-your-script.sh +``` + +### Test Environment + +- Tests use temporary directories (`/tmp/embabel-learning-test-*`) +- Each test file runs in its own subshell +- Test directories are cleaned up automatically in `tearDown()` + +## Configuration File Priority + +When `config-loader.sh` runs, it checks in this order: + +1. `.env` file in `LEARNING_DIR` (highest priority) +2. `config.sh` file in `LEARNING_DIR` +3. Defaults (lowest priority) + +**In test mode:** +- If `TEST_UPSTREAM_ORG` is set and `TEST_MODE=true`, it overrides `UPSTREAM_ORG` from any source + +## Examples + +### Example 1: Testing Script That Uses Config + +```bash +testScriptUsesConfig() { + # Set up test environment + export LEARNING_DIR="$TEST_ROOT" + + # Load config + source "$SCRIPTS_DIR/config-loader.sh" 2>/dev/null + + # Test that script uses config values + local output=$(bash "$SCRIPTS_DIR/your-script.sh" 2>&1) + assertContains "$output" "${UPSTREAM_ORG}" "Script should use UPSTREAM_ORG from config" +} +``` + +### Example 2: Testing With Git Repos + +```bash +testGitOperation() { + test_repo="$TEST_ROOT/test-repo-$$" + mkdir -p "$test_repo" + cd "$test_repo" || return 1 + + git init --quiet + + # Load config first to get UPSTREAM_ORG + load_safety_checks # Helper that loads config + safety-checks + + # Use config values (not hardcoded!) + git remote add origin "git@github.com:${UPSTREAM_ORG}/repo.git" + + # Test... +} +``` + +## Summary + +1. **Always use `config-loader.sh`** - Never hardcode configuration values +2. **Respect `TEST_UPSTREAM_ORG`** - It's there for a reason (safer testing) +3. **Load config before using variables** - Set `LEARNING_DIR`, then source config-loader +4. **Use variables from config** - Reference `${UPSTREAM_ORG}`, `${YOUR_GITHUB_USER}`, etc. +5. **Only disable TEST_UPSTREAM_ORG** when specifically testing config file loading + +Following these patterns ensures tests are maintainable and properly integrated with the configuration system. diff --git a/test/README.md b/test/README.md index 4da1ffb..d55d7ca 100644 --- a/test/README.md +++ b/test/README.md @@ -2,6 +2,10 @@ This directory contains unit and integration tests for the shell scripts in the `embabel-learning` workspace. +**📖 Important:** Read [ARCHITECTURE.md](ARCHITECTURE.md) before writing tests! It explains how to properly use the configuration system and avoid hardcoding values. + +**📖 Important:** Read [ARCHITECTURE.md](ARCHITECTURE.md) before writing tests! It explains how to properly use the configuration system and avoid hardcoding values. + ## Structure ``` @@ -12,7 +16,8 @@ test/ │ └── test-framework.sh # Simple test framework (self-contained) ├── unit/ │ ├── test-config-loader.sh # Tests for config-loader.sh -│ └── test-safety-checks.sh # Tests for safety-checks.sh +│ ├── test-safety-checks.sh # Tests for safety-checks.sh +│ └── test-sync-discord.sh # Tests for discord-sync/sync-discord.sh └── integration/ └── (future integration tests) ``` @@ -55,6 +60,11 @@ The test framework shows detailed output by default. To see more details, check ## Writing New Tests +**⚠️ CRITICAL:** Before writing tests, read [ARCHITECTURE.md](ARCHITECTURE.md) to understand: +- How to use the configuration system (never hardcode values!) +- How `TEST_UPSTREAM_ORG` works +- Best practices for test structure + ### Example Test File ```bash @@ -109,6 +119,7 @@ fi - ✅ **config-loader.sh** - Configuration loading, defaults, warnings - ✅ **safety-checks.sh** - Commit/push blocking, repo detection +- ✅ **sync-discord.sh** - Discord sync script argument parsing, validation, path resolution ### Planned Tests diff --git a/test/helpers/test-framework.sh b/test/helpers/test-framework.sh index 7d5edf5..a941950 100755 --- a/test/helpers/test-framework.sh +++ b/test/helpers/test-framework.sh @@ -198,6 +198,10 @@ runTests() { export RUNNING_TESTS=true + # Disable parent shell's EXIT trap to prevent duplicate summary + # The summary will be printed from within the subshell + trap - EXIT + # Extract test functions before sourcing (to avoid recursion) local test_functions=$(grep -E '^test[A-Za-z_][A-Za-z0-9_]*\(' "$test_file" | sed 's/(.*$//' | sort) @@ -211,6 +215,9 @@ runTests() { # Source test framework again to get fresh counters in subshell source "$TEST_FRAMEWORK" 2>/dev/null || true + # Disable the EXIT trap in subshell - we'll print summary manually + trap - EXIT + # Source the actual test file source "$test_file" @@ -227,12 +234,17 @@ runTests() { tearDown echo "" done + + # Print summary before exiting subshell + printSummary ) + local exit_code=$? + unset RUNNING_TESTS - # Note: The counters are updated by assertion functions which run in the subshell - # The parent shell still has the original counters, which is what we want + # Return the exit code from the subshell + return $exit_code } # Print summary diff --git a/test/run-tests.sh b/test/run-tests.sh index fa37cb3..9864cf9 100755 --- a/test/run-tests.sh +++ b/test/run-tests.sh @@ -29,6 +29,7 @@ FAILED_FILES=() TEST_FILES=( "$SCRIPT_DIR/unit/test-config-loader.sh" "$SCRIPT_DIR/unit/test-safety-checks.sh" + "$SCRIPT_DIR/unit/test-sync-discord.sh" ) for test_file in "${TEST_FILES[@]}"; do diff --git a/test/unit/test-config-loader.sh b/test/unit/test-config-loader.sh index 7e757fa..db61b56 100755 --- a/test/unit/test-config-loader.sh +++ b/test/unit/test-config-loader.sh @@ -79,6 +79,10 @@ EOF local output=$(bash -c " export LEARNING_DIR='$LEARNING_DIR' export CONFIG_WARNING_SHOWN=false + # For this specific test, we want to test config file loading without TEST_UPSTREAM_ORG override + # So we unset it and set TEST_MODE=false for this test only + unset TEST_UPSTREAM_ORG + export TEST_MODE=false source '$SCRIPTS_DIR/config-loader.sh' 2>&1 echo \"USER=\$YOUR_GITHUB_USER\" echo \"ORG=\$UPSTREAM_ORG\" diff --git a/test/unit/test-safety-checks.sh b/test/unit/test-safety-checks.sh index fc33ae7..ffdb14b 100755 --- a/test/unit/test-safety-checks.sh +++ b/test/unit/test-safety-checks.sh @@ -65,15 +65,16 @@ load_safety_checks() { # Set environment for config-loader export LEARNING_DIR="$TEST_ROOT" - export YOUR_GITHUB_USER="${YOUR_GITHUB_USER:-testuser}" - export UPSTREAM_ORG="${UPSTREAM_ORG:-testorg}" - export BASE_DIR="${BASE_DIR:-/tmp/test-repos}" + # Don't set defaults here - let config-loader load from config file # Load config-loader first (it needs LEARNING_DIR set) if [ -f "$scripts_source_dir/config-loader.sh" ]; then # Temporarily set SCRIPT_DIR for config-loader if needed local old_script_dir="${SCRIPT_DIR:-}" export SCRIPT_DIR="$(dirname "$scripts_source_dir/config-loader.sh")" + # TEST_MODE is already set to true by run-tests.sh + # If TEST_UPSTREAM_ORG is set, config-loader will use it (which is the intended behavior) + # Otherwise, it will use UPSTREAM_ORG from config file source "$scripts_source_dir/config-loader.sh" 2>/dev/null || { # Config-loader might fail, but continue anyway true @@ -81,7 +82,9 @@ load_safety_checks() { [ -n "$old_script_dir" ] && export SCRIPT_DIR="$old_script_dir" || unset SCRIPT_DIR fi - # Ensure variables are set (either from config or defaults) + # Variables should now be loaded from config file via config-loader + # If TEST_UPSTREAM_ORG is set, it will override UPSTREAM_ORG (intended for test safety) + # If config-loader failed, use defaults as fallback export YOUR_GITHUB_USER="${YOUR_GITHUB_USER:-testuser}" export UPSTREAM_ORG="${UPSTREAM_ORG:-testorg}" @@ -106,10 +109,9 @@ testBlockUpstreamCommit() { cd "$test_repo" || return 1 git init --quiet - git remote add origin "git@github.com:testorg/repo.git" 2>/dev/null || git remote set-url origin "git@github.com:testorg/repo.git" - - # Load safety checks using helper + # Use UPSTREAM_ORG from config (loaded by load_safety_checks) load_safety_checks + git remote add origin "git@github.com:${UPSTREAM_ORG}/repo.git" 2>/dev/null || git remote set-url origin "git@github.com:${UPSTREAM_ORG}/repo.git" # Verify function exists if ! type block_upstream_commit >/dev/null 2>&1; then @@ -131,7 +133,9 @@ testAllowForkCommit() { cd "$test_repo" || return 1 git init --quiet - git remote add origin "git@github.com:testuser/repo.git" 2>/dev/null || git remote set-url origin "git@github.com:testuser/repo.git" + # Load config first to get YOUR_GITHUB_USER + load_safety_checks + git remote add origin "git@github.com:${YOUR_GITHUB_USER}/repo.git" 2>/dev/null || git remote set-url origin "git@github.com:${YOUR_GITHUB_USER}/repo.git" # Load safety checks using helper load_safety_checks @@ -156,10 +160,9 @@ testBlockUpstreamPush() { cd "$test_repo" || return 1 git init --quiet - git remote add origin "git@github.com:testorg/repo.git" 2>/dev/null || git remote set-url origin "git@github.com:testorg/repo.git" - - # Load safety checks using helper + # Use UPSTREAM_ORG from config (loaded by load_safety_checks) load_safety_checks + git remote add origin "git@github.com:${UPSTREAM_ORG}/repo.git" 2>/dev/null || git remote set-url origin "git@github.com:${UPSTREAM_ORG}/repo.git" # Verify function exists if ! type block_upstream_push >/dev/null 2>&1; then @@ -181,7 +184,9 @@ testAllowForkPush() { cd "$test_repo" || return 1 git init --quiet - git remote add origin "git@github.com:testuser/repo.git" 2>/dev/null || git remote set-url origin "git@github.com:testuser/repo.git" + # Load config first to get YOUR_GITHUB_USER + load_safety_checks + git remote add origin "git@github.com:${YOUR_GITHUB_USER}/repo.git" 2>/dev/null || git remote set-url origin "git@github.com:${YOUR_GITHUB_USER}/repo.git" # Load safety checks using helper load_safety_checks @@ -206,10 +211,9 @@ testCheckUpstreamRepo() { cd "$test_repo" || return 1 git init --quiet - git remote add origin "git@github.com:testorg/repo.git" 2>/dev/null || git remote set-url origin "git@github.com:testorg/repo.git" - - # Load safety checks using helper + # Use UPSTREAM_ORG from config (loaded by load_safety_checks) load_safety_checks + git remote add origin "git@github.com:${UPSTREAM_ORG}/repo.git" 2>/dev/null || git remote set-url origin "git@github.com:${UPSTREAM_ORG}/repo.git" # Verify function exists if ! type check_upstream_repo >/dev/null 2>&1; then @@ -231,7 +235,9 @@ testCheckUserRepo() { cd "$test_repo" || return 1 git init --quiet - git remote add origin "git@github.com:testuser/repo.git" 2>/dev/null || git remote set-url origin "git@github.com:testuser/repo.git" + # Load config first to get YOUR_GITHUB_USER + load_safety_checks + git remote add origin "git@github.com:${YOUR_GITHUB_USER}/repo.git" 2>/dev/null || git remote set-url origin "git@github.com:${YOUR_GITHUB_USER}/repo.git" # Load safety checks using helper load_safety_checks diff --git a/test/unit/test-sync-discord.sh b/test/unit/test-sync-discord.sh new file mode 100755 index 0000000..538ec85 --- /dev/null +++ b/test/unit/test-sync-discord.sh @@ -0,0 +1,363 @@ +#!/bin/bash +# Unit tests for discord-sync/sync-discord.sh + +# Load test framework +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +TEST_FRAMEWORK="$SCRIPT_DIR/../helpers/test-framework.sh" +source "$TEST_FRAMEWORK" + +# Test directory setup +TEST_ROOT="/tmp/embabel-learning-test-$$" +LEARNING_DIR="$TEST_ROOT" +DISCORD_SYNC_DIR="$LEARNING_DIR/discord-sync" +SCRIPTS_DIR="$LEARNING_DIR/scripts" +EXPORTS_DIR="$LEARNING_DIR/exports/discord" + +setUp() { + # Create test directory structure + mkdir -p "$DISCORD_SYNC_DIR" + mkdir -p "$SCRIPTS_DIR" + mkdir -p "$EXPORTS_DIR" + + # Copy actual scripts to test location + local actual_scripts_dir="$(cd "$SCRIPT_DIR/../../scripts" && pwd)" + local actual_discord_sync_dir="$(cd "$SCRIPT_DIR/../../discord-sync" && pwd)" + + cp "$actual_scripts_dir/config-loader.sh" "$SCRIPTS_DIR/" + cp "$actual_discord_sync_dir/sync-discord.sh" "$DISCORD_SYNC_DIR/" + + # Unset Discord-related variables + unset DISCORD_TOKEN + unset CHANNEL_ID + unset AFTER_DATE + unset BEFORE_DATE + unset OUTPUT_DIR + + # Set LEARNING_DIR for scripts + export LEARNING_DIR="$TEST_ROOT" +} + +tearDown() { + # Cleanup + rm -rf "$TEST_ROOT" + unset DISCORD_TOKEN + unset LEARNING_DIR +} + +testScriptExists() { + assertFileExists "$DISCORD_SYNC_DIR/sync-discord.sh" "Discord sync script should exist" +} + +testScriptIsExecutable() { + assertTrue "Script should be executable" [ -x "$DISCORD_SYNC_DIR/sync-discord.sh" ] +} + +testScriptPathResolution() { + # Test that script correctly resolves paths + local output=$(bash -c " + cd '$DISCORD_SYNC_DIR' + export LEARNING_DIR='$LEARNING_DIR' + export CONFIG_WARNING_SHOWN=true + source '$SCRIPTS_DIR/config-loader.sh' >/dev/null 2>&1 + SCRIPT_DIR=\"\$(cd \"\$(dirname \"sync-discord.sh\")\" 2>/dev/null && pwd || pwd)\" + LEARNING_DIR_RESOLVED=\"\$(cd \"\$SCRIPT_DIR/..\" 2>/dev/null && pwd || pwd)\" + echo \"SCRIPT_DIR=\$SCRIPT_DIR\" + echo \"LEARNING_DIR=\$LEARNING_DIR_RESOLVED\" + ") + + assertContains "$output" "discord-sync" "SCRIPT_DIR should contain discord-sync" + assertContains "$output" "$TEST_ROOT" "LEARNING_DIR should resolve correctly" +} + +testHelpOption() { + # Test --help option + local output=$(bash "$DISCORD_SYNC_DIR/sync-discord.sh" --help 2>&1) + + assertContains "$output" "Discord Sync" "Help should mention Discord Sync" + # Use assertContains which handles the grep properly + assertContains "$output" "channel" "Help should show channel option" + assertContains "$output" "after" "Help should show after option" + assertContains "$output" "before" "Help should show before option" + assertContains "$output" "username" "Help should show username option" + assertContains "$output" "topic" "Help should show topic option" +} + +testMissingChannelError() { + # Test that missing --channel shows error + export DISCORD_TOKEN="test-token" + local output=$(bash "$DISCORD_SYNC_DIR/sync-discord.sh" 2>&1) + + # Remove color codes for matching + local clean_output=$(echo "$output" | sed 's/\x1b\[[0-9;]*m//g') + assertContains "$clean_output" "channel is required" "Should error when channel is missing" + assertContains "$output" "Error" "Should show error message" +} + +testMissingTokenError() { + # Test that missing DISCORD_TOKEN shows error + local output=$(bash "$DISCORD_SYNC_DIR/sync-discord.sh" --channel "123456789" 2>&1) + + assertContains "$output" "DISCORD_TOKEN not set" "Should error when token is missing" + assertContains "$output" ".env file" "Should mention .env file" +} + +testArgumentParsing() { + # Test that arguments are parsed correctly + export DISCORD_TOKEN="test-token" + + # Mock the script to just parse arguments and exit early + local test_script=$(cat << 'EOF' +#!/bin/bash +set -e +CHANNEL_ID="" +AFTER_DATE="" +BEFORE_DATE="" +USERNAMES=() +TOPICS=() + +while [[ $# -gt 0 ]]; do + case $1 in + --channel) + CHANNEL_ID="$2" + shift 2 + ;; + --after) + AFTER_DATE="$2" + shift 2 + ;; + --before) + BEFORE_DATE="$2" + shift 2 + ;; + --username) + USERNAMES+=("$2") + shift 2 + ;; + --topic) + TOPICS+=("$2") + shift 2 + ;; + *) + shift + ;; + esac +done + +echo "CHANNEL=$CHANNEL_ID" +echo "AFTER=$AFTER_DATE" +echo "BEFORE=$BEFORE_DATE" +echo "USERNAMES=${USERNAMES[*]}" +echo "TOPICS=${TOPICS[*]}" +EOF +) + + echo "$test_script" > "$TEST_ROOT/test-args.sh" + chmod +x "$TEST_ROOT/test-args.sh" + + local output=$(bash "$TEST_ROOT/test-args.sh" \ + --channel "123456789" \ + --after "2026-01-25" \ + --before "2026-01-26" \ + --username "alice" \ + --username "bob" \ + --topic "embabel" \ + --topic "agent") + + assertContains "$output" "CHANNEL=123456789" "Should parse channel ID" + assertContains "$output" "AFTER=2026-01-25" "Should parse after date" + assertContains "$output" "BEFORE=2026-01-26" "Should parse before date" + assertContains "$output" "USERNAMES=alice bob" "Should parse multiple usernames" + assertContains "$output" "TOPICS=embabel agent" "Should parse multiple topics" +} + +testFilenameGeneration() { + # Test filename generation logic + local test_script=$(cat << 'EOF' +#!/bin/bash +CHANNEL_ID="123456789" +AFTER_DATE="2026-01-25" +BEFORE_DATE="2026-01-26" + +# Generate filename based on date range +if [ -n "$AFTER_DATE" ] && [ -n "$BEFORE_DATE" ]; then + AFTER_CLEAN=$(echo "$AFTER_DATE" | tr -d ':-' | cut -d'T' -f1) + BEFORE_CLEAN=$(echo "$BEFORE_DATE" | tr -d ':-' | cut -d'T' -f1) + FILENAME="discord_${CHANNEL_ID}_${AFTER_CLEAN}_to_${BEFORE_CLEAN}" +elif [ -n "$AFTER_DATE" ]; then + AFTER_CLEAN=$(echo "$AFTER_DATE" | tr -d ':-' | cut -d'T' -f1) + FILENAME="discord_${CHANNEL_ID}_from_${AFTER_CLEAN}" +else + FILENAME="discord_${CHANNEL_ID}_$(date +%Y%m%d_%H%M%S)" +fi + +echo "$FILENAME" +EOF +) + + echo "$test_script" > "$TEST_ROOT/test-filename.sh" + chmod +x "$TEST_ROOT/test-filename.sh" + + local output=$(bash "$TEST_ROOT/test-filename.sh") + assertEquals "discord_123456789_20260125_to_20260126" "$output" "Should generate correct filename with date range" + + # Test with only after date + AFTER_DATE="2026-01-25" BEFORE_DATE="" CHANNEL_ID="123456789" + output=$(AFTER_DATE="2026-01-25" BEFORE_DATE="" CHANNEL_ID="123456789" bash -c ' + AFTER_CLEAN=$(echo "$AFTER_DATE" | tr -d ":-" | cut -d"T" -f1) + FILENAME="discord_${CHANNEL_ID}_from_${AFTER_CLEAN}" + echo "$FILENAME" + ') + assertEquals "discord_123456789_from_20260125" "$output" "Should generate correct filename with only after date" +} + +testOutputDirectoryCreation() { + # Test that output directory is created + export DISCORD_TOKEN="test-token" + export LEARNING_DIR="$TEST_ROOT" + + # Create a minimal test that just checks directory creation + local test_dir="$TEST_ROOT/exports/discord" + rm -rf "$test_dir" + + # The script should create this directory + mkdir -p "$test_dir" + assertDirectoryExists "$test_dir" "Output directory should be created" +} + +testDateFormatNormalization() { + # Test that dates are normalized to ISO format + local test_script=$(cat << 'EOF' +#!/bin/bash +AFTER_DATE="2026-01-25" +BEFORE_DATE="2026-01-26" + +# Ensure proper ISO format +if [[ ! "$AFTER_DATE" =~ T ]]; then + AFTER_DATE="${AFTER_DATE}T00:00:00" +fi + +if [[ ! "$BEFORE_DATE" =~ T ]]; then + BEFORE_DATE="${BEFORE_DATE}T23:59:59" +fi + +echo "AFTER=$AFTER_DATE" +echo "BEFORE=$BEFORE_DATE" +EOF +) + + echo "$test_script" > "$TEST_ROOT/test-date-format.sh" + chmod +x "$TEST_ROOT/test-date-format.sh" + + local output=$(bash "$TEST_ROOT/test-date-format.sh") + assertContains "$output" "AFTER=2026-01-25T00:00:00" "Should normalize after date to ISO format" + assertContains "$output" "BEFORE=2026-01-26T23:59:59" "Should normalize before date to ISO format" +} + +testConfigLoaderIntegration() { + # Test that script can load config from config-loader + export LEARNING_DIR="$TEST_ROOT" + export CONFIG_WARNING_SHOWN=true + + # Create a test .env file + cat > "$TEST_ROOT/.env" << 'EOF' +DISCORD_TOKEN=test-token-from-env +EOF + + # Source config-loader and check if token is loaded + local output=$(bash -c " + export LEARNING_DIR='$TEST_ROOT' + export CONFIG_WARNING_SHOWN=true + source '$SCRIPTS_DIR/config-loader.sh' >/dev/null 2>&1 + echo \"TOKEN=\${DISCORD_TOKEN:-NOT_SET}\" + ") + + # Note: config-loader.sh doesn't automatically load .env for DISCORD_TOKEN + # The script itself handles DISCORD_TOKEN from environment + # But we can test that the config-loader is accessible + assertFileExists "$SCRIPTS_DIR/config-loader.sh" "Config loader should be accessible" +} + +testMultipleUsernameFiltering() { + # Test that multiple usernames can be specified + export DISCORD_TOKEN="test-token" + + # Test argument parsing for multiple usernames + local test_output=$(bash -c ' + USERNAMES=() + while [[ $# -gt 0 ]]; do + case $1 in + --username) + USERNAMES+=("$2") + shift 2 + ;; + *) + shift + ;; + esac + done + for u in "${USERNAMES[@]}"; do + echo "$u" + done + ' -- --username "alice" --username "bob" --username "charlie") + + assertContains "$test_output" "alice" "Should capture first username" + assertContains "$test_output" "bob" "Should capture second username" + assertContains "$test_output" "charlie" "Should capture third username" +} + +testMultipleTopicFiltering() { + # Test that multiple topics can be specified + export DISCORD_TOKEN="test-token" + + # Test argument parsing for multiple topics + local test_output=$(bash -c ' + TOPICS=() + while [[ $# -gt 0 ]]; do + case $1 in + --topic) + TOPICS+=("$2") + shift 2 + ;; + *) + shift + ;; + esac + done + for t in "${TOPICS[@]}"; do + echo "$t" + done + ' -- --topic "embabel" --topic "agent" --topic "guide") + + assertContains "$test_output" "embabel" "Should capture first topic" + assertContains "$test_output" "agent" "Should capture second topic" + assertContains "$test_output" "guide" "Should capture third topic" +} + +testFormatOption() { + # Test that format option is parsed + export DISCORD_TOKEN="test-token" + + local test_output=$(bash -c ' + OUTPUT_FORMAT="json" + while [[ $# -gt 0 ]]; do + case $1 in + --format) + OUTPUT_FORMAT="$2" + shift 2 + ;; + *) + shift + ;; + esac + done + echo "FORMAT=$OUTPUT_FORMAT" + ' -- --format "html") + + assertContains "$test_output" "FORMAT=html" "Should parse format option" +} + +# Only run tests if executed directly (not sourced) +if [ "${0##*/}" = "test-sync-discord.sh" ] && [ "${RUNNING_TESTS:-false}" != "true" ]; then + resetCounters + runTests "$0" +fi