knowledge/.github/workflows/sync.yml at main · elizaOS/knowledge · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
name: Sync Knowledge Sources

on:
  schedule:
    - cron: '0 8 * * *'  # Runs daily at 8:00 UTC (after ai-news CDN upload at 7:30 UTC)
  workflow_dispatch:  # Allow manual triggering

# Add permissions configuration
permissions:
  contents: write  # Allow pushing to the repository

concurrency:
  group: knowledge-repo-writes
  cancel-in-progress: false

jobs:
  sync-sources:
    runs-on: ubuntu-latest

    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
        with:
          fetch-depth: 0  # Fetch all history for proper comparisons

      - name: Setup Node.js
        uses: actions/setup-node@v4
        with:
          node-version: '23'

      # Step 1: Create directory structure if it doesn't exist
      - name: Ensure directories exist
        run: |
          mkdir -p docs
          mkdir -p github
          mkdir -p daily-silk
          mkdir -p ai-news
          mkdir -p clanktank/episodes
          mkdir -p the-council/episodes

      # Step 2: ElizaOS Docs
      - name: Sync ElizaOS Documentation
        continue-on-error: true # Added for robustness
        run: |
          # Clone the repo temporarily
          git clone -b develop --depth 1 https://github.com/elizaOS/eliza.git temp-elizaos # Changed branch

          # Use tar to find and copy markdown files while preserving directory structure
          find temp-elizaos/packages/docs -name "*.md" -o -name "*.mdx" | tar -cf - -T - | tar -xf - --strip-components=3 -C docs/

          # Clean up
          rm -rf temp-elizaos

      # Step 3: Daily Silk Docs
      - name: Sync Daily Silk Documentation
        continue-on-error: true # Added for robustness
        run: |
          # Clone the repo temporarily
          git clone --depth 1 https://github.com/madjin/daily-silk.git temp-daily-silk

          # Copy only markdown files from data directory
          find temp-daily-silk/data -name "*.md" | tar -cf - -T - | tar -xf - --strip-components=2 -C daily-silk/

          # Clean up
          rm -rf temp-daily-silk

      # Step 4: GitHub Activity Logs
      - name: Sync GitHub Activity Logs
        continue-on-error: true # Added for robustness
        run: |
          # Clone the repo temporarily
          git clone -b _data --single-branch https://github.com/elizaos/elizaos.github.io.git temp-github-data

          # Copy files from the correct paths
          rsync -av temp-github-data/data/elizaos_eliza/ github/
          rsync -av temp-github-data/data/dump/ github/
          rsync -av temp-github-data/data/summaries/ github/summaries/
          rsync -av temp-github-data/data/api/ github/api/
          rsync -av temp-github-data/data/contributors/ github/contributors/

          # Clean up
          rm -rf temp-github-data

      # Step 5: Sync AI News (ElizaOS & Hyperfy)
      - name: Sync AI News (ElizaOS & Hyperfy)
        continue-on-error: true # Added for robustness
        run: |
          # Clone the repo temporarily
          git clone -b gh-pages https://github.com/M3-org/ai-news.git temp-ai-news

          # Copy the elizaos and hyperfy folders
          rsync -av --delete temp-ai-news/elizaos/ ai-news/elizaos/
          rsync -av --delete temp-ai-news/hyperfy/ ai-news/hyperfy/

          # Clean up
          rm -rf temp-ai-news

          # Update daily.json and daily.md from latest dated files
          # Note: json/ already contains CDN URLs (json-cdn/ is deprecated)
          LATEST_JSON=$(ls -t ai-news/elizaos/json/????-??-??.json 2>/dev/null | head -1)
          LATEST_MD=$(ls -t ai-news/elizaos/md/????-??-??.md 2>/dev/null | head -1)
          [ -f "$LATEST_JSON" ] && cp "$LATEST_JSON" ai-news/elizaos/json/daily.json
          [ -f "$LATEST_MD" ] && cp "$LATEST_MD" ai-news/elizaos/md/daily.md

      # Step 6: Sync Clanktank Episodes
      - name: Sync Clanktank Episodes
        continue-on-error: true # Added for robustness
        run: |
          # Clone the repo temporarily
          git clone --depth 1 https://github.com/m3-org/clanktank.git temp-clanktank

          # Copy JSON files from episodes directory
          rsync -av --include="*.json" --exclude="*" temp-clanktank/episodes/ clanktank/episodes/

          # Clean up
          rm -rf temp-clanktank

      # Step 7: Sync The-Council Episodes
      - name: Sync The-Council Episodes
        continue-on-error: true # Added for robustness
        run: |
          # Clone the repo temporarily
          git clone --depth 1 https://github.com/m3-org/the-council.git temp-the-council

          # Copy JSON files from episodes directory
          rsync -av --include="*.json" --exclude="*" temp-the-council/episodes/ the-council/episodes/

          # Clean up
          rm -rf temp-the-council

      # Step 8: Commit and push changes
      - name: Commit and push changes
        uses: stefanzweifel/git-auto-commit-action@v5
        with:
          commit_message: "Daily Knowledge Sync"
          file_pattern: "docs/ github/ daily-silk/ ai-news/ clanktank/ the-council/"
          commit_user_name: "github-actions[bot]"
          commit_user_email: "github-actions[bot]@users.noreply.github.com"
          commit_author: "github-actions[bot] <github-actions[bot]@users.noreply.github.com>"

      - name: Alert on failure
        if: failure()
        uses: elizaOS/knowledge/.github/actions/alert-failure@main
        with:
          webhook-url: ${{ secrets.ALERT_WEBHOOK_URL }}
          workflow-name: 'Sync Knowledge Sources'