git.iko.soy/Multigram
on main help sign in
Add the Sync with Forkgram workflow
d251617Claude
.github/workflows/sync-forkgram.ymlnew file+177
compare with main
unifiedsplit
@@ -0,0 +1,177 @@
1+name: Sync with Forkgram
2+run-name: Sync with Forkgram${{ inputs.use_next && ' (use multigram-next)' || '' }}${{ inputs.allow_drop && ' (allow drop)' || '' }}${{ inputs.force && ' (force)' || '' }}
3+
4+# Every 6 hours: snapshot a new Forkgram release onto "forkgram", replay the MultiGram
5+# stack onto it as "multigram-next", compile that, and only if it compiles move "multigram"
6+# to it. Conflicts and failures go to an issue labelled forkgram-sync; multigram stays as
7+# it was. A multigram-next changed by hand is never replaced. See README.md and
8+# scripts/sync-forkgram.sh.
9+on:
10+ schedule:
11+ - cron: '23 */6 * * *'
12+ workflow_dispatch:
13+ inputs:
14+ force:
15+ description: Rebuild multigram-next even without a new Forkgram release (tests the pipeline, or retries after a failed compile)
16+ type: boolean
17+ default: false
18+ use_next:
19+ description: Skip the rebase; compile and promote the multigram-next branch you fixed by hand
20+ type: boolean
21+ default: false
22+ allow_drop:
23+ description: Only with use_next. Accept a multigram-next that leaves out some of multigram's commits (Forkgram has them now, or you dropped them on purpose)
24+ type: boolean
25+ default: false
26+
27+permissions: {}
28+
29+concurrency:
30+ group: forkgram-sync
31+ cancel-in-progress: false
32+
33+jobs:
34+ sync:
35+ name: Snapshot Forkgram and replay the stack
36+ runs-on: ubuntu-latest
37+ timeout-minutes: 60
38+ permissions:
39+ contents: write # push forkgram, multigram-next and refs/multigram-sync/next
40+ issues: read # see whether the report job has anything to do
41+ outputs:
42+ status: ${{ steps.sync.outputs.status }}
43+ message: ${{ steps.sync.outputs.message }}
44+ error: ${{ steps.sync.outputs.error }}
45+ new_snapshot: ${{ steps.sync.outputs.new_snapshot }}
46+ forkgram_commit: ${{ steps.sync.outputs.forkgram_commit }}
47+ forkgram_version: ${{ steps.sync.outputs.forkgram_version }}
48+ snapshot: ${{ steps.sync.outputs.snapshot }}
49+ previous_snapshot: ${{ steps.sync.outputs.previous_snapshot }}
50+ multigram: ${{ steps.sync.outputs.multigram }}
51+ multigram_base: ${{ steps.sync.outputs.multigram_base }}
52+ candidate: ${{ steps.sync.outputs.candidate }}
53+ tag_version: ${{ steps.sync.outputs.tag_version }}
54+ conflict_commit: ${{ steps.sync.outputs.conflict_commit }}
55+ conflict_subject: ${{ steps.sync.outputs.conflict_subject }}
56+ conflict_position: ${{ steps.sync.outputs.conflict_position }}
57+ conflict_files: ${{ steps.sync.outputs.conflict_files }}
58+ conflict_details: ${{ steps.sync.outputs.conflict_details }}
59+ dropped: ${{ steps.sync.outputs.dropped }}
60+ existing_next: ${{ steps.sync.outputs.existing_next }}
61+ forkgram_tip: ${{ steps.sync.outputs.forkgram_tip }}
62+ report: ${{ steps.need.outputs.report }}
63+ steps:
64+ # Blobless: the history and trees of forkgram and multigram come cheap, and file
65+ # contents are fetched only when the rebase needs them.
66+ - uses: actions/checkout@v4
67+ with:
68+ fetch-depth: 1
69+ filter: blob:none
70+ - name: Sync
71+ id: sync
72+ env:
73+ FORCE: ${{ inputs.force }}
74+ USE_NEXT: ${{ inputs.use_next }}
75+ ALLOW_DROP: ${{ inputs.allow_drop }}
76+ run: |
77+ git config user.name 'github-actions[bot]'
78+ git config user.email '41898282+github-actions[bot]@users.noreply.github.com'
79+ args=(--remote origin --push)
80+ if [ "$FORCE" = true ]; then args+=(--force); fi
81+ if [ "$USE_NEXT" = true ]; then args+=(--use-next); fi
82+ if [ "$ALLOW_DROP" = true ]; then args+=(--allow-drop); fi
83+ bash scripts/sync-forkgram.sh "${args[@]}"
84+ # Most scheduled runs find nothing new. Skip the report job then, since every job
85+ # costs at least a billed minute: it only has work when an issue is open to close
86+ # (up-to-date), or none is open yet (behind).
87+ - name: Check whether the report job has anything to do
88+ id: need
89+ if: steps.sync.outputs.status == 'up-to-date' || steps.sync.outputs.status == 'behind'
90+ env:
91+ GH_TOKEN: ${{ github.token }}
92+ GH_REPO: ${{ github.repository }}
93+ STATUS: ${{ steps.sync.outputs.status }}
94+ run: |
95+ open=$(gh issue list --label forkgram-sync --state open --json number --jq length) || open=unknown
96+ echo "Open forkgram-sync issues: $open"
97+ case "$STATUS:$open" in
98+ up-to-date:0 | behind:[1-9]*) echo report=no >>"$GITHUB_OUTPUT" ;;
99+ *) echo report=yes >>"$GITHUB_OUTPUT" ;;
100+ esac
101+
102+ compile:
103+ name: Compile multigram-next
104+ needs: sync
105+ if: needs.sync.outputs.candidate != ''
106+ permissions:
107+ contents: read
108+ uses: ./.github/workflows/build.yml
109+ with:
110+ ref: ${{ needs.sync.outputs.candidate }}
111+
112+ promote:
113+ name: Move multigram to multigram-next
114+ needs: [sync, compile]
115+ if: needs.compile.result == 'success'
116+ runs-on: ubuntu-latest
117+ timeout-minutes: 15
118+ permissions:
119+ contents: write # move multigram, tag its old tip, delete multigram-next
120+ outputs:
121+ status: ${{ steps.promote.outputs.status }}
122+ tag: ${{ steps.promote.outputs.tag }}
123+ steps:
124+ - uses: actions/checkout@v4
125+ with:
126+ fetch-depth: 1
127+ filter: blob:none
128+ - name: Promote
129+ id: promote
130+ env:
131+ CANDIDATE: ${{ needs.sync.outputs.candidate }}
132+ PREVIOUS: ${{ needs.sync.outputs.multigram }}
133+ VERSION: ${{ needs.sync.outputs.tag_version }}
134+ run: |
135+ bash scripts/sync-forkgram.sh promote --remote origin --shallow \
136+ --candidate "$CANDIDATE" --previous "$PREVIOUS" --version "$VERSION"
137+
138+ report:
139+ name: Report on the forkgram-sync issue
140+ needs: [sync, compile, promote]
141+ if: ${{ !cancelled() && needs.sync.outputs.report != 'no' }}
142+ runs-on: ubuntu-latest
143+ timeout-minutes: 10
144+ permissions:
145+ contents: read # the report script
146+ issues: write # open, comment on and close the forkgram-sync issue
147+ steps:
148+ - uses: actions/checkout@v4
149+ with:
150+ sparse-checkout: scripts
151+ - name: Open, update or close the issue
152+ env:
153+ GH_TOKEN: ${{ github.token }}
154+ GH_REPO: ${{ github.repository }}
155+ RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
156+ SYNC_RESULT: ${{ needs.sync.result }}
157+ COMPILE_RESULT: ${{ needs.compile.result }}
158+ PROMOTE_RESULT: ${{ needs.promote.result }}
159+ PROMOTE_TAG: ${{ needs.promote.outputs.tag }}
160+ SYNC_STATUS: ${{ needs.sync.outputs.status }}
161+ SYNC_MESSAGE: ${{ needs.sync.outputs.message }}
162+ SYNC_ERROR: ${{ needs.sync.outputs.error }}
163+ SYNC_FORKGRAM_COMMIT: ${{ needs.sync.outputs.forkgram_commit }}
164+ SYNC_FORKGRAM_VERSION: ${{ needs.sync.outputs.forkgram_version }}
165+ SYNC_SNAPSHOT: ${{ needs.sync.outputs.snapshot }}
166+ SYNC_PREVIOUS_SNAPSHOT: ${{ needs.sync.outputs.previous_snapshot }}
167+ SYNC_MULTIGRAM_BASE: ${{ needs.sync.outputs.multigram_base }}
168+ SYNC_CANDIDATE: ${{ needs.sync.outputs.candidate }}
169+ SYNC_CONFLICT_COMMIT: ${{ needs.sync.outputs.conflict_commit }}
170+ SYNC_CONFLICT_SUBJECT: ${{ needs.sync.outputs.conflict_subject }}
171+ SYNC_CONFLICT_POSITION: ${{ needs.sync.outputs.conflict_position }}
172+ SYNC_CONFLICT_FILES: ${{ needs.sync.outputs.conflict_files }}
173+ SYNC_CONFLICT_DETAILS: ${{ needs.sync.outputs.conflict_details }}
174+ SYNC_DROPPED: ${{ needs.sync.outputs.dropped }}
175+ SYNC_EXISTING_NEXT: ${{ needs.sync.outputs.existing_next }}
176+ SYNC_FORKGRAM_TIP: ${{ needs.sync.outputs.forkgram_tip }}
177+ run: python3 scripts/sync-report.py
scripts/sync-report.pynew file+272
@@ -0,0 +1,272 @@
1+#!/usr/bin/env python3
2+"""Report a "Sync with Forkgram" run on the GitHub issue labelled forkgram-sync.
3+
4+Runs in the report job of .github/workflows/sync-forkgram.yml, after the sync, compile and
5+promote jobs. It reads their results and the sync outputs from the environment and:
6+
7+ - conflict, compile failure, promote failure, a sync error, or a multigram-next changed by
8+ hand that the sync would have replaced: opens an issue, or comments on the open one, with
9+ what happened and the local commands to fix it (the same event is reported once, however
10+ often the schedule runs);
11+ - multigram still behind with no new release: opens an issue only if none is open;
12+ - promoted, or multigram in step with Forkgram: closes the open issue with a note (naming
13+ any MultiGram commits the new multigram leaves out).
14+
15+Environment: SYNC_RESULT, COMPILE_RESULT, PROMOTE_RESULT (job results); SYNC_* (the sync
16+job's outputs, e.g. SYNC_STATUS, SYNC_FORKGRAM_VERSION); PROMOTE_TAG; RUN_URL; GH_TOKEN
17+and GH_REPO for the gh CLI. REPORT_GH names another gh binary (the self-test uses a fake).
18+Standard library only.
19+"""
20+import hashlib
21+import json
22+import os
23+import re
24+import subprocess
25+import sys
26+import tempfile
27+
28+LABEL = 'forkgram-sync'
29+GH = os.environ.get('REPORT_GH', 'gh')
30+SHA = re.compile(r'^[0-9a-f]{40}$')
31+VERSION = re.compile(r'^[0-9A-Za-z][0-9A-Za-z._+-]{0,39}$')
32+
33+
34+def env(name):
35+ return os.environ.get(name, '').strip()
36+
37+
38+def sha(name):
39+ value = env(name)
40+ return value if SHA.match(value) else ''
41+
42+
43+def short(value):
44+ return value[:10] if value else '?'
45+
46+
47+def version():
48+ value = env('SYNC_FORKGRAM_VERSION')
49+ return value if VERSION.match(value) else 'unknown'
50+
51+
52+def block(text):
53+ """Untrusted text (commit subjects, file names, errors) as a Markdown code block."""
54+ text = text.replace('`', "'").strip() or '(none)'
55+ return '```text\n' + text + '\n```'
56+
57+
58+def gh(*args):
59+ run = subprocess.run([GH, *args], capture_output=True, text=True)
60+ if run.returncode != 0:
61+ sys.stderr.write(run.stderr)
62+ raise SystemExit(f'gh {args[0]} {args[1] if len(args) > 1 else ""} failed')
63+ return run.stdout
64+
65+
66+def gh_body(*args, body):
67+ with tempfile.NamedTemporaryFile('w', suffix='.md', delete=False) as f:
68+ f.write(body)
69+ path = f.name
70+ try:
71+ return gh(*args, '--body-file', path)
72+ finally:
73+ os.unlink(path)
74+
75+
76+def open_issues():
77+ issues = json.loads(gh('issue', 'list', '--label', LABEL, '--state', 'open',
78+ '--json', 'number,title', '--limit', '20') or '[]')
79+ return sorted(i['number'] for i in issues)
80+
81+
82+def open_issue():
83+ numbers = open_issues()
84+ return numbers[0] if numbers else None
85+
86+
87+def already_reported(number, key):
88+ data = json.loads(gh('issue', 'view', str(number), '--json', 'body,comments'))
89+ texts = [data.get('body') or ''] + [c.get('body') or '' for c in data.get('comments', [])]
90+ return any(key in t for t in texts)
91+
92+
93+def marker(key):
94+ return f'<!-- forkgram-sync-key: {key} -->'
95+
96+
97+def title():
98+ return f'Forkgram sync needs a hand (Forkgram {version()})'
99+
100+
101+def run_line():
102+ url = env('RUN_URL')
103+ return f'Run: {url}' if url.startswith('https://') else ''
104+
105+
106+def use_next_step():
107+ return ('Then open **Actions > Sync with Forkgram > Run workflow**, tick **use_next** and run '
108+ 'it. It compiles `multigram-next` and, if that passes, moves `multigram` to it (the '
109+ 'old tip is kept as the tag `multigram-before-<version>`) and closes this issue.')
110+
111+
112+def body_conflict():
113+ snap, prev = sha('SYNC_SNAPSHOT') or '<new snapshot>', sha('SYNC_PREVIOUS_SNAPSHOT') or '<old snapshot>'
114+ commit = sha('SYNC_CONFLICT_COMMIT')
115+ return f"""**Forkgram {version()}** is out and is now on `forkgram` (snapshot `{short(snap)}`), but the MultiGram stack does not apply to it. `multigram` is unchanged.
116+
117+MultiGram commit `{short(commit)}` ({env('SYNC_CONFLICT_POSITION') or '?'}) stops the rebase:
118+{block(env('SYNC_CONFLICT_SUBJECT'))}
119+Files:
120+{block(env('SYNC_CONFLICT_DETAILS') or env('SYNC_CONFLICT_FILES'))}
121+{run_line()}
122+
123+### Fix it locally
124+
125+```sh
126+git fetch origin
127+git switch -C multigram-next origin/multigram
128+git config rerere.enabled true # reuses your fixes if you redo this rebase in this clone
129+git rebase --onto {snap} {prev}
130+# fix the files git names, then:
131+git add <files>
132+git rebase --continue # repeat until the rebase is done
133+git push --force origin multigram-next
134+```
135+
136+To leave a MultiGram commit out instead (Forkgram now does the same thing, say), run `git rebase --skip` at that commit.
137+
138+{use_next_step()}
139+"""
140+
141+
142+def body_compile():
143+ snap, cand = sha('SYNC_SNAPSHOT') or '<snapshot>', sha('SYNC_CANDIDATE')
144+ return f"""The MultiGram stack applied cleanly to **Forkgram {version()}**, but the compile check failed. `multigram` is unchanged; the candidate is on `multigram-next` (`{short(cand)}`).
145+
146+{run_line()}
147+
148+If the log shows a runner or network hiccup, use **Re-run failed jobs** on the run page. Otherwise:
149+
150+### Fix it locally
151+
152+```sh
153+git fetch origin
154+git switch -C multigram-next origin/multigram-next
155+# fix the build, then fold the fix into the MultiGram commit it belongs to:
156+git commit -a --fixup=<that commit>
157+GIT_SEQUENCE_EDITOR=: git rebase -i --autosquash {snap}
158+git push --force origin multigram-next
159+```
160+
161+{use_next_step()}
162+"""
163+
164+
165+def body_promote():
166+ cand = sha('SYNC_CANDIDATE')
167+ return f"""The candidate for **Forkgram {version()}** compiled, but moving `multigram` to it failed, usually because someone pushed to `multigram` or `multigram-next` while the sync ran. `multigram` is unchanged; the candidate is still on `multigram-next` (`{short(cand)}`).
168+
169+{run_line()}
170+
171+Check that `multigram-next` holds everything `multigram` has now (rebase any new commits onto it and push it), then run **Actions > Sync with Forkgram** with **use_next**.
172+"""
173+
174+
175+def body_error():
176+ return f"""The sync stopped with an error. Nothing was pushed after the error; `multigram` is unchanged.
177+
178+{block(env('SYNC_ERROR') or env('SYNC_MESSAGE') or 'The sync job failed before it could say why; see the run log.')}
179+{run_line()}
180+
181+Fix the cause (the message says what it is), then run **Actions > Sync with Forkgram** again.
182+"""
183+
184+
185+def body_next_edited():
186+ nxt, tip, fc = sha('SYNC_EXISTING_NEXT'), sha('SYNC_FORKGRAM_TIP'), sha('SYNC_FORKGRAM_COMMIT')
187+ if tip and tip != fc:
188+ what = f"Forkgram's `dev` has moved on (`{short(tip)}`), but the sync did not take it"
189+ else:
190+ what = 'The sync was asked to rebuild `multigram-next` (force), but did not'
191+ return f"""{what}: `multigram-next` (`{short(nxt)}`) has commits the sync did not make, most likely a fix by hand, and the sync never replaces those. Nothing was pushed; `forkgram` and `multigram` are unchanged.
192+
193+{run_line()}
194+
195+- If `multigram-next` is ready, run **Actions > Sync with Forkgram > Run workflow** with **use_next**: it compiles `multigram-next` and, if that passes, moves `multigram` to it. The next scheduled sync then takes the new Forkgram commit.
196+- If you do not need it any more, delete it (`git push origin --delete multigram-next`); the next sync rebuilds it from `multigram`.
197+"""
198+
199+
200+def body_behind():
201+ return f"""`forkgram` has **Forkgram {version()}** (snapshot `{short(sha('SYNC_SNAPSHOT'))}`), but `multigram` is still based on the older snapshot `{short(sha('SYNC_MULTIGRAM_BASE'))}`: an earlier sync could not move it and no issue is open about it.
202+
203+{run_line()}
204+
205+Run **Actions > Sync with Forkgram** with **force** to try again; if the stack still does not apply, this issue gets the details and the commands to fix it.
206+"""
207+
208+
209+def report(key, body):
210+ number = open_issue()
211+ text = marker(key) + '\n' + body
212+ if number is None:
213+ gh('label', 'create', LABEL, '--color', 'D93F0B', '--force',
214+ '--description', 'The automatic sync with Forkgram needs a hand')
215+ print(gh_body('issue', 'create', '--title', title(), '--label', LABEL, body=text).strip())
216+ return
217+ if key and already_reported(number, marker(key)):
218+ print(f'issue #{number} already reports this; nothing to add')
219+ return
220+ gh('issue', 'edit', str(number), '--title', title())
221+ gh_body('issue', 'comment', str(number), body=text)
222+ print(f'commented on issue #{number}')
223+
224+
225+def close(note):
226+ for number in open_issues():
227+ gh('issue', 'close', str(number), '--reason', 'completed', '--comment', note)
228+ print(f'closed issue #{number}')
229+
230+
231+def main():
232+ results = [env('SYNC_RESULT'), env('COMPILE_RESULT'), env('PROMOTE_RESULT')]
233+ status = env('SYNC_STATUS')
234+ print(f'sync={results[0]} compile={results[1]} promote={results[2]} status={status or "-"}')
235+ if 'cancelled' in results:
236+ print('a job was cancelled; nothing to report')
237+ return
238+ fc = sha('SYNC_FORKGRAM_COMMIT')
239+ cand = sha('SYNC_CANDIDATE')
240+ if status == 'conflict':
241+ report(f'conflict {fc} {sha("SYNC_CONFLICT_COMMIT")}', body_conflict())
242+ elif results[0] != 'success':
243+ err = env('SYNC_ERROR') or 'unknown'
244+ report('error ' + hashlib.sha1(err.encode()).hexdigest()[:12], body_error())
245+ elif status == 'next-edited':
246+ report(f'next-edited {sha("SYNC_EXISTING_NEXT")} {sha("SYNC_FORKGRAM_TIP")}', body_next_edited())
247+ elif status == 'candidate' and results[1] == 'failure':
248+ report(f'compile {cand}', body_compile())
249+ elif status == 'candidate' and results[1] == 'success' and results[2] == 'failure':
250+ report(f'promote {cand}', body_promote())
251+ elif status == 'candidate' and results[2] == 'success':
252+ tag = env('PROMOTE_TAG')
253+ moved = f'the old tip is tagged `{tag}`' if re.match(r'^[0-9A-Za-z._-]+$', tag) else 'it already was the candidate'
254+ note = f'Done: `multigram` now carries Forkgram {version()} (`{short(cand)}`); {moved}. {run_line()}'.strip()
255+ if env('SYNC_DROPPED'):
256+ note += ('\n\nThese MultiGram commits are not on the new `multigram` (Forkgram already has '
257+ 'them, or `multigram-next` left them out with allow_drop):\n' + block(env('SYNC_DROPPED')))
258+ close(note)
259+ elif status == 'behind':
260+ if open_issue() is None:
261+ report('', body_behind())
262+ else:
263+ print('multigram is behind and an issue is already open; nothing to add')
264+ elif status == 'up-to-date':
265+ if open_issue() is not None:
266+ close(f'`multigram` is in step with Forkgram {version()} again. {run_line()}'.strip())
267+ else:
268+ print('nothing to report')
269+
270+
271+if __name__ == '__main__':
272+ main()