scripts/sync-forkgram.sh · Multigram

git.iko.soy/Multigram

on main help sign in

Multigram/scripts/sync-forkgram.sh

848 lines 40 kB history raw

1#!/usr/bin/env bash

2# Keep MultiGram in step with Forkgram.

4# scripts/sync-forkgram.sh [sync] [--remote NAME [--push]] [--force | --use-next [--allow-drop]]

5# [--url URL] [--branch NAME] [--history-depth N]

6# scripts/sync-forkgram.sh promote --remote NAME --candidate SHA --previous SHA

7# --version VERSION [--shallow]

9# sync (the default)

10# 1. Reads the tip of Forkgram's dev branch. If it is the commit the latest "forkgram"

11# snapshot was made from, there is nothing to do (exit 0), unless --force.

12# 2. Otherwise it makes a snapshot commit on "forkgram": the tip's tree without the

13# .github folder (.gitmodules and the submodule links stay), parented on the previous

14# snapshot, with the Forkgram-Commit, Forkgram-Version and Telegram-Base lines.

15# 3. It replays the MultiGram stack (previous-snapshot..multigram) onto the snapshot with

16# "git rebase --onto", in a temporary worktree, as branch "multigram-next". rerere is on,

17# so conflicts already resolved in this clone are resolved the same way (a fresh CI clone

18# has no recorded resolutions). On a conflict it aborts and reports the stack commit and

19# the files (exit 2). The stack must be linear: a merge commit that carries changes of

20# its own stops the sync, because the replay flattens merges.

21# It never replaces a multigram-next that holds commits it did not make (a fix by hand):

22# it then stops with status next-edited and changes nothing. refs/multigram-sync/next

23# (pushed with multigram-next, and kept locally) records the last multigram-next it built.

24# --remote NAME read forkgram and multigram from that remote (default: the local branches)

25# --push push forkgram and multigram-next to NAME (one atomic push, with leases)

26# --force no new Forkgram release: rebuild multigram-next on the current snapshot

27# (tests the pipeline, or retries after a failed compile)

28# --use-next take the multigram-next branch as it is (a stack fixed by hand) as the

29# candidate; nothing is fetched from Forkgram and nothing is rebuilt. It

30# must still carry every commit of multigram's stack (matched by author and

31# subject, or by patch id); --allow-drop accepts one that leaves some out.

32# The results are left in the local branches "forkgram" and "multigram-next". The

33# checked-out worktree is never touched, a branch checked out anywhere is not moved, and

34# a local multigram-next whose commits are on no other branch or tag is not moved either.

35#

36# promote

37# Moves "multigram" to a candidate that passed the compile check: tags the old tip

38# multigram-before-VERSION, moves multigram with --force-with-lease and deletes

39# multigram-next, in one atomic push. --shallow fetches only the two tips.

40#

41# Environment: FORKGRAM_URL (default https://github.com/forkgram/TelegramAndroid),

42# FORKGRAM_BRANCH (default dev), FORKGRAM_HISTORY_DEPTH (default 600: how many Forkgram

43# commits to search for Telegram's "update to X" commit; 0 = do not search).

44# When GITHUB_OUTPUT / GITHUB_STEP_SUMMARY are set, the outputs and a summary go there.

45#

46# Outputs (key=value, printed at the end and appended to $GITHUB_OUTPUT):

48# promoted | unchanged | error (promote)

49# message one line for people

50# new_snapshot yes | no

51# forkgram_commit, forkgram_version, telegram_base, snapshot

52# the Forkgram commit, its version, its Telegram base and the forkgram

53# snapshot the candidate is built on (new or current)

54# previous_snapshot, previous_forkgram_commit, previous_version

55# the forkgram tip before this run

56# multigram the multigram tip the candidate was built from

57# multigram_base the snapshot multigram is based on (differs from previous_snapshot

58# when multigram is behind)

59# stack_count MultiGram commits on multigram; candidate_count: on multigram-next

60# candidate multigram-next's commit, empty when there is none

61# existing_next the multigram-next this run found (next-edited: the one it left alone)

62# tag_version the version, safe for a tag name

63# dropped MultiGram commits the candidate leaves out ("sha subject" lines with

64# --allow-drop, else git's lines about commits Forkgram already has)

65# missing --use-next without --allow-drop: the commits multigram-next lacks

66# rerere_resolved conflicts resolved from recorded resolutions ("commit path" lines)

67# conflict_commit, conflict_subject, conflict_position,

68# conflict_files (paths), conflict_details ("what happened: path" lines)

69# forkgram_tip the tip of Forkgram's branch this run saw

70# pushed the branches pushed; tag (promote): the tag made for the old multigram

71# error what went wrong

72# Exit codes: 0 done (see status), 1 error, 2 conflict.

73set -euo pipefail

75FORKGRAM_URL=${FORKGRAM_URL:-https://github.com/forkgram/TelegramAndroid}

76FORKGRAM_BRANCH=${FORKGRAM_BRANCH:-dev}

77FORKGRAM_HISTORY_DEPTH=${FORKGRAM_HISTORY_DEPTH:-600}

79FG=forkgram

80MG=multigram

81NEXT=multigram-next

82TIP_REF=refs/multigram-sync/forkgram-tip

83REC_REF=refs/multigram-sync/next # the last multigram-next this script built by replaying

84export GIT_TERMINAL_PROMPT=0

86NL=$'\n'

87WORK=

88WT=

89ERROR_MSG=

90OUT_KEYS=()

91GITID=()

93# ---------------------------------------------------------------- output and logging

95log() { printf 'sync: %s\n' "$*" >&2; }

97annotate() { # level message

98 if [ -n "${GITHUB_ACTIONS:-}" ]; then

99 local m=$2

100 m=${m//'%'/'%25'}; m=${m//$'\r'/'%0D'}; m=${m//$'\n'/'%0A'}

101 printf '::%s::%s\n' "$1" "$m"

102 else

103 printf 'sync: %s: %s\n' "$1" "$2" >&2

104 fi

105}

106

107warn() { annotate warning "$*"; }

108

109die() {

110 ERROR_MSG=$*

111 out status error

112 out error "$ERROR_MSG"

113 out message "$ERROR_MSG"

114 exit 1

115}

116

117out() { # key value...

118 local k=$1 have

119 shift

120 printf -v "o_$k" '%s' "$*"

121 have=" ${OUT_KEYS[*]-} "

122 case $have in *" $k "*) ;; *) OUT_KEYS+=("$k") ;; esac

123}

124

125get() { local n="o_$1"; printf '%s' "${!n-}"; }

126

127random_hex() { od -An -N16 -tx1 /dev/urandom | tr -d ' \n'; }

128

129# Prints text that may hold other people's words (commit subjects, file names) so that the

130# Actions runner does not read it as workflow commands.

131print_untrusted() {

132 if [ -n "${GITHUB_ACTIONS:-}" ]; then

133 local token

134 token=$(random_hex)

135 printf '::stop-commands::%s\n%s\n::%s::\n' "$token" "$1" "$token"

136 else

137 printf '%s\n' "$1"

138 fi

139}

140

141emit_outputs() {

142 [ ${#OUT_KEYS[@]} -gt 0 ] || return 0

143 local k v delim text=

144 delim=ghadelimiter_$(random_hex)

145 for k in "${OUT_KEYS[@]}"; do

146 v=$(get "$k")

147 if [ -n "${GITHUB_OUTPUT:-}" ]; then

148 printf '%s<<%s\n%s\n%s\n' "$k" "$delim" "$v" "$delim" >>"$GITHUB_OUTPUT"

149 fi

150 case $v in

151 *$'\n'*) text+="$k:"$'\n'"$(printf '%s\n' "$v" | sed 's/^/ /')"$'\n' ;;

152 *) text+="$k=$v"$'\n' ;;

153 esac

154 done

155 print_untrusted "---- outputs ----"$'\n'"${text%$'\n'}"

156}

157

158md() { printf '%s' "$1" | tr '`' "'"; } # untrusted text for a Markdown code span or block

159

160write_summary() {

161 [ -n "${GITHUB_STEP_SUMMARY:-}" ] && [ ${#OUT_KEYS[@]} -gt 0 ] || return 0

162 local k v

163 {

164 printf '### Forkgram sync: %s\n\n' "$(get status)"

165 printf '```text\n%s\n```\n\n' "$(md "$(get message)")"

166 printf '| | |\n|---|---|\n'

167 for k in forkgram_version forkgram_commit telegram_base new_snapshot snapshot previous_version \

168 previous_snapshot forkgram_tip multigram multigram_base stack_count existing_next candidate \

169 candidate_count pushed tag; do

170 v=$(get "$k")

171 [ -z "$v" ] || printf '| %s | `%s` |\n' "$k" "$(md "$v" | tr '|' '/')"

172 done

173 for k in conflict_commit conflict_subject conflict_position conflict_details dropped missing rerere_resolved error; do

174 v=$(get "$k")

175 [ -z "$v" ] || printf '\n**%s**\n```text\n%s\n```\n' "$k" "$(md "$v")"

176 done

177 } >>"$GITHUB_STEP_SUMMARY"

178}

179

180cleanup() {

181 local rc=$?

182 set +e

183 if [ -n "$WT" ] && [ -e "$WT" ]; then

184 git -C "$WT" rebase --abort >/dev/null 2>&1

185 git worktree remove --force "$WT" >/dev/null 2>&1 || { rm -rf "$WT"; git worktree prune; }

186 fi

187 git update-ref -d "$TIP_REF" >/dev/null 2>&1

188 [ -z "$WORK" ] || rm -rf "$WORK"

189 if [ $rc -ne 0 ] && [ -z "$(get status)" ]; then

190 out status error

191 out error "${ERROR_MSG:-the script stopped unexpectedly (exit $rc); see the log above}"

192 fi

193 if [ "$(get status)" = error ]; then annotate error "$(get error)"; fi

194 emit_outputs

195 write_summary

196 exit $rc

197}

198trap cleanup EXIT

199

200# ---------------------------------------------------------------- git helpers

201

202short() { git rev-parse --short "$1" 2>/dev/null || printf '%s' "${1:0:10}"; }

203

204enter_repo() {

205 local top

206 top=$(git rev-parse --show-toplevel 2>/dev/null) || die "Run this inside a clone of the MultiGram repository."

207 cd "$top"

208 WORK=$(mktemp -d "${RUNNER_TEMP:-${TMPDIR:-/tmp}}/multigram-sync.XXXXXX")

209 # Commits made here (the snapshot, the replayed stack) need a committer identity.

210 if ! git config user.email >/dev/null 2>&1; then

211 GITID=(-c "user.name=MultiGram sync" -c "user.email=multigram-sync@users.noreply.github.com")

212 fi

213}

214

215# git for the commits this script makes: the fallback identity, and no signing (a local

216# signing setup could prompt, or fail, in the middle of the replay).

217g() { git -c commit.gpgSign=false ${GITID[@]+"${GITID[@]}"} "$@"; }

218

219remote_ref() { # "ls-remote output" refname -> sha or empty

220 printf '%s\n' "$1" | awk -v r="$2" '$2 == r { print $1; exit }'

221}

222

223check_free_branches() { # branch... : this script moves these branches

224 local list b

225 list=$(git worktree list --porcelain)

226 for b; do

227 if grep -qxF "branch refs/heads/$b" <<<"$list"; then

228 die "The branch $b is checked out in a worktree. Switch that worktree to another branch first; this script moves $b."

229 fi

230 done

231}

232

233set_branch() { # branch sha why

234 local old

235 old=$(git rev-parse -q --verify "refs/heads/$1" || true)

236 [ "$old" != "$2" ] || return 0

237 git update-ref -m "sync-forkgram: $3" "refs/heads/$1" "$2"

238 if [ -n "$old" ]; then

239 log "local branch $1: $(short "$old") -> $(short "$2")"

240 else

241 log "local branch $1: created at $(short "$2")"

242 fi

243}

244

245# True when replacing multigram-next loses nothing a person made: there is none, it is the

246# one this script built last (the record), or multigram already has all of it.

247next_is_ours() { # next record multigram

248 [ -z "$1" ] || [ "$1" = "$2" ] || git merge-base --is-ancestor "$1" "$3" 2>/dev/null

249}

250

251# Stops before the local multigram-next moves away from commits that are on no other

252# branch, tag or record (say a fix by hand that was not pushed yet).

253check_local_next() { # [the sha it would move to]

254 local cur others

255 cur=$(git rev-parse -q --verify "refs/heads/$NEXT" || true)

256 [ -n "$cur" ] && [ "$cur" != "${1:-}" ] || return 0

257 others=$(git for-each-ref --format='%(refname)' --contains "$cur" | grep -vxF "refs/heads/$NEXT" || true)

258 [ -n "$others" ] ||

259 die "Your local $NEXT ($(short "$cur")) has commits that are on no other branch or tag, so this script does not move it. Push it, or keep it under another name (git branch -m $NEXT <name>), then run again."

260}

261

262# Prints "sha subject" for each commit of multigram's stack (base..mg) that the hand-made

263# multigram-next (old..next) does not carry. A commit counts as carried when next has one

264# with the same author (e-mail and time) and subject, which a rebase, a conflict fix, a

265# reword-free fixup or git am all keep, or one with the same patch id.

266missing_commits() { # base mg old next

267 local f=$WORK/missing

268 {

269 git log --no-merges --no-color --no-ext-diff --format='commit %H' -p "$3..$4" | git patch-id --stable | awk '{ print "P\t" $1 }'

270 git log --no-merges --format='N%x09%ae %at%x09%s' "$3..$4"

271 git log --no-merges --no-color --no-ext-diff --format='commit %H' -p "$1..$2" | git patch-id --stable | awk '{ print "Q\t" $2 "\t" $1 }'

272 git log --no-merges --reverse --format='M%x09%H%x09%ae %at%x09%s' "$1..$2"

273 } >"$f"

274 awk -F'\t' '

275 $1 == "P" { pid[$2] = 1; next }

276 $1 == "N" { k = $0; sub(/^[^\t]*\t/, "", k); have[k] = 1; next }

277 $1 == "Q" { mine[$2] = $3; next }

278 $1 == "M" {

279 k = $0; sub(/^[^\t]*\t[^\t]*\t/, "", k)

280 if ((k in have) || (($2 in mine) && (mine[$2] in pid))) next

281 s = k; sub(/^[^\t]*\t/, "", s)

282 print $2 " " s

283 }' "$f"

284}

285

286# The replay flattens merge commits (git rebase without --rebase-merges). That loses nothing

287# when a merge only joins its parents, so those just get a warning; a merge with changes of

288# its own (a conflict fix, glue code) would be lost, so it stops the sync.

289check_merges() { # old mg

290 local m parents tree bad= clean=0

291 for m in $(git rev-list --merges "$1..$2"); do

292 parents=$(git rev-list --parents -n 1 "$m" | cut -d' ' -f2-)

293 if [ "$(printf '%s\n' $parents | grep -c .)" -eq 2 ] &&

294 tree=$(git merge-tree --write-tree --no-messages $parents 2>/dev/null) &&

295 [ "${tree%%"$NL"*}" = "$(git rev-parse "$m^{tree}")" ]; then

296 clean=$((clean + 1))

297 continue

298 fi

299 bad+="$(short "$m") $(git log -1 --format=%s "$m")$NL"

300 done

301 if [ -n "$bad" ]; then

302 die "$MG has merge commits with changes of their own (the check needs git 2.38 or newer):$NL$bad""The sync replays the stack without merges, so those changes would be lost. Make $MG linear first and keep what the merges added as a commit of its own: git switch -C $MG origin/$MG && git rebase origin/$FG (fix any conflicts), then git restore --source=origin/$MG --staged --worktree :/ && git commit -m 'Keep the changes made in merges', and push it with git push --force-with-lease origin $MG."

303 fi

304 [ "$clean" -eq 0 ] || warn "$MG has $clean merge commit(s) on top of $FG that only join their parents; the replay flattens them into a linear stack."

305}

306

307safe_version() { # prints the version, or "unknown" when it holds odd characters

308 if [[ $1 =~ ^[0-9A-Za-z][0-9A-Za-z._+-]{0,39}$ ]]; then printf '%s' "$1"; else printf unknown; fi

309}

310

311tag_version() { # version fallback -> a string that is safe in a tag name

312 local v

313 v=$(printf '%s' "$1" | tr -c '0-9A-Za-z._-' '-')

314 if [ "$v" = unknown ] || ! git check-ref-format "refs/tags/multigram-before-$v"; then v=$2; fi

315 printf '%s' "$v"

316}

317

318has_github() { [ -n "$(git ls-tree --name-only "$1" -- .github)" ]; }

319

320check_no_github() { # from to what : every commit in from..to

321 local c

322 for c in $(git rev-list "$1..$2"); do

323 if has_github "$c"; then

324 die "Refusing to go on: $3 would contain a .github folder (commit $(short "$c")). The forkgram snapshots and the MultiGram stack must never carry .github (GITHUB_TOKEN cannot push workflow files, and the workflows live on main)."

325 fi

326 done

327}

328

329# ---------------------------------------------------------------- snapshot helpers

330

331S_COMMIT= S_VERSION= S_BASE=

332read_snapshot() { # commit : reads the Forkgram-* lines of a snapshot commit

333 local body

334 body=$(git log -1 --format=%B "$1")

335 S_COMMIT=$(printf '%s\n' "$body" | sed -n 's/^Forkgram-Commit:[[:space:]]*$[0-9a-f]\{40\}$[[:space:]]*$/\1/p' | tail -n 1)

336 S_VERSION=$(printf '%s\n' "$body" | sed -n 's/^Forkgram-Version:[[:space:]]*//p' | tail -n 1 | tr -d '\r')

337 S_BASE=$(printf '%s\n' "$body" | sed -n 's/^Telegram-Base:[[:space:]]*//p' | tail -n 1 | tr -d '\r')

338 [ -n "$S_COMMIT" ] || return 1

339 S_VERSION=$(safe_version "$S_VERSION")

340}

341

342forkgram_version() { # tree subject -> APP_VERSION_NAME from gradle.properties, else the "[Release] X." subject

343 local v

344 v=$(git cat-file -p "$1:gradle.properties" 2>/dev/null |

345 sed -n 's/^[[:space:]]*APP_VERSION_NAME[[:space:]]*=[[:space:]]*//p' | tail -n 1 | tr -d '\r' || true)

346 v=${v%%[[:space:]]*}

347 if [ -z "$v" ]; then

348 v=$(printf '%s\n' "$2" | sed -n 's/^\[Release\][[:space:]]*$[0-9][0-9A-Za-z._-]*[0-9A-Za-z]$\.\{0,1\}[[:space:]]*$/\1/p')

349 fi

350 safe_version "$v"

351}

352

353TBASE=

354find_telegram_base() { # tip : sets TBASE to "sha (update to X)" when Forkgram's history shows it

355 local tip=$1 hist=$WORK/history.git lines line sha ver

356 TBASE=

357 [ "$FORKGRAM_HISTORY_DEPTH" -gt 0 ] || return 0

358 # Commits only (no trees or files): cheap even a few hundred commits deep.

359 if ! git clone --quiet --bare --filter=tree:0 --depth "$FORKGRAM_HISTORY_DEPTH" --single-branch \

360 --branch "$FORKGRAM_BRANCH" --no-tags "$FORKGRAM_URL" "$hist" >"$WORK/history.log" 2>&1; then

361 warn "Could not read Forkgram's history, so the snapshot has no Telegram-Base line."

362 return 0

363 fi

364 if ! git -C "$hist" cat-file -e "$tip^{commit}" 2>/dev/null; then

365 warn "Forkgram's $FORKGRAM_BRANCH moved during the sync, so the snapshot has no Telegram-Base line."

366 return 0

367 fi

368 lines=$(git -C "$hist" log --first-parent --format='%H %s' "$tip")

369 line=$(printf '%s\n' "$lines" | grep -m 1 -E '^[0-9a-f]{40} update to [0-9]' || true)

370 if [ -z "$line" ]; then

371 warn "No Telegram \"update to X\" commit in the last $FORKGRAM_HISTORY_DEPTH Forkgram commits, so the snapshot has no Telegram-Base line."

372 return 0

373 fi

374 sha=${line%% *}

375 ver=$(printf '%s\n' "${line#* }" | sed -n 's/^update to $[0-9][0-9A-Za-z._-]*$.*/\1/p')

376 TBASE="$sha (update to $(safe_version "$ver"))"

377}

378

379source_link() { # commit

380 local u=${FORKGRAM_URL%/}

381 u=${u%.git}

382 case $u in

383 https://github.com/*/*) printf '%s/commit/%s' "$u" "$1" ;;

384 *) printf '%s (%s) at %s' "$FORKGRAM_URL" "$FORKGRAM_BRANCH" "$1" ;;

385 esac

386}

387

388snapshot_tree() { # tree -> the same tree without the top-level .github

389 local entry

390 while IFS= read -r -d '' entry; do

391 [ "${entry#*$'\t'}" = .github ] && continue

392 printf '%s\0' "$entry"

393 done < <(git ls-tree -z "$1") | git mktree -z

394}

395

396NEW_SNAPSHOT=

397make_snapshot() { # tip tip_tree parent version telegram_base : sets NEW_SNAPSHOT

398 local tip=$1 tree msg=$WORK/snapshot-message

399 tree=$(snapshot_tree "$2") || die "Could not build the snapshot tree of $tip."

400 {

401 printf 'Forkgram %s snapshot\n\n' "$4"

402 printf 'Source: %s\n' "$(source_link "$tip")"

403 printf 'Forkgram-Commit: %s\n' "$tip"

404 printf 'Forkgram-Version: %s\n' "$4"

405 if [ -n "$5" ]; then printf 'Telegram-Base: %s\n' "$5"; fi

406 printf "\\nTree is Forkgram's %s branch at that commit with the .github folder removed.\\n" "$FORKGRAM_BRANCH"

407 } >"$msg"

408 NEW_SNAPSHOT=$(g commit-tree "$tree" -p "$3" -F "$msg") || die "Could not create the snapshot commit."

409}

410

411# Gets Forkgram's tip commit's files into this repository. Sets TIP (the commit, re-read

412# from the fetch), TIP_TREE and TIP_SUBJECT.

413TIP= TIP_TREE= TIP_SUBJECT=

414fetch_tip() { # sha from ls-remote

415 local tipgit=(git)

416 TIP=$1

417 if [ -z "$(git config extensions.partialClone || true)" ] && git cat-file -e "$TIP^{commit}" 2>/dev/null; then

418 log "Forkgram ${TIP:0:10} is already here; not fetching it"

419 elif [ "$(git rev-parse --is-shallow-repository)" = true ]; then

420 # A shallow clone (as in CI) takes one more shallow commit in its stride.

421 log "Fetching Forkgram ${TIP:0:10}"

422 git fetch --no-tags --quiet --depth=1 "$FORKGRAM_URL" "+refs/heads/$FORKGRAM_BRANCH:$TIP_REF" ||

423 die "Could not fetch Forkgram's $FORKGRAM_BRANCH from $FORKGRAM_URL."

424 TIP=$(git rev-parse --verify "$TIP_REF^{commit}")

425 else

426 # A full clone: a shallow fetch would turn it into a shallow repository. Fetch into a

427 # scratch repository instead and copy only the tip's tree (its files) into this one.

428 log "Fetching Forkgram ${TIP:0:10} (into a scratch repository)"

429 tipgit=(git -C "$WORK/tip.git")

430 git init --quiet --bare "$WORK/tip.git"

431 "${tipgit[@]}" fetch --no-tags --quiet --depth=1 "$FORKGRAM_URL" "+refs/heads/$FORKGRAM_BRANCH:refs/tip" ||

432 die "Could not fetch Forkgram's $FORKGRAM_BRANCH from $FORKGRAM_URL."

433 TIP=$("${tipgit[@]}" rev-parse --verify "refs/tip^{commit}")

434 "${tipgit[@]}" rev-list --objects "$TIP^{tree}" | "${tipgit[@]}" pack-objects --quiet --stdout |

435 git index-pack --stdin >/dev/null || die "Could not copy Forkgram's files into this repository."

436 fi

437 TIP_TREE=$("${tipgit[@]}" rev-parse --verify "$TIP^{tree}")

438 TIP_SUBJECT=$("${tipgit[@]}" log -1 --format=%s "$TIP")

439}

440

441set_snapshot_outputs() { # snapshot new_snapshot(yes|no)

442 read_snapshot "$1" || die "The snapshot $(short "$1") has no Forkgram-Commit line."

443 out snapshot "$1"

444 out new_snapshot "$2"

445 out forkgram_commit "$S_COMMIT"

446 out forkgram_version "$S_VERSION"

447 out telegram_base "$S_BASE"

448 out tag_version "$(tag_version "$S_VERSION" "$(git rev-parse --short=10 "$S_COMMIT" 2>/dev/null || printf '%s' "${S_COMMIT:0:10}")")"

449}

450

451# ---------------------------------------------------------------- the replay

452

453rebase_in_progress() {

454 local d

455 for d in rebase-merge rebase-apply; do

456 [ ! -d "$(git -C "$WT" rev-parse --path-format=absolute --git-path "$d")" ] || return 0

457 done

458 return 1

459}

460

461conflict_kinds() { # "stages<TAB>path" lines -> "what happened: path" lines

462 awk -F'\t' '{

463 k = $1

464 if (k == "123") w = "changed on both sides"

465 else if (k == "23") w = "added on both sides"

466 else if (k == "13") w = "Forkgram deleted it, MultiGram changed it"

467 else if (k == "12") w = "MultiGram deleted it, Forkgram changed it"

468 else if (k == "2") w = "only on the Forkgram side (rename or delete conflict)"

469 else if (k == "3") w = "only on the MultiGram side (rename or delete conflict)"

470 else w = "conflict (stages " k ")"

471 print w ": " $2

472 }'

473}

474

475# Replays upstream..tip onto onto in a temporary worktree. Sets REPLAY to "ok" and

476# CANDIDATE, or to "conflict" and the conflict_* outputs.

477CANDIDATE=

478REPLAY=

479replay() { # onto upstream tip

480 local onto=$1 upstream=$2 tip=$3 rc=0 rounds=0 from=0 stopped unmerged stages files stack pos total resolved

481 local rlog=$WORK/rebase.log

482 WT=$WORK/replay

483 git worktree add --quiet --detach "$WT" "$tip" >"$WORK/worktree.log" 2>&1 ||

484 { cat "$WORK/worktree.log" >&2; die "Could not create a temporary worktree for the rebase."; }

485 # The caller's git config must not change the replay: no update-refs (it would move the

486 # caller's own branches), no merge-preserving rebase, no autostash, no hooks, no signing.

487 local rb=(git -C "$WT" -c rerere.enabled=true -c rerere.autoUpdate=true -c rebase.updateRefs=false

488 -c rebase.rebaseMerges=false -c rebase.autoStash=false -c rebase.backend=merge

489 -c core.hooksPath=/dev/null -c commit.gpgSign=false ${GITID[@]+"${GITID[@]}"})

490 log "Replaying $(git rev-list --count "$upstream..$tip") MultiGram commit(s) onto $(short "$onto")"

491 # LC_ALL=C: the messages below are read back (rerere and dropped commits).

492 GIT_EDITOR=true LC_ALL=C "${rb[@]}" rebase --no-autosquash --empty=drop --onto "$onto" "$upstream" >"$rlog" 2>&1 || rc=$?

493 while [ $rc -ne 0 ]; do

494 if ! rebase_in_progress; then

495 print_untrusted "$(tail -n 40 "$rlog" | tr '\r' '\n')"

496 die "git rebase failed without a conflict; see the log above."

497 fi

498 unmerged=$(git -C "$WT" ls-files -u)

499 [ -z "$unmerged" ] || break

500 # rerere resolved every conflict of this commit from a recorded resolution: go on.

501 stopped=$(git -C "$WT" rev-parse -q --verify REBASE_HEAD || echo unknown)

502 resolved=$(tail -c +$((from + 1)) "$rlog" | tr '\r' '\n' |

503 sed -n -E "s/^(Resolved|Staged) '(.*)' using previous resolution\.\$/$(short "$stopped") \2/p")

504 [ -n "$resolved" ] || resolved="$(short "$stopped") (files not named by git)"

505 out rerere_resolved "$(get rerere_resolved)${o_rerere_resolved:+$NL}$resolved"

506 rounds=$((rounds + 1))

507 [ $rounds -le 1000 ] || die "git rebase keeps stopping; giving up."

508 log "rerere resolved the conflicts in $(short "$stopped") from a recorded resolution; continuing"

509 rc=0

510 from=$(wc -c <"$rlog")

511 GIT_EDITOR=true LC_ALL=C "${rb[@]}" rebase --continue >>"$rlog" 2>&1 || rc=$?

512 done

513

514 local dropped

515 dropped=$(tr '\r' '\n' <"$rlog" | grep -E '^(dropping |warning: skipped previously applied commit)' || true)

516 [ -z "$dropped" ] || out dropped "$dropped"

517

518 if [ $rc -eq 0 ]; then

519 CANDIDATE=$(git -C "$WT" rev-parse HEAD)

520 REPLAY=ok

521 return 0

522 fi

523

524 stopped=$(git -C "$WT" rev-parse -q --verify REBASE_HEAD || true)

525 stages=$(printf '%s\n' "$unmerged" | awk -F'\t' '{ split($1, a, " "); s[$2] = s[$2] a[3] } END { for (p in s) print s[p] "\t" p }' | sort -t "$(printf '\t')" -k 2)

526 files=$(printf '%s\n' "$stages" | cut -f 2-)

527 stack=$(git rev-list --reverse "$upstream..$tip")

528 total=$(printf '%s\n' "$stack" | grep -c . || true)

529 pos=$(printf '%s\n' "$stack" | grep -n -x -F "${stopped:-none}" | cut -d: -f1 || true)

530 out conflict_commit "$stopped"

531 out conflict_subject "$( [ -z "$stopped" ] || git log -1 --format=%s "$stopped")"

532 out conflict_position "${pos:-?} of $total"

533 out conflict_files "$files"

534 out conflict_details "$(printf '%s\n' "$stages" | conflict_kinds)"

535 print_untrusted "$(tr '\r' '\n' <"$rlog" | grep -v '^Rebasing (' | tail -n 40)"

536 git -C "$WT" rebase --abort >/dev/null 2>&1 || true

537 REPLAY=conflict

538}

539

540# ---------------------------------------------------------------- sync

541

542sync_main() {

543 local remote= push=no force=no use_next=no allow_drop=no

544 while [ $# -gt 0 ]; do

545 case $1 in

546 --remote) remote=${2:?--remote needs a name}; shift 2 ;;

547 --remote=*) remote=${1#*=}; shift ;;

548 --push) push=yes; shift ;;

549 --force) force=yes; shift ;;

550 --use-next) use_next=yes; shift ;;

551 --allow-drop) allow_drop=yes; shift ;;

552 --url) FORKGRAM_URL=${2:?--url needs a URL}; shift 2 ;;

553 --url=*) FORKGRAM_URL=${1#*=}; shift ;;

554 --branch) FORKGRAM_BRANCH=${2:?--branch needs a name}; shift 2 ;;

555 --branch=*) FORKGRAM_BRANCH=${1#*=}; shift ;;

556 --history-depth) FORKGRAM_HISTORY_DEPTH=${2:?--history-depth needs a number}; shift 2 ;;

557 --history-depth=*) FORKGRAM_HISTORY_DEPTH=${1#*=}; shift ;;

558 -h | --help) sed -n '2,/^set -euo/p' "$0" | sed '$d; s/^# \{0,1\}//'; exit 0 ;;

559 *) die "Unknown option: $1 (see --help)" ;;

560 esac

561 done

562 [ "$push" = no ] || [ -n "$remote" ] || die "--push needs --remote."

563 [ "$force" = no ] || [ "$use_next" = no ] || die "--force and --use-next do not go together."

564 [ "$allow_drop" = no ] || [ "$use_next" = yes ] || die "--allow-drop only goes with --use-next."

565 [[ $FORKGRAM_HISTORY_DEPTH =~ ^[0-9]+$ ]] || die "--history-depth must be a number."

566 enter_repo

567 check_free_branches "$FG" "$NEXT"

568

569 # 1. Where forkgram, multigram and multigram-next are now, and the multigram-next this

570 # script built last (rec_prev).

571 local old mg next_prev= rec_prev=

572 if [ -n "$remote" ]; then

573 local heads specs what

574 heads=$(git ls-remote "$remote" "refs/heads/$FG" "refs/heads/$MG" "refs/heads/$NEXT" "$REC_REF") ||

575 die "Could not read the branches of $remote."

576 [ -n "$(remote_ref "$heads" "refs/heads/$FG")" ] ||

577 die "$remote has no $FG branch. It holds the Forkgram snapshots; create it first (see README.md)."

578 [ -n "$(remote_ref "$heads" "refs/heads/$MG")" ] ||

579 die "$remote has no $MG branch yet. Create it (the latest $FG snapshot plus the MultiGram commits) and push it; the sync replays that stack and cannot start without it."

580 next_prev=$(remote_ref "$heads" "refs/heads/$NEXT")

581 rec_prev=$(remote_ref "$heads" "$REC_REF")

582 specs=("+refs/heads/$FG:refs/remotes/$remote/$FG" "+refs/heads/$MG:refs/remotes/$remote/$MG")

583 what="$FG and $MG"

584 if [ -n "$next_prev" ]; then

585 specs+=("+refs/heads/$NEXT:refs/remotes/$remote/$NEXT")

586 what="$FG, $MG and $NEXT"

587 fi

588 log "Fetching $what from $remote"

589 git fetch --no-tags --quiet "$remote" "${specs[@]}" || die "Could not fetch $what from $remote."

590 old=$(git rev-parse --verify "refs/remotes/$remote/$FG^{commit}")

591 mg=$(git rev-parse --verify "refs/remotes/$remote/$MG^{commit}")

592 [ -z "$next_prev" ] || next_prev=$(git rev-parse --verify "refs/remotes/$remote/$NEXT^{commit}")

593 else

594 old=$(git rev-parse -q --verify "refs/heads/$FG^{commit}") ||

595 die "There is no $FG branch here. It holds the Forkgram snapshots; fetch it or pass --remote."

596 mg=$(git rev-parse -q --verify "refs/heads/$MG^{commit}") ||

597 die "There is no $MG branch yet. Create it (the latest $FG snapshot plus the MultiGram commits); the sync replays that stack and cannot start without it."

598 next_prev=$(git rev-parse -q --verify "refs/heads/$NEXT^{commit}" || true)

599 rec_prev=$(git rev-parse -q --verify "$REC_REF^{commit}" || true)

600 fi

601 out existing_next "$next_prev"

602

603 # 2. The latest snapshot and the stack on top of it.

604 read_snapshot "$old" ||

605 die "The tip of $FG ($(short "$old")) has no Forkgram-Commit line, so it is not a Forkgram snapshot."

606 local old_commit=$S_COMMIT old_version=$S_VERSION base behind stack_count

607 out previous_snapshot "$old"

608 out previous_forkgram_commit "$old_commit"

609 out previous_version "$old_version"

610 out multigram "$mg"

611 base=$(git merge-base "$old" "$mg") || die "$MG shares no history with $FG."

612 out multigram_base "$base"

613 if [ "$base" = "$old" ]; then behind=no; else behind=yes; fi

614 if [ "$behind" = yes ] && ! read_snapshot "$base"; then

615 warn "$MG is based on $(short "$base"), which is not a $FG snapshot."

616 fi

617 stack_count=$(git rev-list --count "$old..$mg")

618 out stack_count "$stack_count"

619 [ "$stack_count" -gt 0 ] || warn "$MG has no commits on top of $FG."

620

621 # 3a. A stack fixed by hand, already pushed to multigram-next.

622 if [ "$use_next" = yes ]; then

623 local missing n

624 [ -n "$next_prev" ] || die "There is no $NEXT branch. Push the fixed stack to $NEXT first (see README.md)."

625 git merge-base --is-ancestor "$old" "$next_prev" ||

626 die "$NEXT is not based on the latest $FG snapshot ($(short "$old"), Forkgram $old_version). Rebase it onto $FG first."

627 [ "$next_prev" != "$old" ] || die "$NEXT has no commits on top of $FG."

628 set_snapshot_outputs "$old" no

629 check_no_github "$old" "$next_prev" "$NEXT"

630 # Everything on multigram must still be there: commits pushed to multigram after the fix

631 # was started would otherwise leave the app branch at the promotion.

632 missing=$(missing_commits "$base" "$mg" "$old" "$next_prev")

633 if [ -n "$missing" ]; then

634 n=$(printf '%s\n' "$missing" | grep -c .)

635 if [ "$allow_drop" = yes ]; then

636 out dropped "$missing"

637 warn "$NEXT leaves out $n of $MG's commit(s); --allow-drop accepts that (see the dropped output)."

638 else

639 out missing "$missing"

640 die "$NEXT does not carry $n of $MG's commit(s):$NL$missing${NL}They were probably pushed to $MG after the fix on $NEXT was started. Add them to $NEXT (git switch $NEXT && git cherry-pick <commit>...), push it and run with use_next again. If $NEXT leaves them out on purpose (Forkgram has them now, or you folded or dropped them), run use_next with allow_drop (--allow-drop)."

641 fi

642 fi

643 CANDIDATE=$next_prev

644 out candidate "$CANDIDATE"

645 out candidate_count "$(git rev-list --count "$old..$CANDIDATE")"

646 check_local_next "$CANDIDATE"

647 set_branch "$FG" "$old" "Forkgram $old_version snapshot"

648 set_branch "$NEXT" "$CANDIDATE" "use multigram-next as the candidate"

649 out pushed ""

650 out status candidate

651 out message "Using $NEXT ($(short "$CANDIDATE")) as it is: $(get candidate_count) commit(s) on the Forkgram $old_version snapshot."

652 log "$(get message)"

653 return 0

654 fi

655

656 # 3b. Forkgram's tip.

657 local tip new new_snapshot tbase version

658 log "Reading $FORKGRAM_URL ($FORKGRAM_BRANCH)"

659 tip=$(git ls-remote "$FORKGRAM_URL" "refs/heads/$FORKGRAM_BRANCH") || die "Could not reach Forkgram at $FORKGRAM_URL."

660 tip=$(remote_ref "$tip" "refs/heads/$FORKGRAM_BRANCH")

661 [ -n "$tip" ] || die "Forkgram has no $FORKGRAM_BRANCH branch at $FORKGRAM_URL."

662 out forkgram_tip "$tip"

663

664 if [ "$tip" = "$old_commit" ] && [ "$force" = no ]; then

665 set_snapshot_outputs "$old" no

666 out candidate ""

667 out pushed ""

668 if [ "$behind" = no ]; then

669 out status up-to-date

670 out message "Nothing to do: Forkgram's $FORKGRAM_BRANCH is still $(short "$tip") (Forkgram $old_version), the latest $FG snapshot, and $MG is based on it."

671 log "$(get message)"

672 else

673 out status behind

674 out message "No new Forkgram release, but $MG is still based on $(short "$base"), not on the Forkgram $old_version snapshot ($(short "$old")): the last sync could not move it. Fix the stack on $NEXT and run with --use-next, or retry with --force."

675 warn "$(get message)"

676 fi

677 return 0

678 fi

679

680 # From here on the run builds a new multigram-next. It never replaces one that holds

681 # commits it did not make (a fix by hand): that one waits for --use-next, or for someone

682 # to delete it. Nothing is snapshotted meanwhile, so the fix stays based on the latest

683 # snapshot and --use-next can take it.

684 if ! next_is_ours "$next_prev" "$rec_prev" "$mg"; then

685 set_snapshot_outputs "$old" no

686 out candidate ""

687 out pushed ""

688 out status next-edited

689 if [ "$tip" = "$old_commit" ]; then

690 out message "$NEXT ($(short "$next_prev")) has commits the sync did not make (a fix by hand?), so --force does not replace it. Run with --use-next to compile and promote it, or delete $NEXT to let the sync rebuild it from $MG."

691 else

692 out message "Forkgram's $FORKGRAM_BRANCH has moved to $(short "$tip"), but $NEXT ($(short "$next_prev")) has commits the sync did not make (a fix by hand?), so the sync changes nothing: no new snapshot, no new $NEXT. Run with --use-next to compile and promote $NEXT, or delete it to let the sync rebuild it from $MG; the next run then takes the new Forkgram commit."

693 fi

694 warn "$(get message)"

695 return 0

696 fi

697 check_local_next

698 check_merges "$old" "$mg"

699

700 if [ "$tip" = "$old_commit" ]; then

701 log "No new Forkgram release; --force: rebuilding $NEXT on the current snapshot"

702 new=$old

703 new_snapshot=no

704 else

705 fetch_tip "$tip"

706 [ "$TIP" != "$old_commit" ] ||

707 die "Forkgram's $FORKGRAM_BRANCH moved back to the latest snapshot's commit during the sync; run it again."

708 tip=$TIP

709 out forkgram_tip "$tip"

710 version=$(forkgram_version "$TIP_TREE" "$TIP_SUBJECT")

711 [ "$version" != unknown ] || warn "Could not read the Forkgram version (APP_VERSION_NAME in gradle.properties)."

712 find_telegram_base "$tip"

713 tbase=$TBASE

714 make_snapshot "$tip" "$TIP_TREE" "$old" "$version" "$tbase"

715 new=$NEW_SNAPSHOT

716 new_snapshot=yes

717 log "New snapshot $(short "$new"): Forkgram $version ($(short "$tip")), Telegram base: ${tbase:-not found}"

718 if [ "$version" = "$old_version" ]; then

719 warn "Forkgram's $FORKGRAM_BRANCH moved but its version is still $version."

720 fi

721 fi

722 set_snapshot_outputs "$new" "$new_snapshot"

723 version=$(get forkgram_version)

724 check_no_github "$old" "$new" "$FG"

725

726 # 4. The replay.

727 replay "$new" "$old" "$mg"

728 if [ "$REPLAY" = ok ]; then

729 check_no_github "$new" "$CANDIDATE" "$NEXT"

730 check_local_next "$CANDIDATE"

731 fi

732 set_branch "$FG" "$new" "Forkgram $version snapshot"

733 if [ "$REPLAY" = ok ]; then

734 set_branch "$NEXT" "$CANDIDATE" "replay of $MG onto Forkgram $version"

735 git update-ref -m "sync-forkgram: the $NEXT it built" "$REC_REF" "$CANDIDATE"

736 out candidate "$CANDIDATE"

737 out candidate_count "$(git rev-list --count "$new..$CANDIDATE")"

738 else

739 out candidate ""

740 fi

741

742 # 5. The push. refs/multigram-sync/next records the multigram-next the sync built, so a

743 # later run can tell it from one changed by hand.

744 local refspecs=() leases=() pushed=()

745 if [ "$new_snapshot" = yes ]; then

746 refspecs+=("$new:refs/heads/$FG")

747 leases+=("--force-with-lease=refs/heads/$FG:$old")

748 pushed+=("$FG")

749 fi

750 if [ -n "$CANDIDATE" ]; then

751 refspecs+=("$CANDIDATE:refs/heads/$NEXT" "$CANDIDATE:$REC_REF")

752 leases+=("--force-with-lease=refs/heads/$NEXT:$next_prev" "--force-with-lease=$REC_REF:$rec_prev")

753 pushed+=("$NEXT")

754 fi

755 if [ "$push" = yes ] && [ ${#refspecs[@]} -gt 0 ]; then

756 log "Pushing ${pushed[*]} to $remote"

757 git push --atomic --quiet "${leases[@]}" "$remote" "${refspecs[@]}" || die "The push of ${pushed[*]} to $remote failed."

758 out pushed "${pushed[*]}"

759 else

760 out pushed ""

761 fi

762

763 if [ "$REPLAY" = ok ]; then

764 out status candidate

765 out message "$NEXT ($(short "$CANDIDATE")) is $MG's $(get stack_count) commit(s) replayed onto Forkgram $version ($(short "$(get forkgram_commit)")); $(get candidate_count) commit(s) on top of the snapshot."

766 log "$(get message)"

767 [ -z "$(get dropped)" ] || warn "Some MultiGram commits were left out because Forkgram already has them; see the dropped output."

768 return 0

769 fi

770 out status conflict

771 out message "The MultiGram stack does not apply to Forkgram $version: commit $(short "$(get conflict_commit)") ($(get conflict_position)) conflicts in $(printf '%s\n' "$(get conflict_files)" | grep -c .) file(s). $MG is unchanged."

772 annotate error "$(get message)"

773 return 2

774}

775

776# ---------------------------------------------------------------- promote

777

778promote_main() {

779 local remote= cand= prev= version= shallow=no

780 while [ $# -gt 0 ]; do

781 case $1 in

782 --remote) remote=${2:?}; shift 2 ;;

783 --candidate) cand=${2:?}; shift 2 ;;

784 --previous) prev=${2:?}; shift 2 ;;

785 --version) version=${2:?}; shift 2 ;;

786 --shallow) shallow=yes; shift ;;

787 -h | --help) sed -n '2,/^set -euo/p' "$0" | sed '$d; s/^# \{0,1\}//'; exit 0 ;;

788 *) die "Unknown option: $1 (see --help)" ;;

789 esac

790 done

791 [ -n "$remote" ] || die "promote needs --remote."

792 [[ $cand =~ ^[0-9a-f]{40}$ ]] || die "promote needs --candidate with a full commit id."

793 [[ $prev =~ ^[0-9a-f]{40}$ ]] || die "promote needs --previous with a full commit id."

794 [[ $version =~ ^[0-9A-Za-z._-]+$ ]] || die "promote needs --version (letters, digits, dot, dash)."

795 enter_repo

796

797 local heads now_mg now_next depth=() tag base n existing

798 heads=$(git ls-remote "$remote" "refs/heads/$MG" "refs/heads/$NEXT") || die "Could not read the branches of $remote."

799 now_mg=$(remote_ref "$heads" "refs/heads/$MG")

800 now_next=$(remote_ref "$heads" "refs/heads/$NEXT")

801 [ "$now_mg" = "$prev" ] ||

802 die "$MG on $remote is ${now_mg:-missing}, not $prev that the candidate was built from: it was pushed to meanwhile. Not promoting."

803 [ "$now_next" = "$cand" ] ||

804 die "$NEXT on $remote is ${now_next:-missing}, not the compiled candidate $cand. Not promoting."

805 [ "$shallow" = no ] || depth=(--depth=1)

806 git fetch --no-tags --quiet ${depth[@]+"${depth[@]}"} "$remote" \

807 "+refs/heads/$MG:refs/remotes/$remote/$MG" "+refs/heads/$NEXT:refs/remotes/$remote/$NEXT" ||

808 die "Could not fetch $MG and $NEXT from $remote."

809 ! has_github "$cand" || die "Refusing to promote: the candidate contains a .github folder."

810 out candidate "$cand"

811 out previous_multigram "$prev"

812

813 if [ "$cand" = "$prev" ]; then

814 git push --atomic --quiet "--force-with-lease=refs/heads/$NEXT:$cand" "$remote" ":refs/heads/$NEXT" ||

815 die "Could not delete $NEXT on $remote."

816 out status unchanged

817 out tag ""

818 out message "$MG already is the candidate ($(short "$cand")); deleted $NEXT."

819 log "$(get message)"

820 return 0

821 fi

822

823 base=multigram-before-$version

824 existing=$(git ls-remote --tags "$remote" "refs/tags/$base*" | awk '{ print $2 }') ||

825 die "Could not read the tags of $remote."

826 tag=$base

827 n=1

828 while grep -qxF "refs/tags/$tag" <<<"$existing"; do

829 n=$((n + 1))

830 tag=$base-$n

831 done

832 log "Tagging $(short "$prev") as $tag, moving $MG to $(short "$cand"), deleting $NEXT"

833 git push --atomic --quiet \

834 "--force-with-lease=refs/heads/$MG:$prev" "--force-with-lease=refs/heads/$NEXT:$cand" \

835 "$remote" "$cand:refs/heads/$MG" "$prev:refs/tags/$tag" ":refs/heads/$NEXT" ||

836 die "The push that moves $MG failed; $MG is unchanged."

837 out status promoted

838 out tag "$tag"

839 out message "$MG moved from $(short "$prev") (tagged $tag) to $(short "$cand"); $NEXT deleted."

840 log "$(get message)"

841}

842

843cmd=sync

844case ${1:-} in

845 sync | promote) cmd=$1; shift ;;

846esac

847# A conflict makes sync_main return 2, which (set -e) ends the script with exit code 2.

848"${cmd}_main" "$@"