diff --git a/src/backend/api/CreateBackend.test.ts b/src/backend/api/CreateBackend.test.ts new file mode 100644 index 000000000..9dc438793 --- /dev/null +++ b/src/backend/api/CreateBackend.test.ts @@ -0,0 +1,35 @@ +import {suite} from '@alinea/suite' +import {createRemote} from './CreateBackend.js' + +const test = suite(import.meta) + +test('createRemote routes write/getBlobs to later smart implementation and keeps history on github implementation', async () => { + let writes = 0 + let revisions = 0 + const remote = createRemote( + { + revisions: async () => { + revisions++ + return [] + }, + revisionData: async () => undefined, + getTreeIfDifferent: async () => undefined + }, + { + write: async () => { + writes++ + return {sha: 'next'} + }, + getBlobs: async function* () { + yield ['a'.repeat(40), new Uint8Array([1])] + } + } + ) + await remote.write({} as any) + const blobs = Array<[string, Uint8Array]>() + for await (const blob of remote.getBlobs(['a'.repeat(40)])) blobs.push(blob) + await remote.revisions('x') + test.is(writes, 1) + test.is(revisions, 1) + test.is(blobs.length, 1) +}) diff --git a/src/backend/api/CreateBackend.ts b/src/backend/api/CreateBackend.ts index 9bde48692..35094a9b4 100644 --- a/src/backend/api/CreateBackend.ts +++ b/src/backend/api/CreateBackend.ts @@ -4,6 +4,7 @@ import {assert} from 'alinea/core/util/Assert' import * as driver from 'rado/driver' import {BasicAuth} from './BasicAuth.js' import {DatabaseApi} from './DatabaseApi.js' +import {GitSmartApi} from './GitSmartApi.js' import {GithubApi, type GithubOptions} from './GithubApi.js' import {OAuth2, type OAuth2Options} from './OAuth2.js' @@ -57,12 +58,16 @@ export function createBackend( author, ...options.github }) + const smartApi = new GitSmartApi({ + author, + ...options.github + }) const dbApi = new DatabaseApi(context, {db}) assert(options.oauth2 ?? options.auth, 'No auth method provided') const auth = options.oauth2 ? new OAuth2(context, config, options.oauth2) : new BasicAuth(context, options.auth!) - return createRemote(ghApi, dbApi, auth) + return createRemote(ghApi, smartApi, dbApi, auth) } } diff --git a/src/backend/api/GitSmartApi.test.ts b/src/backend/api/GitSmartApi.test.ts new file mode 100644 index 000000000..7d5a3baca --- /dev/null +++ b/src/backend/api/GitSmartApi.test.ts @@ -0,0 +1,486 @@ +import {suite} from '@alinea/suite' +import {createGitDelta} from 'alinea/core/source/GitDelta' +import {hashBlob} from 'alinea/core/source/GitUtils' +import type {CommitRequest} from 'alinea/core/db/CommitRequest' +import {ShaMismatchError} from 'alinea/core/source/ShaMismatchError' +import {GitSmartApi} from './GitSmartApi.js' +import {parsePack} from './gitSmart/GitSmartPack.js' +import { + flushPkt, + pktLine +} from './gitSmart/GitSmartProtocol.js' + +const test = suite(import.meta) +const encoder = new TextEncoder() +const decoder = new TextDecoder() +const HEAD_SHA = '1'.repeat(40) +const ROOT_TREE_SHA = '2'.repeat(40) +const REPO_TREE_SHA = '3'.repeat(40) +const CONTENT_TREE_SHA = '4'.repeat(40) +const EXISTING_BLOB_SHA = '5'.repeat(40) + +function json(body: unknown, status = 200) { + return new Response(JSON.stringify(body), { + status, + headers: {'Content-Type': 'application/json'} + }) +} + +function requestUrl(input: RequestInfo | URL): string { + if (typeof input === 'string') return input + if (input instanceof URL) return input.toString() + return input.url +} + +async function requestBytes(init?: RequestInit) { + if (!init?.body) return new Uint8Array(0) + return new Uint8Array(await new Response(init.body).arrayBuffer()) +} + +function createApi( + options: Partial[0]> = {} +) { + return new GitSmartApi({ + author: {name: 'Ben', email: 'ben@example.com'}, + owner: 'acme', + repo: 'site', + branch: 'main', + authToken: 'token', + rootDir: 'repo', + contentDir: 'content', + ...options + }) +} + +function baseRequest(request: Partial = {}): CommitRequest { + return { + description: 'Update content', + fromSha: CONTENT_TREE_SHA, + intoSha: 'unused', + checks: [], + changes: [], + ...request + } +} + +function receivePackAdvertisement(head = HEAD_SHA) { + return concat([ + pktLine('# service=git-receive-pack\n'), + flushPkt(), + pktLine( + `${head} refs/heads/main\0report-status side-band-64k ofs-delta\n` + ) + ]) +} + +function uploadPackAdvertisement(head = HEAD_SHA) { + return concat([ + pktLine('# service=git-upload-pack\n'), + flushPkt(), + pktLine( + `${head} refs/heads/main\0side-band-64k ofs-delta allow-reachable-sha1-in-want\n` + ) + ]) +} + +function packResult(payload: Uint8Array) { + return new Response( + concat([pktLine(withChannel(1, concat([encoder.encode('NAK\n'), payload]))), flushPkt()]) + ) +} + +function receivePackResult(status = 'ok') { + const text = + status === 'ok' + ? 'unpack ok\nok refs/heads/main\n' + : `unpack ok\nng refs/heads/main ${status}\n` + return new Response(concat([pktLine(withChannel(1, encoder.encode(text))), flushPkt()])) +} + +test('write pushes pack objects through receive-pack', async () => { + const api = createApi() + const request = baseRequest({ + changes: [ + { + op: 'addContent', + path: 'pages/home.json', + sha: 'sha-add', + contents: '{"title":"Hello"}' + } + ] + }) + const originalFetch = globalThis.fetch + let receiveBody = new Uint8Array(0) + globalThis.fetch = (async (input, init) => { + const url = requestUrl(input) + const parsed = new URL(url) + const method = init?.method ?? 'GET' + if (url === 'https://api.github.com/repos/acme/site/git/ref/heads/main') + return json({object: {sha: HEAD_SHA}}) + if ( + url === `https://api.github.com/repos/acme/site/contents/repo?ref=${HEAD_SHA}` + ) + return json([{path: 'repo/content', sha: CONTENT_TREE_SHA}]) + if (url === `https://api.github.com/repos/acme/site/git/commits/${HEAD_SHA}`) + return json({tree: {sha: ROOT_TREE_SHA}}) + if ( + url === + `https://api.github.com/repos/acme/site/git/trees/${ROOT_TREE_SHA}?recursive=true` + ) { + return json({ + sha: ROOT_TREE_SHA, + truncated: false, + tree: [ + {path: 'repo', mode: '040000', type: 'tree', sha: REPO_TREE_SHA}, + { + path: 'repo/content', + mode: '040000', + type: 'tree', + sha: CONTENT_TREE_SHA + }, + { + path: 'repo/content/existing.json', + mode: '100644', + type: 'blob', + sha: EXISTING_BLOB_SHA + } + ] + }) + } + if ( + url === 'https://github.com/acme/site.git/info/refs?service=git-receive-pack' + ) { + return new Response(receivePackAdvertisement()) + } + if (url === 'https://github.com/acme/site.git/git-receive-pack') { + receiveBody = await requestBytes(init) + return receivePackResult() + } + return new Response(`Unexpected ${method} ${parsed.pathname}`, {status: 500}) + }) as typeof fetch + try { + const result = await api.write(request) + test.is(result.sha.length, 40) + const bodyText = decoder.decode(receiveBody.subarray(0, 256)) + test.ok(bodyText.includes(`${HEAD_SHA} `)) + const pack = await parsePack(receiveBody.subarray(findPackOffset(receiveBody))) + test.is(pack.filter(object => object.type === 'blob').length, 1) + test.is(pack.filter(object => object.type === 'tree').length, 4) + test.is(pack.filter(object => object.type === 'commit').length, 1) + const blob = pack.find(object => object.type === 'blob') + test.is(decoder.decode(blob!.data), '{"title":"Hello"}') + } finally { + globalThis.fetch = originalFetch + } +}) + +test('write uploads binary content through receive-pack', async () => { + const api = createApi() + const request = baseRequest({ + changes: [ + { + op: 'uploadFile', + location: 'uploads/file.bin', + url: 'https://upload.local/file.bin' + } + ] + }) + const originalFetch = globalThis.fetch + let receiveBody = new Uint8Array(0) + globalThis.fetch = (async (input, init) => { + const url = requestUrl(input) + if (url === 'https://upload.local/file.bin') return new Response('uploaded') + if (url === 'https://api.github.com/repos/acme/site/git/ref/heads/main') + return json({object: {sha: HEAD_SHA}}) + if ( + url === `https://api.github.com/repos/acme/site/contents/repo?ref=${HEAD_SHA}` + ) + return json([{path: 'repo/content', sha: CONTENT_TREE_SHA}]) + if (url === `https://api.github.com/repos/acme/site/git/commits/${HEAD_SHA}`) + return json({tree: {sha: ROOT_TREE_SHA}}) + if ( + url === + `https://api.github.com/repos/acme/site/git/trees/${ROOT_TREE_SHA}?recursive=true` + ) { + return json({ + sha: ROOT_TREE_SHA, + truncated: false, + tree: [ + {path: 'repo', mode: '040000', type: 'tree', sha: REPO_TREE_SHA}, + { + path: 'repo/content', + mode: '040000', + type: 'tree', + sha: CONTENT_TREE_SHA + }, + { + path: 'repo/content/existing.json', + mode: '100644', + type: 'blob', + sha: EXISTING_BLOB_SHA + } + ] + }) + } + if ( + url === 'https://github.com/acme/site.git/info/refs?service=git-receive-pack' + ) { + return new Response(receivePackAdvertisement()) + } + if (url === 'https://github.com/acme/site.git/git-receive-pack') { + receiveBody = await requestBytes(init) + return receivePackResult() + } + return new Response(`Unexpected ${url}`, {status: 500}) + }) as typeof fetch + try { + await api.write(request) + const pack = await parsePack(receiveBody.subarray(findPackOffset(receiveBody))) + const blob = pack.find(object => object.type === 'blob') + test.is(decoder.decode(blob!.data), 'uploaded') + } finally { + globalThis.fetch = originalFetch + } +}) + +test('write maps receive-pack ref rejection to ShaMismatchError', async () => { + const api = createApi() + const request = baseRequest({ + changes: [ + { + op: 'addContent', + path: 'pages/home.json', + sha: 'sha-add', + contents: '{"title":"Hello"}' + } + ] + }) + const originalFetch = globalThis.fetch + let refReads = 0 + globalThis.fetch = (async input => { + const url = requestUrl(input) + if (url === 'https://api.github.com/repos/acme/site/git/ref/heads/main') { + refReads++ + return json({ + object: { + sha: refReads === 1 ? HEAD_SHA : '6'.repeat(40) + } + }) + } + if ( + url === `https://api.github.com/repos/acme/site/contents/repo?ref=${HEAD_SHA}` + ) + return json([{path: 'repo/content', sha: CONTENT_TREE_SHA}]) + if (url === `https://api.github.com/repos/acme/site/git/commits/${HEAD_SHA}`) + return json({tree: {sha: ROOT_TREE_SHA}}) + if ( + url === + `https://api.github.com/repos/acme/site/git/trees/${ROOT_TREE_SHA}?recursive=true` + ) { + return json({ + sha: ROOT_TREE_SHA, + truncated: false, + tree: [ + {path: 'repo', mode: '040000', type: 'tree', sha: REPO_TREE_SHA}, + { + path: 'repo/content', + mode: '040000', + type: 'tree', + sha: CONTENT_TREE_SHA + }, + { + path: 'repo/content/existing.json', + mode: '100644', + type: 'blob', + sha: EXISTING_BLOB_SHA + } + ] + }) + } + if ( + url === 'https://github.com/acme/site.git/info/refs?service=git-receive-pack' + ) { + return new Response(receivePackAdvertisement()) + } + if (url === 'https://github.com/acme/site.git/git-receive-pack') { + return receivePackResult('non-fast-forward') + } + return new Response(`Unexpected ${url}`, {status: 500}) + }) as typeof fetch + try { + let caught: unknown + try { + await api.write(request) + } catch (error) { + caught = error + } + test.ok(caught instanceof ShaMismatchError) + } finally { + globalThis.fetch = originalFetch + } +}) + +test('getBlobs fetches multiple blobs through upload-pack', async () => { + const api = createApi() + const hello = encoder.encode('hello') + const world = encoder.encode('world') + const helloSha = await hashBlob(hello) + const worldSha = await hashBlob(world) + const originalFetch = globalThis.fetch + globalThis.fetch = (async (input, init) => { + const url = requestUrl(input) + if ( + url === 'https://github.com/acme/site.git/info/refs?service=git-upload-pack' + ) { + return new Response(uploadPackAdvertisement()) + } + if (url === 'https://github.com/acme/site.git/git-upload-pack') { + const body = decoder.decode(await requestBytes(init)) + test.ok(body.includes(`want ${helloSha}`)) + test.ok(body.includes(`want ${worldSha}`)) + const pack = await import('./gitSmart/GitSmartPack.js').then(mod => + mod.createPack([ + {type: 'blob', data: hello}, + {type: 'blob', data: world} + ]) + ) + return packResult(pack) + } + return new Response(`Unexpected ${url}`, {status: 500}) + }) as typeof fetch + try { + const entries = Array<[string, Uint8Array]>() + for await (const entry of api.getBlobs([helloSha, worldSha])) { + entries.push(entry) + } + test.equal( + entries.map(([sha, value]) => [sha, decoder.decode(value)]), + [ + [helloSha, 'hello'], + [worldSha, 'world'] + ] + ) + } finally { + globalThis.fetch = originalFetch + } +}) + +test('getBlobs decodes delta objects from upload-pack', async () => { + const api = createApi() + const base = encoder.encode('abc123\nline2\nline3\n') + const target = encoder.encode('abc123\nline2 updated\nline3\nline4\n') + const targetSha = await hashBlob(target) + const originalFetch = globalThis.fetch + globalThis.fetch = (async input => { + const url = requestUrl(input) + if ( + url === 'https://github.com/acme/site.git/info/refs?service=git-upload-pack' + ) { + return new Response(uploadPackAdvertisement()) + } + if (url === 'https://github.com/acme/site.git/git-upload-pack') { + return packResult(await createRefDeltaPack(base, target)) + } + return new Response(`Unexpected ${url}`, {status: 500}) + }) as typeof fetch + try { + const entries = Array<[string, Uint8Array]>() + for await (const entry of api.getBlobs([targetSha])) entries.push(entry) + test.is(entries.length, 1) + test.is(entries[0][0], targetSha) + test.is(decoder.decode(entries[0][1]), decoder.decode(target)) + } finally { + globalThis.fetch = originalFetch + } +}) + +test('write requires an author', async () => { + const api = createApi({author: undefined}) + await test.throws( + () => api.write(baseRequest()), + 'GitSmartApi requires an author' + ) +}) + +function concat(parts: Array) { + const length = parts.reduce((sum, part) => sum + part.length, 0) + const out = new Uint8Array(length) + let offset = 0 + for (const part of parts) { + out.set(part, offset) + offset += part.length + } + return out +} + +function withChannel(channel: number, payload: Uint8Array) { + const out = new Uint8Array(payload.length + 1) + out[0] = channel + out.set(payload, 1) + return out +} + +async function createRefDeltaPack(base: Uint8Array, target: Uint8Array) { + const {Blob, CompressionStream, Response} = await import('@alinea/iso') + const {hexToBytes} = await import('alinea/core/source/Utils') + const baseSha = await hashBlob(base) + const delta = createGitDelta(base, target) + const header = new Uint8Array(12) + header.set(encoder.encode('PACK'), 0) + new DataView(header.buffer).setUint32(4, 2, false) + new DataView(header.buffer).setUint32(8, 2, false) + const baseEntry = concat([ + encodeHeader(3, base.length), + await compressed(base, Blob, CompressionStream, Response) + ]) + const deltaEntry = concat([ + encodeHeader(7, delta.length), + hexToBytes(baseSha), + await compressed(delta, Blob, CompressionStream, Response) + ]) + const payload = concat([header, baseEntry, deltaEntry]) + const {sha1Bytes} = await import('alinea/core/source/Utils') + return concat([payload, await sha1Bytes(payload)]) +} + +function encodeHeader(type: number, size: number) { + const bytes = Array() + let n = size >>> 4 + let first = ((type & 0x07) << 4) | (size & 0x0f) + if (n !== 0) first |= 0x80 + bytes.push(first) + while (n !== 0) { + let next = n & 0x7f + n >>>= 7 + if (n !== 0) next |= 0x80 + bytes.push(next) + } + return Uint8Array.from(bytes) +} + +async function compressed( + data: Uint8Array, + BlobClass: any, + Compression: any, + ResponseClass: any +) { + const blob = new BlobClass([data]) + const response = new ResponseClass( + blob.stream().pipeThrough(new Compression('deflate')) + ) + return new Uint8Array(await response.arrayBuffer()) +} + +function findPackOffset(bytes: Uint8Array) { + for (let i = 0; i <= bytes.length - 4; i++) { + if ( + bytes[i] === 0x50 && + bytes[i + 1] === 0x41 && + bytes[i + 2] === 0x43 && + bytes[i + 3] === 0x4b + ) { + return i + } + } + throw new Error('Missing PACK marker') +} diff --git a/src/backend/api/GitSmartApi.ts b/src/backend/api/GitSmartApi.ts new file mode 100644 index 000000000..f76381bbe --- /dev/null +++ b/src/backend/api/GitSmartApi.ts @@ -0,0 +1,346 @@ +import type {CommitApi, SyncApi} from 'alinea/core/Connection' +import type {CommitChange, CommitRequest} from 'alinea/core/db/CommitRequest' +import {HttpError} from 'alinea/core/HttpError' +import {GithubSource} from 'alinea/core/source/GithubSource' +import {ShaMismatchError} from 'alinea/core/source/ShaMismatchError' +import {ReadonlyTree} from 'alinea/core/source/Tree' +import {hashBlob, serializeTreeEntries} from 'alinea/core/source/GitUtils' +import {join} from 'alinea/core/util/Paths' +import {assert} from 'alinea/core/util/Assert' +import type {GithubOptions} from './GithubApi.js' +import { + createPack, + parsePack, + type PackObject +} from './gitSmart/GitSmartPack.js' +import { + findPackStart, + buildReceivePackRequest, + buildUploadPackRequest, + extractSidebandData, + gitBasicAuth, + parseAdvertisement, + parseReceivePackStatus +} from './gitSmart/GitSmartProtocol.js' +import { + hashCommitObject, + serializeCommitObject +} from './gitSmart/GitSmartObjects.js' + +const encoder = new TextEncoder() + +export class GitSmartApi extends GithubSource implements CommitApi, SyncApi { + #options: GithubOptions + + constructor(options: GithubOptions) { + super(options) + this.#options = options + } + + async write(request: CommitRequest): Promise<{sha: string}> { + const author = this.#requireAuthor() + const head = await this.#getLatestCommitOid() + const currentSha = await this.shaAt(head) + if (currentSha !== request.fromSha) + throw new ShaMismatchError(currentSha, request.fromSha) + + const commitMessage = this.#commitMessage(request.description) + const planned = await this.#planCommit( + head, + request.changes, + commitMessage, + author + ) + if (!planned) return {sha: currentSha} + + const adv = await this.#advertise('git-receive-pack') + const refName = `refs/heads/${this.#options.branch}` + const advertisedHead = adv.refs.get(refName) + if (advertisedHead && advertisedHead !== head) + throw new ShaMismatchError(advertisedHead, head) + + const pack = await createPack(planned.objects) + const body = buildReceivePackRequest({ + oldSha: head, + newSha: planned.commitSha, + ref: refName, + capabilities: adv.capabilities, + pack + }) + const response = await this.#gitRequest('git-receive-pack', { + method: 'POST', + headers: { + 'Content-Type': 'application/x-git-receive-pack-request', + Accept: 'application/x-git-receive-pack-result' + }, + body + }) + const bytes = new Uint8Array(await response.arrayBuffer()) + const status = parseReceivePackStatus(bytes) + if (status.unpackStatus && status.unpackStatus !== 'ok') { + throw new HttpError( + 500, + `Receive-pack unpack failed: ${status.unpackStatus}` + ) + } + const refStatus = status.refStatus.get(refName) + if (refStatus && refStatus !== 'ok') { + if (/(non-fast-forward|stale info|fetch first)/i.test(refStatus)) { + const actualHead = await this.#getLatestCommitOid() + throw new ShaMismatchError(actualHead, head) + } + throw new HttpError(500, `Receive-pack failed: ${refStatus}`) + } + return {sha: planned.contentSha} + } + + async *getBlobs( + shas: Array + ): AsyncGenerator<[sha: string, blob: Uint8Array]> { + if (shas.length === 0) return + const unique = Array.from(new Set(shas)) + const adv = await this.#advertise('git-upload-pack') + if (!adv.capabilities.has('allow-reachable-sha1-in-want')) { + yield* super.getBlobs(unique) + return + } + const body = buildUploadPackRequest(unique, adv.capabilities) + const response = await this.#gitRequest('git-upload-pack', { + method: 'POST', + headers: { + 'Content-Type': 'application/x-git-upload-pack-request', + Accept: 'application/x-git-upload-pack-result' + }, + body + }) + const bytes = new Uint8Array(await response.arrayBuffer()) + const {channel1} = extractSidebandData(bytes) + const payload = channel1.length > 0 ? channel1 : bytes + const packStart = findPackStart(payload) + assert(packStart >= 0, 'Upload-pack response did not contain a pack') + const objects = await parsePack(payload.subarray(packStart)) + const bySha = new Map(objects.map(object => [object.sha, object])) + for (const sha of shas) { + const object = bySha.get(sha) + if (!object || object.type !== 'blob') { + throw new Error(`Blob ${sha} not found in upload-pack response`) + } + yield [sha, object.data] + } + } + + async #planCommit( + head: string, + changes: Array, + message: string, + author: NonNullable + ): Promise< + | { + commitSha: string + contentSha: string + objects: Array + } + | undefined + > { + const baseCommit = await this.#restRequest<{tree: {sha: string}}>( + `/repos/${this.#options.owner}/${this.#options.repo}/git/commits/${head}` + ) + const flat = await this.#restRequest<{ + sha: string + tree: Array<{path: string; mode: string; sha: string; type: string}> + truncated: boolean + }>( + `/repos/${this.#options.owner}/${this.#options.repo}/git/trees/${baseCommit.tree.sha}?recursive=true` + ) + assert(flat.truncated === false, 'Repository tree response was truncated') + const currentRoot = ReadonlyTree.fromFlat(flat) + const nextRoot = currentRoot.clone() + const additions = await this.#loadAdditions(changes) + for (const addition of additions) nextRoot.add(addition.path, addition.sha) + for (const deletion of this.#deletions(changes)) + nextRoot.remove(deletion.path) + const compiledRoot = await nextRoot.compile(currentRoot) + const contentEntry = compiledRoot.get(this.contentLocation) + assert( + contentEntry instanceof ReadonlyTree, + `Missing content tree ${this.contentLocation}` + ) + if (compiledRoot.sha === currentRoot.sha) { + return undefined + } + const treeObjects = await this.#collectChangedTreeObjects( + currentRoot, + compiledRoot + ) + const commitData = serializeCommitObject({ + tree: compiledRoot.sha, + parent: head, + message, + author + }) + const commitSha = await hashCommitObject({ + tree: compiledRoot.sha, + parent: head, + message, + author + }) + const blobObjects = additions.map(addition => ({ + type: 'blob' as const, + data: addition.data + })) + return { + commitSha, + contentSha: contentEntry.sha, + objects: [ + ...blobObjects, + ...treeObjects, + {type: 'commit', data: commitData} + ] + } + } + + async #collectChangedTreeObjects( + previous: ReadonlyTree | undefined, + next: ReadonlyTree + ): Promise> { + const objects = Array() + for (const entry of next.entries) { + if (!entry.entries) continue + const node = next.get(entry.name) + assert(node instanceof ReadonlyTree) + const previousNode = previous?.get(entry.name) + objects.push( + ...(await this.#collectChangedTreeObjects( + previousNode instanceof ReadonlyTree ? previousNode : undefined, + node + )) + ) + } + if (previous?.sha !== next.sha) { + objects.push({ + type: 'tree', + data: serializeTreeEntries(next.entries) + }) + } + return objects + } + + async #loadAdditions(changes: Array) { + const additions = Array<{path: string; sha: string; data: Uint8Array}>() + for (const change of changes) { + switch (change.op) { + case 'addContent': { + const data = encoder.encode(change.contents!) + additions.push({ + path: join(this.contentLocation, change.path), + data, + sha: await hashBlob(data) + }) + break + } + case 'uploadFile': { + const data = await this.#fetchUpload(change.url) + additions.push({ + path: join(this.#options.rootDir, change.location), + data, + sha: await hashBlob(data) + }) + break + } + } + } + return additions + } + + #deletions(changes: Array) { + const deletions = Array<{path: string}>() + for (const change of changes) { + switch (change.op) { + case 'deleteContent': + deletions.push({path: join(this.contentLocation, change.path)}) + break + case 'removeFile': + deletions.push({path: join(this.#options.rootDir, change.location)}) + break + } + } + return deletions + } + + #commitMessage(message: string) { + const {author} = this.#options + if (!author) return message + return `${message}\n\nCo-authored-by: ${author.name} <${author.email}>` + } + + #requireAuthor(): NonNullable { + const {author} = this.#options + if (!author) { + throw new Error( + 'GitSmartApi requires an author with name and email to create commits' + ) + } + return author + } + + async #fetchUpload(url: string): Promise { + const response = await fetch(url) + if (!response.ok) + throw new HttpError(response.status, await response.text()) + const data = new Uint8Array(await response.arrayBuffer()) + const {maxBlobBytes} = this.#options + if (maxBlobBytes && data.byteLength > maxBlobBytes) { + throw new HttpError( + 413, + `Upload exceeds max blob size (${data.byteLength} > ${maxBlobBytes})` + ) + } + return data + } + + async #advertise(service: 'git-upload-pack' | 'git-receive-pack') { + const response = await this.#gitRequest(`info/refs?service=${service}`, { + method: 'GET', + headers: { + Accept: `application/x-${service}-advertisement` + } + }) + return parseAdvertisement(new Uint8Array(await response.arrayBuffer())) + } + + async #gitRequest(suffix: string, init: RequestInit): Promise { + const {owner, repo, authToken} = this.#options + const response = await fetch( + `https://github.com/${owner}/${repo}.git/${suffix}`, + { + ...init, + headers: { + Authorization: gitBasicAuth(authToken), + ...init.headers + } + } + ) + if (!response.ok) + throw new HttpError(response.status, await response.text()) + return response + } + + async #getLatestCommitOid(): Promise { + const ref = await this.#restRequest<{object: {sha: string}}>( + `/repos/${this.#options.owner}/${this.#options.repo}/git/ref/heads/${this.#options.branch}` + ) + return ref.object.sha + } + + async #restRequest(path: string): Promise { + const response = await fetch(`https://api.github.com${path}`, { + headers: { + Authorization: `Bearer ${this.#options.authToken}`, + 'Content-Type': 'application/json' + } + }) + if (!response.ok) + throw new HttpError(response.status, await response.text()) + return response.json() as Promise + } +} diff --git a/src/backend/api/GithubApi.test.ts b/src/backend/api/GithubApi.test.ts new file mode 100644 index 000000000..b95ab5e9b --- /dev/null +++ b/src/backend/api/GithubApi.test.ts @@ -0,0 +1,734 @@ +import {suite} from '@alinea/suite' +import type {CommitRequest} from 'alinea/core/db/CommitRequest' +import {btoa} from 'alinea/core/util/Encoding' +import {ShaMismatchError} from 'alinea/core/source/ShaMismatchError' +import {GithubApi} from './GithubApi.js' + +const test = suite(import.meta) + +function json(body: unknown, status = 200) { + return new Response(JSON.stringify(body), { + status, + headers: {'Content-Type': 'application/json'} + }) +} + +function requestUrl(input: RequestInfo | URL): string { + if (typeof input === 'string') return input + if (input instanceof URL) return input.toString() + return input.url +} + +async function requestBody(init?: RequestInit) { + if (!init?.body) return undefined + if (typeof init.body === 'string') return JSON.parse(init.body) + return JSON.parse(await new Response(init.body).text()) +} + +function createApi( + options: Partial[0]> = {} +) { + return new GithubApi({ + owner: 'acme', + repo: 'site', + branch: 'main', + authToken: 'token', + rootDir: 'repo', + contentDir: 'content', + ...options + }) +} + +function baseRequest(request: Partial = {}): CommitRequest { + return { + description: 'Update content', + fromSha: 'tree-old', + intoSha: 'unused', + checks: [], + changes: [], + ...request + } +} + +test('write commits through REST git endpoints', async () => { + const api = createApi() + const request = baseRequest({ + changes: [ + { + op: 'addContent', + path: 'pages/home.json', + sha: 'sha-add', + contents: '{"title":"Hello"}' + }, + { + op: 'uploadFile', + location: 'uploads/file.bin', + url: 'https://upload.local/file.bin' + }, + {op: 'deleteContent', path: 'pages/old.json', sha: 'sha-del'}, + {op: 'removeFile', location: 'uploads/remove.bin'} + ] + }) + const originalFetch = globalThis.fetch + const blobBodies = Array() + let treeBody: any + let commitBody: any + let refBody: any + globalThis.fetch = (async (input, init) => { + const url = requestUrl(input) + if (url === 'https://upload.local/file.bin') return new Response('uploaded') + const parsed = new URL(url) + const method = init?.method ?? 'GET' + const pathname = parsed.pathname + const body = await requestBody(init) + + if ( + method === 'GET' && + pathname === '/repos/acme/site/git/ref/heads/main' + ) { + return json({object: {sha: 'head-1'}}) + } + if (method === 'GET' && pathname === '/repos/acme/site/contents/repo') { + const ref = parsed.searchParams.get('ref') + if (ref === 'head-1') return json([{path: 'repo/content', sha: 'tree-old'}]) + if (ref === 'commit-new') + return json([{path: 'repo/content', sha: 'tree-new-sha'}]) + } + if (method === 'GET' && pathname === '/repos/acme/site/git/commits/head-1') { + return json({tree: {sha: 'base-tree'}}) + } + if (method === 'POST' && pathname === '/repos/acme/site/git/blobs') { + blobBodies.push(body) + const sha = + body.content === btoa('{"title":"Hello"}') + ? 'blob-content' + : 'blob-upload' + return json({sha}) + } + if (method === 'POST' && pathname === '/repos/acme/site/git/trees') { + treeBody = body + return json({sha: 'tree-new'}) + } + if (method === 'POST' && pathname === '/repos/acme/site/git/commits') { + commitBody = body + return json({sha: 'commit-new'}) + } + if (method === 'PATCH' && pathname === '/repos/acme/site/git/refs/heads/main') { + refBody = body + return json({updated: true}) + } + return new Response(`Unexpected ${method} ${url}`, {status: 500}) + }) as typeof fetch + + try { + const result = await api.write(request) + test.is(result.sha, 'tree-new-sha') + test.equal(blobBodies, [ + {content: btoa('{"title":"Hello"}'), encoding: 'base64'}, + {content: btoa('uploaded'), encoding: 'base64'} + ]) + test.equal(treeBody, { + base_tree: 'base-tree', + tree: [ + { + path: 'repo/content/pages/home.json', + mode: '100644', + type: 'blob', + sha: 'blob-content' + }, + { + path: 'repo/uploads/file.bin', + mode: '100644', + type: 'blob', + sha: 'blob-upload' + }, + { + path: 'repo/content/pages/old.json', + mode: '100644', + type: 'blob', + sha: null + }, + { + path: 'repo/uploads/remove.bin', + mode: '100644', + type: 'blob', + sha: null + } + ] + }) + test.equal(commitBody, { + message: 'Update content', + tree: 'tree-new', + parents: ['head-1'] + }) + test.equal(refBody, {sha: 'commit-new', force: false}) + } finally { + globalThis.fetch = originalFetch + } +}) + +test('write uploads blobs sequentially by default', async () => { + const api = createApi() + const request = baseRequest({ + changes: [ + { + op: 'uploadFile', + location: 'uploads/one.bin', + url: 'https://upload.local/one.bin' + }, + { + op: 'uploadFile', + location: 'uploads/two.bin', + url: 'https://upload.local/two.bin' + } + ] + }) + const originalFetch = globalThis.fetch + let inFlight = 0 + let maxInFlight = 0 + let blobCalls = 0 + let releaseFirst: () => void + const firstGate = new Promise(resolve => { + releaseFirst = resolve + }) + globalThis.fetch = (async (input, init) => { + const url = requestUrl(input) + const parsed = new URL(url) + const method = init?.method ?? 'GET' + const pathname = parsed.pathname + if (url === 'https://upload.local/one.bin' || url === 'https://upload.local/two.bin') + return new Response('uploaded') + if (method === 'GET' && pathname === '/repos/acme/site/git/ref/heads/main') + return json({object: {sha: 'head-1'}}) + if (method === 'GET' && pathname === '/repos/acme/site/contents/repo') { + const ref = parsed.searchParams.get('ref') + if (ref === 'head-1') return json([{path: 'repo/content', sha: 'tree-old'}]) + if (ref === 'commit-new') + return json([{path: 'repo/content', sha: 'tree-new-sha'}]) + } + if (method === 'GET' && pathname === '/repos/acme/site/git/commits/head-1') + return json({tree: {sha: 'base-tree'}}) + if (method === 'POST' && pathname === '/repos/acme/site/git/blobs') { + const call = ++blobCalls + inFlight++ + maxInFlight = Math.max(maxInFlight, inFlight) + if (call === 1) { + setTimeout(() => releaseFirst(), 25) + await firstGate + } + inFlight-- + return json({sha: call === 1 ? 'blob-one' : 'blob-two'}) + } + if (method === 'POST' && pathname === '/repos/acme/site/git/trees') + return json({sha: 'tree-new'}) + if (method === 'POST' && pathname === '/repos/acme/site/git/commits') + return json({sha: 'commit-new'}) + if (method === 'PATCH' && pathname === '/repos/acme/site/git/refs/heads/main') + return json({updated: true}) + return new Response(`Unexpected ${method} ${url}`, {status: 500}) + }) as typeof fetch + try { + const result = await api.write(request) + test.is(result.sha, 'tree-new-sha') + test.is(blobCalls, 2) + test.is(maxInFlight, 1) + } finally { + globalThis.fetch = originalFetch + } +}) + +test('write uses chunked base64 stream and preserves encoded content', async () => { + const api = createApi({blobChunkBytes: 5}) + const request = baseRequest({ + changes: [ + { + op: 'uploadFile', + location: 'uploads/chunk.bin', + url: 'https://upload.local/chunk.bin' + } + ] + }) + const originalFetch = globalThis.fetch + let blobBody: any + globalThis.fetch = (async (input, init) => { + const url = requestUrl(input) + const parsed = new URL(url) + const method = init?.method ?? 'GET' + const pathname = parsed.pathname + if (url === 'https://upload.local/chunk.bin') return new Response('hello world') + if (method === 'GET' && pathname === '/repos/acme/site/git/ref/heads/main') + return json({object: {sha: 'head-1'}}) + if (method === 'GET' && pathname === '/repos/acme/site/contents/repo') { + const ref = parsed.searchParams.get('ref') + if (ref === 'head-1') return json([{path: 'repo/content', sha: 'tree-old'}]) + if (ref === 'commit-new') + return json([{path: 'repo/content', sha: 'tree-new-sha'}]) + } + if (method === 'GET' && pathname === '/repos/acme/site/git/commits/head-1') + return json({tree: {sha: 'base-tree'}}) + if (method === 'POST' && pathname === '/repos/acme/site/git/blobs') { + blobBody = await requestBody(init) + return json({sha: 'blob-upload'}) + } + if (method === 'POST' && pathname === '/repos/acme/site/git/trees') + return json({sha: 'tree-new'}) + if (method === 'POST' && pathname === '/repos/acme/site/git/commits') + return json({sha: 'commit-new'}) + if (method === 'PATCH' && pathname === '/repos/acme/site/git/refs/heads/main') + return json({updated: true}) + return new Response(`Unexpected ${method} ${url}`, {status: 500}) + }) as typeof fetch + try { + const result = await api.write(request) + test.is(result.sha, 'tree-new-sha') + test.equal(blobBody, {content: btoa('hello world'), encoding: 'base64'}) + } finally { + globalThis.fetch = originalFetch + } +}) + +test('write throws when upload response has no readable body stream', async () => { + const api = createApi() + const request = baseRequest({ + changes: [ + { + op: 'uploadFile', + location: 'uploads/fallback.bin', + url: 'https://upload.local/fallback.bin' + } + ] + }) + const originalFetch = globalThis.fetch + let uploadFetches = 0 + globalThis.fetch = (async (input, init) => { + const url = requestUrl(input) + const parsed = new URL(url) + const method = init?.method ?? 'GET' + const pathname = parsed.pathname + if (url === 'https://upload.local/fallback.bin') { + uploadFetches++ + return { + ok: true, + status: 200, + body: null, + arrayBuffer: async () => new TextEncoder().encode('uploaded').buffer, + text: async () => 'uploaded' + } as Response + } + if (method === 'GET' && pathname === '/repos/acme/site/git/ref/heads/main') + return json({object: {sha: 'head-1'}}) + if (method === 'GET' && pathname === '/repos/acme/site/contents/repo') { + const ref = parsed.searchParams.get('ref') + if (ref === 'head-1') return json([{path: 'repo/content', sha: 'tree-old'}]) + if (ref === 'commit-new') + return json([{path: 'repo/content', sha: 'tree-new-sha'}]) + } + if (method === 'GET' && pathname === '/repos/acme/site/git/commits/head-1') + return json({tree: {sha: 'base-tree'}}) + if ( + pathname === '/repos/acme/site/git/blobs' || + pathname === '/repos/acme/site/git/trees' || + pathname === '/repos/acme/site/git/commits' || + pathname === '/repos/acme/site/git/refs/heads/main' + ) + return new Response(`Unexpected ${method} ${url}`, {status: 500}) + return new Response(`Unexpected ${method} ${url}`, {status: 500}) + }) as typeof fetch + try { + await test.throws( + () => api.write(request), + 'Upload response body is not readable' + ) + test.is(uploadFetches, 1) + } finally { + globalThis.fetch = originalFetch + } +}) + +test('write rejects upload bigger than maxBlobBytes', async () => { + const api = createApi({maxBlobBytes: 5}) + const request = baseRequest({ + changes: [ + { + op: 'uploadFile', + location: 'uploads/too-big.bin', + url: 'https://upload.local/too-big.bin' + } + ] + }) + const originalFetch = globalThis.fetch + globalThis.fetch = (async (input, init) => { + const url = requestUrl(input) + const parsed = new URL(url) + const method = init?.method ?? 'GET' + const pathname = parsed.pathname + if (url === 'https://upload.local/too-big.bin') { + return new Response('1234567890', { + headers: {'content-length': '10'} + }) + } + if (method === 'GET' && pathname === '/repos/acme/site/git/ref/heads/main') + return json({object: {sha: 'head-1'}}) + if (method === 'GET' && pathname === '/repos/acme/site/contents/repo') + return json([{path: 'repo/content', sha: 'tree-old'}]) + if (method === 'GET' && pathname === '/repos/acme/site/git/commits/head-1') + return json({tree: {sha: 'base-tree'}}) + if (method === 'POST' && pathname === '/repos/acme/site/git/blobs') { + await requestBody(init) + return json({sha: 'blob-upload'}) + } + return new Response(`Unexpected ${method} ${url}`, {status: 500}) + }) as typeof fetch + try { + await test.throws(() => api.write(request), 'Upload exceeds max blob size') + } finally { + globalThis.fetch = originalFetch + } +}) + +test('write throws ShaMismatchError when ref update races', async () => { + const api = createApi() + const request = baseRequest({ + changes: [ + { + op: 'addContent', + path: 'pages/home.json', + sha: 'sha-add', + contents: '{"title":"Hello"}' + } + ] + }) + const originalFetch = globalThis.fetch + let refsRead = 0 + globalThis.fetch = (async (input, init) => { + const url = requestUrl(input) + const parsed = new URL(url) + const method = init?.method ?? 'GET' + const pathname = parsed.pathname + if ( + method === 'GET' && + pathname === '/repos/acme/site/git/ref/heads/main' + ) { + refsRead++ + const sha = refsRead === 1 ? 'head-1' : 'head-2' + return json({object: {sha}}) + } + if (method === 'GET' && pathname === '/repos/acme/site/contents/repo') { + return json([{path: 'repo/content', sha: 'tree-old'}]) + } + if (method === 'GET' && pathname === '/repos/acme/site/git/commits/head-1') { + return json({tree: {sha: 'base-tree'}}) + } + if (method === 'POST' && pathname === '/repos/acme/site/git/blobs') { + return json({sha: 'blob-content'}) + } + if (method === 'POST' && pathname === '/repos/acme/site/git/trees') { + return json({sha: 'tree-new'}) + } + if (method === 'POST' && pathname === '/repos/acme/site/git/commits') { + return json({sha: 'commit-new'}) + } + if (method === 'PATCH' && pathname === '/repos/acme/site/git/refs/heads/main') { + return new Response('Reference update failed', {status: 422}) + } + return new Response(`Unexpected ${method} ${url}`, {status: 500}) + }) as typeof fetch + + try { + let caught: unknown + try { + await api.write(request) + } catch (error) { + caught = error + } + test.ok(caught instanceof ShaMismatchError) + test.is((caught as ShaMismatchError).actual, 'head-2') + test.is((caught as ShaMismatchError).expected, 'head-1') + } finally { + globalThis.fetch = originalFetch + } +}) + +test('write appends co-author to commit message', async () => { + const api = createApi({author: {name: 'Ben', email: 'ben@example.com'}}) + const request = baseRequest({ + changes: [ + { + op: 'addContent', + path: 'pages/home.json', + sha: 'sha-add', + contents: '{"title":"Hello"}' + } + ] + }) + const originalFetch = globalThis.fetch + let commitMessage = '' + globalThis.fetch = (async (input, init) => { + const url = requestUrl(input) + const parsed = new URL(url) + const method = init?.method ?? 'GET' + const pathname = parsed.pathname + const body = await requestBody(init) + if (method === 'GET' && pathname === '/repos/acme/site/git/ref/heads/main') + return json({object: {sha: 'head-1'}}) + if (method === 'GET' && pathname === '/repos/acme/site/contents/repo') { + const ref = parsed.searchParams.get('ref') + if (ref === 'head-1') return json([{path: 'repo/content', sha: 'tree-old'}]) + if (ref === 'commit-new') + return json([{path: 'repo/content', sha: 'tree-new-sha'}]) + } + if (method === 'GET' && pathname === '/repos/acme/site/git/commits/head-1') + return json({tree: {sha: 'base-tree'}}) + if (method === 'POST' && pathname === '/repos/acme/site/git/blobs') + return json({sha: 'blob-content'}) + if (method === 'POST' && pathname === '/repos/acme/site/git/trees') + return json({sha: 'tree-new'}) + if (method === 'POST' && pathname === '/repos/acme/site/git/commits') { + commitMessage = body.message + return json({sha: 'commit-new'}) + } + if (method === 'PATCH' && pathname === '/repos/acme/site/git/refs/heads/main') + return json({updated: true}) + return new Response(`Unexpected ${method} ${url}`, {status: 500}) + }) as typeof fetch + try { + const result = await api.write(request) + test.is(result.sha, 'tree-new-sha') + test.is( + commitMessage, + 'Update content\n\nCo-authored-by: Ben ' + ) + } finally { + globalThis.fetch = originalFetch + } +}) + +test('write no-op returns existing sha and skips git write endpoints', async () => { + const api = createApi() + const originalFetch = globalThis.fetch + let writeCalls = 0 + globalThis.fetch = (async (input, init) => { + const url = requestUrl(input) + const parsed = new URL(url) + const method = init?.method ?? 'GET' + const pathname = parsed.pathname + if (method === 'GET' && pathname === '/repos/acme/site/git/ref/heads/main') + return json({object: {sha: 'head-1'}}) + if (method === 'GET' && pathname === '/repos/acme/site/contents/repo') + return json([{path: 'repo/content', sha: 'tree-old'}]) + if ( + pathname === '/repos/acme/site/git/blobs' || + pathname === '/repos/acme/site/git/trees' || + pathname === '/repos/acme/site/git/commits' || + pathname === '/repos/acme/site/git/refs/heads/main' + ) { + writeCalls++ + } + if (method === 'GET' && pathname === '/repos/acme/site/git/commits/head-1') + return json({tree: {sha: 'base-tree'}}) + return new Response(`Unexpected ${method} ${url}`, {status: 500}) + }) as typeof fetch + try { + const result = await api.write(baseRequest()) + test.is(result.sha, 'tree-old') + test.is(writeCalls, 0) + } finally { + globalThis.fetch = originalFetch + } +}) + +test('write throws ShaMismatchError on preflight tree mismatch', async () => { + const api = createApi() + const originalFetch = globalThis.fetch + globalThis.fetch = (async (input, init) => { + const url = requestUrl(input) + const parsed = new URL(url) + const method = init?.method ?? 'GET' + const pathname = parsed.pathname + if (method === 'GET' && pathname === '/repos/acme/site/git/ref/heads/main') + return json({object: {sha: 'head-1'}}) + if (method === 'GET' && pathname === '/repos/acme/site/contents/repo') + return json([{path: 'repo/content', sha: 'tree-different'}]) + return new Response(`Unexpected ${method} ${url}`, {status: 500}) + }) as typeof fetch + try { + let caught: unknown + try { + await api.write(baseRequest({fromSha: 'tree-old'})) + } catch (error) { + caught = error + } + test.ok(caught instanceof ShaMismatchError) + test.is((caught as ShaMismatchError).actual, 'tree-different') + test.is((caught as ShaMismatchError).expected, 'tree-old') + } finally { + globalThis.fetch = originalFetch + } +}) + +test('write rethrows original error when ref update fails without head change', async () => { + const api = createApi() + const request = baseRequest({ + changes: [ + { + op: 'addContent', + path: 'pages/home.json', + sha: 'sha-add', + contents: '{"title":"Hello"}' + } + ] + }) + const originalFetch = globalThis.fetch + globalThis.fetch = (async (input, init) => { + const url = requestUrl(input) + const parsed = new URL(url) + const method = init?.method ?? 'GET' + const pathname = parsed.pathname + if (method === 'GET' && pathname === '/repos/acme/site/git/ref/heads/main') + return json({object: {sha: 'head-1'}}) + if (method === 'GET' && pathname === '/repos/acme/site/contents/repo') + return json([{path: 'repo/content', sha: 'tree-old'}]) + if (method === 'GET' && pathname === '/repos/acme/site/git/commits/head-1') + return json({tree: {sha: 'base-tree'}}) + if (method === 'POST' && pathname === '/repos/acme/site/git/blobs') + return json({sha: 'blob-content'}) + if (method === 'POST' && pathname === '/repos/acme/site/git/trees') + return json({sha: 'tree-new'}) + if (method === 'POST' && pathname === '/repos/acme/site/git/commits') + return json({sha: 'commit-new'}) + if (method === 'PATCH' && pathname === '/repos/acme/site/git/refs/heads/main') + return new Response('Ref update failed', {status: 409}) + return new Response(`Unexpected ${method} ${url}`, {status: 500}) + }) as typeof fetch + try { + await test.throws(() => api.write(request), 'Ref update failed') + } finally { + globalThis.fetch = originalFetch + } +}) + +test('revisionData parses JSON and returns undefined for invalid data', async () => { + const api = createApi() + const originalFetch = globalThis.fetch + let graphqlRead = 0 + globalThis.fetch = (async (input, init) => { + const url = requestUrl(input) + const method = init?.method ?? 'GET' + if (url === 'https://api.github.com/graphql' && method === 'POST') { + graphqlRead++ + if (graphqlRead === 1) + return json({data: {repository: {object: {text: '{"id":"entry-1"}'}}}}) + return json({data: {repository: {object: {text: '{invalid json'}}}}) + } + return new Response(`Unexpected ${method} ${url}`, {status: 500}) + }) as typeof fetch + try { + const valid = await api.revisionData('pages/home.json', 'commit-1') + const invalid = await api.revisionData('pages/home.json', 'commit-2') + test.equal(valid, {id: 'entry-1'}) + test.is(invalid, undefined) + } finally { + globalThis.fetch = originalFetch + } +}) + +test('revisions combines history, resolves renames, and sorts newest first', async () => { + const api = createApi() + const originalFetch = globalThis.fetch + let historyCalls = 0 + globalThis.fetch = (async (input, init) => { + const url = requestUrl(input) + const method = init?.method ?? 'GET' + if (url === 'https://api.github.com/graphql' && method === 'POST') { + historyCalls++ + const body = await requestBody(init) + if (!String(body?.query).includes('GetFileHistory')) + return new Response('Unexpected graphql query', {status: 500}) + if (historyCalls === 1) { + return json({ + data: { + repository: { + ref: { + target: { + file0: { + nodes: [ + { + oid: 'c-new', + committedDate: '2024-01-03T00:00:00.000Z', + message: 'Latest', + author: {name: 'Author', email: 'author@example.com'} + }, + { + oid: 'c-old', + committedDate: '2024-01-01T00:00:00.000Z', + message: + 'Old\n\nCo-authored-by: Co ', + author: {name: 'Ignored', email: 'ignored@example.com'} + } + ] + }, + file1: {nodes: []}, + file2: {nodes: []} + } + } + } + } + }) + } + return json({ + data: { + repository: { + ref: { + target: { + file0: { + nodes: [ + { + oid: 'c-rename', + committedDate: '2024-01-02T00:00:00.000Z', + message: 'Renamed file', + author: {name: 'Renamer', email: 'renamer@example.com'} + } + ] + }, + file1: {nodes: []}, + file2: {nodes: []} + } + } + } + } + }) + } + if ( + method === 'GET' && + url === 'https://api.github.com/repos/acme/site/commits/c-old' + ) { + return json({ + files: [ + { + filename: 'repo/pages/home.json', + previous_filename: 'repo/pages/welcome.json' + } + ] + }) + } + if ( + method === 'GET' && + url === 'https://api.github.com/repos/acme/site/commits/c-rename' + ) { + return json({files: [{filename: 'repo/pages/welcome.json'}]}) + } + return new Response(`Unexpected ${method} ${url}`, {status: 500}) + }) as typeof fetch + try { + const revisions = await api.revisions('pages/home.json') + test.equal( + revisions.map(revision => revision.ref), + ['c-new', 'c-rename', 'c-old'] + ) + test.is(revisions[0].file, 'pages/home.json') + test.is(revisions[1].file, 'pages/welcome.json') + test.equal(revisions[2].user, {name: 'Co', email: 'co@example.com'}) + } finally { + globalThis.fetch = originalFetch + } +}) diff --git a/src/backend/api/GithubApi.ts b/src/backend/api/GithubApi.ts index 4907f03ec..f4a32d0d1 100644 --- a/src/backend/api/GithubApi.ts +++ b/src/backend/api/GithubApi.ts @@ -5,9 +5,9 @@ import type { Revision, SyncApi } from 'alinea/core/Connection' +import type {CommitChange, CommitRequest} from 'alinea/core/db/CommitRequest' import type {EntryRecord} from 'alinea/core/EntryRecord' import {HttpError} from 'alinea/core/HttpError' -import type {CommitChange, CommitRequest} from 'alinea/core/db/CommitRequest' import { GithubSource, type GithubSourceOptions @@ -19,6 +19,9 @@ import {join} from 'alinea/core/util/Paths' export interface GithubOptions extends GithubSourceOptions { author?: {name: string; email: string} + blobUploadConcurrency?: number + blobChunkBytes?: number + maxBlobBytes?: number } export class GithubApi @@ -209,58 +212,87 @@ export class GithubApi changes: Array, commitMessage: string ): Promise { + const {owner, repo, branch} = this.#options const {additions, deletions} = await this.#processChanges(changes) - const {owner, repo, branch, authToken} = this.#options - return this.#graphQL( - `mutation CreateCommitOnBranch($input: CreateCommitOnBranchInput!) { - createCommitOnBranch(input: $input) { - commit { - oid - } + const entries = Array<{ + path: string + mode: '100644' + type: 'blob' + sha: string | null + }>() + if (additions.length) { + const blobs = await this.#uploadBlobs(additions) + entries.push( + ...blobs.map(blob => ({ + path: blob.path, + mode: '100644' as const, + type: 'blob' as const, + sha: blob.sha + })) + ) + } + entries.push( + ...deletions.map(deletion => ({ + path: deletion.path, + mode: '100644' as const, + type: 'blob' as const, + sha: null + })) + ) + if (!entries.length) return expectedHeadOid + const baseCommit = await this.#restRequest<{tree: {sha: string}}>( + `/repos/${owner}/${repo}/git/commits/${expectedHeadOid}` + ) + const createdTree = await this.#restRequest<{sha: string}>( + `/repos/${owner}/${repo}/git/trees`, + { + method: 'POST', + body: {base_tree: baseCommit.tree.sha, tree: entries} } - }`, + ) + const createdCommit = await this.#restRequest<{sha: string}>( + `/repos/${owner}/${repo}/git/commits`, { - input: { - branch: { - repositoryNameWithOwner: `${owner}/${repo}`, - branchName: branch - }, - message: {headline: commitMessage}, - fileChanges: {additions, deletions}, - expectedHeadOid + method: 'POST', + body: { + message: commitMessage, + tree: createdTree.sha, + parents: [expectedHeadOid] } - }, - authToken + } ) - .then(result => { - const commitId = result.data.createCommitOnBranch.commit.oid - return commitId - }) - .catch(error => { - if (error instanceof Error) { - const mismatchMessage = /is at ([a-z0-9]+) but expected ([a-z0-9]+)/ - const match = error.message.match(mismatchMessage) - if (match) { - const [_, actual, expected] = match - throw new ShaMismatchError(actual, expected) - } - const expectedMessage = /Expected branch to point to "([a-z0-9]+)"/ - const expectedMatch = error.message.match(expectedMessage) - if (expectedMatch) { - const actualSha = expectedMatch[1] - throw new ShaMismatchError(actualSha, expectedHeadOid) - } + try { + await this.#restRequest( + `/repos/${owner}/${repo}/git/refs/heads/${branch}`, + { + method: 'PATCH', + body: {sha: createdCommit.sha, force: false} } - throw error - }) + ) + } catch (error) { + if ( + error instanceof HttpError && + (error.code === 409 || error.code === 422) + ) { + const actualHead = await this.#getLatestCommitOid() + if (actualHead !== expectedHeadOid) + throw new ShaMismatchError(actualHead, expectedHeadOid) + } + throw error + } + return createdCommit.sha } async #processChanges(changes: Array): Promise<{ - additions: {path: string; contents: string}[] + additions: {path: string; contents?: string; uploadUrl?: string}[] deletions: {path: string}[] }> { const {rootDir} = this.#options - const additions = Array<{path: string; contents: string}>() + const additions = Array<{ + path: string + contents?: string + uploadUrl?: string + }>() const deletions = Array<{path: string}>() for (const change of changes) { @@ -276,7 +308,7 @@ export class GithubApi const file = join(rootDir, change.location) additions.push({ path: file, - contents: await this.#fetchUploadedContent(change.url) + uploadUrl: change.url }) break } @@ -297,24 +329,207 @@ export class GithubApi } async #getLatestCommitOid(): Promise { - const {owner, repo, branch, authToken} = this.#options - return this.#graphQL( - `query GetLatestCommit($owner: String!, $repo: String!, $branch: String!) { - repository(owner: $owner, name: $repo) { - ref(qualifiedName: $branch) { - target { - oid - } + const {owner, repo, branch} = this.#options + const ref = await this.#restRequest<{object: {sha: string}}>( + `/repos/${owner}/${repo}/git/ref/heads/${branch}` + ) + return ref.object.sha + } + + async #uploadBlobs( + additions: Array<{path: string; contents?: string; uploadUrl?: string}> + ): Promise> { + const concurrency = Math.max(1, this.#options.blobUploadConcurrency ?? 1) + const limit = Math.min(concurrency, additions.length) + const result = Array<{path: string; sha: string}>(additions.length) + let next = 0 + const worker = async () => { + while (true) { + const index = next++ + if (index >= additions.length) return + const addition = additions[index] + const blob = await this.#createBlob(addition) + result[index] = {path: addition.path, sha: blob.sha} + } + } + await Promise.all(Array.from({length: limit}, () => worker())) + return result + } + + async #createBlob(addition: { + path: string + contents?: string + uploadUrl?: string + }): Promise<{sha: string}> { + const {owner, repo} = this.#options + if (addition.contents !== undefined) { + return this.#restRequest<{sha: string}>( + `/repos/${owner}/${repo}/git/blobs`, + { + method: 'POST', + body: {content: addition.contents, encoding: 'base64'} } + ) + } + if (addition.uploadUrl) return this.#streamBlobFromUrl(addition.uploadUrl) + throw new Error(`Missing blob contents for ${addition.path}`) + } + + async #streamBlobFromUrl(url: string): Promise<{sha: string}> { + const source = await fetch(url) + if (!source.ok) throw new HttpError(source.status, await source.text()) + const {maxBlobBytes, blobChunkBytes} = this.#options + if (!source.body) throw new Error('Upload response body is not readable') + const chunkSize = Math.max(3, blobChunkBytes ?? 64 * 1024) + const chunked = this.#chunkStream(source.body, chunkSize, maxBlobBytes) + const body = this.#jsonBlobBody(this.#base64EncodeStream(chunked)) + const {owner, repo, authToken} = this.#options + const init: RequestInit & {duplex?: 'half'} = { + method: 'POST', + headers: { + Authorization: `Bearer ${authToken}`, + 'Content-Type': 'application/json' + }, + body, + duplex: 'half' + } + const response = await fetch( + `https://api.github.com/repos/${owner}/${repo}/git/blobs`, + init + ) + if (!response.ok) + throw new HttpError(response.status, await response.text()) + return response.json() as Promise<{sha: string}> + } + + #jsonBlobBody( + base64Stream: ReadableStream + ): ReadableStream { + const encoder = new TextEncoder() + const prefix = encoder.encode('{"content":"') + const suffix = encoder.encode('","encoding":"base64"}') + const reader = base64Stream.getReader() + let sentPrefix = false + return new ReadableStream({ + async pull(controller) { + if (!sentPrefix) { + sentPrefix = true + controller.enqueue(prefix) + return + } + const {done, value} = await reader.read() + if (done) { + controller.enqueue(suffix) + controller.close() + return + } + controller.enqueue(value) + }, + async cancel(reason) { + await reader.cancel(reason) } - }`, - {owner, repo, branch}, - authToken - ).then(result => result.data.repository.ref.target.oid) + }) } - async #fetchUploadedContent(url: string): Promise { - const response = await fetch(url) - return base64.stringify(new Uint8Array(await response.arrayBuffer())) + #base64EncodeStream( + source: ReadableStream + ): ReadableStream { + const alphabet = Uint8Array.from( + 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/', + char => char.charCodeAt(0) + ) + const eq = '='.charCodeAt(0) + let carry = new Uint8Array(0) + return source.pipeThrough( + new TransformStream({ + transform(chunk, controller) { + const bytes = new Uint8Array(carry.length + chunk.length) + bytes.set(carry, 0) + bytes.set(chunk, carry.length) + const fullLength = bytes.length - (bytes.length % 3) + if (fullLength > 0) { + const out = new Uint8Array((fullLength / 3) * 4) + let outPos = 0 + for (let i = 0; i < fullLength; i += 3) { + const n = (bytes[i] << 16) | (bytes[i + 1] << 8) | bytes[i + 2] + out[outPos++] = alphabet[(n >> 18) & 63] + out[outPos++] = alphabet[(n >> 12) & 63] + out[outPos++] = alphabet[(n >> 6) & 63] + out[outPos++] = alphabet[n & 63] + } + controller.enqueue(out) + } + carry = bytes.slice(fullLength) + }, + flush(controller) { + if (carry.length === 0) return + const a = carry[0] + const b = carry.length > 1 ? carry[1] : 0 + const n = (a << 16) | (b << 8) + const out = new Uint8Array(4) + out[0] = alphabet[(n >> 18) & 63] + out[1] = alphabet[(n >> 12) & 63] + out[2] = carry.length > 1 ? alphabet[(n >> 6) & 63] : eq + out[3] = eq + controller.enqueue(out) + } + }) + ) + } + + #chunkStream( + source: ReadableStream, + chunkSize: number, + maxBytes?: number + ): ReadableStream { + let carry = new Uint8Array(0) + let total = 0 + return source.pipeThrough( + new TransformStream({ + transform(chunk, controller) { + total += chunk.byteLength + if (maxBytes && total > maxBytes) { + throw new HttpError( + 413, + `Upload exceeds max blob size (${total} > ${maxBytes})` + ) + } + const merged = new Uint8Array(carry.length + chunk.length) + merged.set(carry, 0) + merged.set(chunk, carry.length) + let offset = 0 + while (merged.length - offset >= chunkSize) { + controller.enqueue(merged.subarray(offset, offset + chunkSize)) + offset += chunkSize + } + carry = merged.subarray(offset) + }, + flush(controller) { + if (carry.length) controller.enqueue(carry) + } + }) + ) + } + + async #restRequest( + path: string, + options: { + method?: 'GET' | 'POST' | 'PATCH' + body?: object + } = {} + ): Promise { + const {authToken} = this.#options + const response = await fetch(`https://api.github.com${path}`, { + method: options.method ?? 'GET', + headers: { + Authorization: `Bearer ${authToken}`, + 'Content-Type': 'application/json' + }, + body: options.body ? JSON.stringify(options.body) : undefined + }) + if (!response.ok) + throw new HttpError(response.status, await response.text()) + if (response.status === 204) return undefined as T + return response.json() as Promise } } diff --git a/src/backend/api/gitSmart/GitSmartObjects.test.ts b/src/backend/api/gitSmart/GitSmartObjects.test.ts new file mode 100644 index 000000000..61a665ccf --- /dev/null +++ b/src/backend/api/gitSmart/GitSmartObjects.test.ts @@ -0,0 +1,60 @@ +import {suite} from '@alinea/suite' +import {hashObject} from 'alinea/core/source/GitUtils' +import { + hashCommitObject, + serializeCommitObject, + type GitSignature +} from './GitSmartObjects.js' + +const test = suite(import.meta) +const decoder = new TextDecoder() + +const author: GitSignature = {name: 'Ben', email: 'ben@example.com'} +const committer: GitSignature = {name: 'CI', email: 'ci@example.com'} +const date = new Date('2024-01-01T00:00:00.000Z') + +test('serializeCommitObject renders author and committer headers', () => { + const bytes = serializeCommitObject({ + tree: 'a'.repeat(40), + parent: 'b'.repeat(40), + message: 'Hello', + author, + committer, + date + }) + const text = decoder.decode(bytes) + test.ok(text.includes(`tree ${'a'.repeat(40)}`)) + test.ok(text.includes(`parent ${'b'.repeat(40)}`)) + test.ok(text.includes('author Ben 1704067200 +0000')) + test.ok(text.includes('committer CI 1704067200 +0000')) + test.ok(text.endsWith('\nHello') || text.endsWith('Hello')) +}) + +test('hashCommitObject matches generic git object hash', async () => { + const commit = { + tree: 'c'.repeat(40), + parent: 'd'.repeat(40), + message: 'Ship it', + author, + committer, + date + } + const serialized = serializeCommitObject(commit) + const specific = await hashCommitObject(commit) + const generic = await hashObject('commit', serialized) + test.is(specific, generic) + test.is(specific.length, 40) +}) + +test('serializeCommitObject uses author as default committer', () => { + const bytes = serializeCommitObject({ + tree: 'e'.repeat(40), + parent: 'f'.repeat(40), + message: 'Default committer', + author, + date + }) + const text = decoder.decode(bytes) + test.ok(text.includes('author Ben 1704067200 +0000')) + test.ok(text.includes('committer Ben 1704067200 +0000')) +}) diff --git a/src/backend/api/gitSmart/GitSmartObjects.ts b/src/backend/api/gitSmart/GitSmartObjects.ts new file mode 100644 index 000000000..5a1bc6abd --- /dev/null +++ b/src/backend/api/gitSmart/GitSmartObjects.ts @@ -0,0 +1,44 @@ +import {hashObject} from 'alinea/core/source/GitUtils' + +const encoder = new TextEncoder() + +export interface GitSignature { + name: string + email: string +} + +export interface CommitObjectInput { + tree: string + parent: string + message: string + author: GitSignature + committer?: GitSignature + date?: Date +} + +export function serializeCommitObject(input: CommitObjectInput): Uint8Array { + const date = input.date ?? new Date() + const signature = formatSignature(input.author, date) + const committer = formatSignature( + input.committer ?? input.author, + date + ) + const body = [ + `tree ${input.tree}`, + `parent ${input.parent}`, + `author ${signature}`, + `committer ${committer}`, + '', + input.message + ].join('\n') + return encoder.encode(body) +} + +export function hashCommitObject(input: CommitObjectInput): Promise { + return hashObject('commit', serializeCommitObject(input)) +} + +function formatSignature(signature: GitSignature, date: Date) { + const seconds = Math.floor(date.getTime() / 1000) + return `${signature.name} <${signature.email}> ${seconds} +0000` +} diff --git a/src/backend/api/gitSmart/GitSmartPack.test.ts b/src/backend/api/gitSmart/GitSmartPack.test.ts new file mode 100644 index 000000000..fde3d99bb --- /dev/null +++ b/src/backend/api/gitSmart/GitSmartPack.test.ts @@ -0,0 +1,95 @@ +import {Blob, CompressionStream, Response} from '@alinea/iso' +import {suite} from '@alinea/suite' +import {createGitDelta} from 'alinea/core/source/GitDelta' +import {hashBlob} from 'alinea/core/source/GitUtils' +import {sha1Bytes} from 'alinea/core/source/Utils' +import {parsePack, createPack} from './GitSmartPack.js' + +const test = suite(import.meta) +const encoder = new TextEncoder() +const decoder = new TextDecoder() + +test('createPack and parsePack roundtrip full objects', async () => { + const objects = [ + {type: 'blob' as const, data: encoder.encode('hello')}, + {type: 'tree' as const, data: encoder.encode('040000 dir\0')}, + {type: 'commit' as const, data: encoder.encode('tree abc\n\nmsg')} + ] + const pack = await createPack(objects) + const parsed = await parsePack(pack) + test.equal( + parsed.map(object => object.type), + ['blob', 'tree', 'commit'] + ) + test.equal( + parsed.map(object => decoder.decode(object.data)), + ['hello', '040000 dir\0', 'tree abc\n\nmsg'] + ) +}) + +test('parsePack decodes ref-delta blobs', async () => { + const base = encoder.encode('abc123\nline2\nline3\n') + const target = encoder.encode('abc123\nline2 updated\nline3\nline4\n') + const pack = await createRefDeltaPack(base, target) + const parsed = await parsePack(pack) + test.is(parsed.length, 2) + test.is(parsed[0].type, 'blob') + test.is(parsed[1].type, 'blob') + test.is(decoder.decode(parsed[1].data), decoder.decode(target)) + test.is(parsed[1].sha, await hashBlob(target)) +}) + +async function createRefDeltaPack(base: Uint8Array, target: Uint8Array) { + const {hexToBytes} = await import('alinea/core/source/Utils') + const baseSha = await hashBlob(base) + const delta = createGitDelta(base, target) + const header = new Uint8Array(12) + header.set(encoder.encode('PACK'), 0) + new DataView(header.buffer).setUint32(4, 2, false) + new DataView(header.buffer).setUint32(8, 2, false) + const baseEntry = concat([ + encodeHeader(3, base.length), + await compressed(base) + ]) + const deltaEntry = concat([ + encodeHeader(7, delta.length), + hexToBytes(baseSha), + await compressed(delta) + ]) + const payload = concat([header, baseEntry, deltaEntry]) + return concat([payload, await sha1Bytes(payload)]) +} + +function encodeHeader(type: number, size: number) { + const bytes = Array() + let n = size >>> 4 + let first = ((type & 0x07) << 4) | (size & 0x0f) + if (n !== 0) first |= 0x80 + bytes.push(first) + while (n !== 0) { + let next = n & 0x7f + n >>>= 7 + if (n !== 0) next |= 0x80 + bytes.push(next) + } + return Uint8Array.from(bytes) +} + +async function compressed(data: Uint8Array) { + const blob = new Blob([data]) + const response = new Response( + blob.stream().pipeThrough(new CompressionStream('deflate')) + ) + return new Uint8Array(await response.arrayBuffer()) +} + +function concat(parts: Array) { + const length = parts.reduce((sum, part) => sum + part.length, 0) + const out = new Uint8Array(length) + let offset = 0 + for (const part of parts) { + out.set(part, offset) + offset += part.length + } + return out +} diff --git a/src/backend/api/gitSmart/GitSmartPack.ts b/src/backend/api/gitSmart/GitSmartPack.ts new file mode 100644 index 000000000..9ebe4808e --- /dev/null +++ b/src/backend/api/gitSmart/GitSmartPack.ts @@ -0,0 +1,191 @@ +import {Blob, CompressionStream, Response} from '@alinea/iso' +import {applyGitDelta} from 'alinea/core/source/GitDelta' +import {hashObject} from 'alinea/core/source/GitUtils' +import { + bytesToHex, + concatUint8Arrays, + hexToBytes +} from 'alinea/core/source/Utils' +import {assert} from 'alinea/core/util/Assert' + +export type GitObjectType = 'commit' | 'tree' | 'blob' + +export interface PackObject { + type: GitObjectType + data: Uint8Array +} + +interface ParsedObject extends PackObject { + sha: string + offset: number +} + +export async function createPack(objects: Array): Promise { + const header = new Uint8Array(12) + header.set(new TextEncoder().encode('PACK')) + writeUint32BE(header, 4, 2) + writeUint32BE(header, 8, objects.length) + const entries = await Promise.all( + objects.map(async object => { + const type = packTypeCode(object.type) + return concatUint8Arrays([ + encodePackObjectHeader(type, object.data.length), + await deflate(object.data) + ]) + }) + ) + const payload = concatUint8Arrays([header, ...entries]) + const checksum = hexToBytes(await hashRaw(payload)) + return concatUint8Arrays([payload, checksum]) +} + +export async function parsePack(pack: Uint8Array): Promise> { + assert(pack.length >= 32, 'Invalid pack') + assert(new TextDecoder().decode(pack.subarray(0, 4)) === 'PACK', 'Invalid pack') + const version = readUint32BE(pack, 4) + assert(version === 2 || version === 3, `Unsupported pack version ${version}`) + const count = readUint32BE(pack, 8) + let pos = 12 + const byOffset = new Map() + const bySha = new Map() + const parsed = Array() + for (let i = 0; i < count; i++) { + const offset = pos + const header = parsePackObjectHeader(pack, pos) + pos = header.pos + let baseOffset: number | undefined + let baseSha: string | undefined + if (header.type === 6) { + const parsedOffset = parseOfsDeltaOffset(pack, pos) + pos = parsedOffset.pos + baseOffset = offset - parsedOffset.distance + } else if (header.type === 7) { + baseSha = bytesToHex(pack.subarray(pos, pos + 20)) + pos += 20 + } + const inflated = await inflateMember(pack, pos) + pos += inflated.bytesConsumed + let object: ParsedObject + if (header.type === 6 || header.type === 7) { + const base = + baseOffset !== undefined ? byOffset.get(baseOffset) : bySha.get(baseSha!) + assert(base, `Missing delta base for pack object at ${offset}`) + const data = applyGitDelta(base.data, inflated.data) + const sha = await hashObject(base.type, data) + object = {offset, type: base.type, data, sha} + } else { + const type = parseObjectType(header.type) + const sha = await hashObject(type, inflated.data) + object = {offset, type, data: inflated.data, sha} + } + parsed.push(object) + byOffset.set(offset, object) + bySha.set(object.sha, object) + } + return parsed +} + +function parseObjectType(type: number): GitObjectType { + switch (type) { + case 1: + return 'commit' + case 2: + return 'tree' + case 3: + return 'blob' + default: + throw new Error(`Unsupported pack object type ${type}`) + } +} + +function packTypeCode(type: GitObjectType): number { + switch (type) { + case 'commit': + return 1 + case 'tree': + return 2 + case 'blob': + return 3 + } +} + +function encodePackObjectHeader(type: number, size: number): Uint8Array { + const bytesOut = Array() + let n = size >>> 4 + let first = ((type & 0x07) << 4) | (size & 0x0f) + if (n !== 0) first |= 0x80 + bytesOut.push(first) + while (n !== 0) { + let next = n & 0x7f + n >>>= 7 + if (n !== 0) next |= 0x80 + bytesOut.push(next) + } + return Uint8Array.from(bytesOut) +} + +function parsePackObjectHeader(pack: Uint8Array, offset: number) { + let pos = offset + let byte = pack[pos++] + const type = (byte >> 4) & 0x07 + let size = byte & 0x0f + let shift = 4 + while (byte & 0x80) { + byte = pack[pos++] + size |= (byte & 0x7f) << shift + shift += 7 + } + return {type, size, pos} +} + +function parseOfsDeltaOffset(pack: Uint8Array, offset: number) { + let pos = offset + let byte = pack[pos++] + let distance = byte & 0x7f + while (byte & 0x80) { + byte = pack[pos++] + distance = ((distance + 1) << 7) | (byte & 0x7f) + } + return {distance, pos} +} + +async function inflateMember(pack: Uint8Array, offset: number) { + const {createInflate} = await import('node:zlib') + const inflater = createInflate() + const chunks = Array() + inflater.on('data', chunk => chunks.push(new Uint8Array(chunk))) + const source = Buffer.from(pack.subarray(offset)) + const end = new Promise((resolve, reject) => { + inflater.on('end', resolve) + inflater.on('error', reject) + }) + inflater.end(source) + await end + return { + data: concatUint8Arrays(chunks), + bytesConsumed: inflater.bytesWritten + } +} + +async function deflate(data: Uint8Array): Promise { + const blob = new Blob([data]) + const compressed = new Response( + blob.stream().pipeThrough(new CompressionStream('deflate')) + ) + return new Uint8Array(await compressed.arrayBuffer()) +} + +async function hashRaw(data: Uint8Array): Promise { + const {sha1Hash} = await import('alinea/core/source/Utils') + return sha1Hash(data) +} + +function writeUint32BE(target: Uint8Array, offset: number, value: number) { + const view = new DataView(target.buffer, target.byteOffset, target.byteLength) + view.setUint32(offset, value, false) +} + +function readUint32BE(target: Uint8Array, offset: number) { + const view = new DataView(target.buffer, target.byteOffset, target.byteLength) + return view.getUint32(offset, false) +} diff --git a/src/backend/api/gitSmart/GitSmartProtocol.test.ts b/src/backend/api/gitSmart/GitSmartProtocol.test.ts new file mode 100644 index 000000000..0af3200c9 --- /dev/null +++ b/src/backend/api/gitSmart/GitSmartProtocol.test.ts @@ -0,0 +1,102 @@ +import {suite} from '@alinea/suite' +import { + buildReceivePackRequest, + buildUploadPackRequest, + flushPkt, + parseAdvertisement, + parseReceivePackStatus, + pktLine +} from './GitSmartProtocol.js' +import { + hashCommitObject, + type GitSignature, + serializeCommitObject +} from './GitSmartObjects.js' + +const test = suite(import.meta) +const encoder = new TextEncoder() +const decoder = new TextDecoder() +const author: GitSignature = {name: 'Ben', email: 'ben@example.com'} + +test('pkt-line advertisement parsing preserves refs and capabilities', () => { + const data = concat([ + pktLine('# service=git-upload-pack\n'), + flushPkt(), + pktLine( + 'abc refs/heads/main\0side-band-64k ofs-delta allow-reachable-sha1-in-want\n' + ), + pktLine('def refs/heads/dev\n') + ]) + const advertisement = parseAdvertisement(data) + test.is(advertisement.refs.get('refs/heads/main'), 'abc') + test.is(advertisement.refs.get('refs/heads/dev'), 'def') + test.ok(advertisement.capabilities.has('side-band-64k')) + test.ok(advertisement.capabilities.has('allow-reachable-sha1-in-want')) +}) + +test('upload-pack and receive-pack requests encode wants and ref updates', () => { + const upload = decoder.decode( + buildUploadPackRequest(['a'.repeat(40), 'b'.repeat(40)], new Set(['ofs-delta'])) + ) + test.ok(upload.includes(`want ${'a'.repeat(40)} ofs-delta no-progress`)) + test.ok(upload.includes(`want ${'b'.repeat(40)}`)) + const receive = decoder.decode( + buildReceivePackRequest({ + oldSha: 'a'.repeat(40), + newSha: 'b'.repeat(40), + ref: 'refs/heads/main', + capabilities: new Set(['report-status', 'ofs-delta']), + pack: encoder.encode('PACK') + }).subarray(0, 200) + ) + test.ok(receive.includes(`refs/heads/main\0report-status ofs-delta`)) +}) + +test('receive-pack status parsing captures unpack and ref errors', () => { + const payload = pktLine( + withChannel(1, encoder.encode('unpack ok\nng refs/heads/main non-fast-forward\n')) + ) + const status = parseReceivePackStatus(concat([payload, flushPkt()])) + test.is(status.unpackStatus, 'ok') + test.is(status.refStatus.get('refs/heads/main'), 'non-fast-forward') +}) + +test('commit object serialization hashes deterministically', async () => { + const data = serializeCommitObject({ + tree: 'a'.repeat(40), + parent: 'b'.repeat(40), + message: 'Hello', + author, + date: new Date('2024-01-01T00:00:00.000Z') + }) + const text = decoder.decode(data) + test.ok(text.includes(`tree ${'a'.repeat(40)}`)) + test.ok(text.includes(`parent ${'b'.repeat(40)}`)) + test.ok(text.includes('author Ben 1704067200 +0000')) + const hash = await hashCommitObject({ + tree: 'a'.repeat(40), + parent: 'b'.repeat(40), + message: 'Hello', + author, + date: new Date('2024-01-01T00:00:00.000Z') + }) + test.is(hash.length, 40) +}) + +function concat(parts: Array) { + const length = parts.reduce((sum, part) => sum + part.length, 0) + const out = new Uint8Array(length) + let offset = 0 + for (const part of parts) { + out.set(part, offset) + offset += part.length + } + return out +} + +function withChannel(channel: number, payload: Uint8Array) { + const out = new Uint8Array(payload.length + 1) + out[0] = channel + out.set(payload, 1) + return out +} diff --git a/src/backend/api/gitSmart/GitSmartProtocol.ts b/src/backend/api/gitSmart/GitSmartProtocol.ts new file mode 100644 index 000000000..b456d08d7 --- /dev/null +++ b/src/backend/api/gitSmart/GitSmartProtocol.ts @@ -0,0 +1,172 @@ +import {btoa} from 'alinea/core/util/Encoding' +import {concatUint8Arrays, hexToBytes} from 'alinea/core/source/Utils' + +const encoder = new TextEncoder() +const decoder = new TextDecoder() + +export interface GitRefAdvertisement { + capabilities: Set + refs: Map +} + +export interface ReceivePackStatus { + unpackStatus?: string + refStatus: Map + messages: Array +} + +export function gitBasicAuth(token: string) { + return `Basic ${btoa(`x-access-token:${token}`)}` +} + +export function pktLine(data: string | Uint8Array): Uint8Array { + const bytes = typeof data === 'string' ? encoder.encode(data) : data + const length = (bytes.length + 4).toString(16).padStart(4, '0') + return concatUint8Arrays([encoder.encode(length), bytes]) +} + +export function flushPkt(): Uint8Array { + return encoder.encode('0000') +} + +export function parseAdvertisement(data: Uint8Array): GitRefAdvertisement { + const packets = parsePktLines(data) + const refs = new Map() + const capabilities = new Set() + let firstRef = true + for (const packet of packets) { + if (packet === null) continue + const text = decoder.decode(packet) + if (text.startsWith('# service=')) continue + const line = text.replace(/\n$/, '') + if (!line) continue + if (firstRef) { + firstRef = false + const nulIndex = line.indexOf('\0') + const refPart = nulIndex === -1 ? line : line.slice(0, nulIndex) + const capabilityPart = nulIndex === -1 ? '' : line.slice(nulIndex + 1) + const [sha, ref] = refPart.split(' ') + if (sha && ref) refs.set(ref, sha) + for (const capability of capabilityPart.split(' ')) { + if (capability) capabilities.add(capability) + } + continue + } + const [sha, ref] = line.split(' ') + if (sha && ref) refs.set(ref, sha) + } + return {capabilities, refs} +} + +export function buildUploadPackRequest( + wants: Array, + capabilities: Set +): Uint8Array { + const requestedCaps = [ + capabilities.has('side-band-64k') ? 'side-band-64k' : undefined, + capabilities.has('ofs-delta') ? 'ofs-delta' : undefined, + 'no-progress' + ].filter(Boolean) + const packets: Array = [] + wants.forEach((sha, index) => { + const line = + index === 0 && requestedCaps.length + ? `want ${sha} ${requestedCaps.join(' ')}\n` + : `want ${sha}\n` + packets.push(pktLine(line)) + }) + packets.push(flushPkt(), pktLine('done\n')) + return concatUint8Arrays(packets) +} + +export function buildReceivePackRequest(input: { + oldSha: string + newSha: string + ref: string + capabilities: Set + pack: Uint8Array +}): Uint8Array { + const requestedCaps = [ + input.capabilities.has('report-status') ? 'report-status' : undefined, + input.capabilities.has('side-band-64k') ? 'side-band-64k' : undefined, + input.capabilities.has('ofs-delta') ? 'ofs-delta' : undefined + ].filter(Boolean) + const command = `${input.oldSha} ${input.newSha} ${input.ref}\0${requestedCaps.join(' ')}\n` + return concatUint8Arrays([pktLine(command), flushPkt(), input.pack]) +} + +export function extractSidebandData(data: Uint8Array): { + channel1: Uint8Array + messages: Array +} { + const packets = parsePktLines(data) + const channel1 = Array() + const messages = Array() + for (const packet of packets) { + if (!packet || packet.length === 0) continue + const channel = packet[0] + const payload = packet.subarray(1) + if (channel === 1) channel1.push(payload) + else messages.push(decoder.decode(payload).replace(/\n$/, '')) + } + return { + channel1: concatUint8Arrays(channel1), + messages + } +} + +export function parseReceivePackStatus(data: Uint8Array): ReceivePackStatus { + const {channel1, messages} = extractSidebandData(data) + const text = decoder.decode(channel1) + const refStatus = new Map() + let unpackStatus: string | undefined + for (const rawLine of text.split('\n')) { + const line = rawLine.trim() + if (!line) continue + if (line.startsWith('unpack ')) { + unpackStatus = line.slice('unpack '.length) + continue + } + if (line.startsWith('ok ')) { + refStatus.set(line.slice(3), 'ok') + continue + } + if (line.startsWith('ng ')) { + const [ref, ...reason] = line.slice(3).split(' ') + refStatus.set(ref, reason.join(' ')) + } + } + return {unpackStatus, refStatus, messages} +} + +export function findPackStart(data: Uint8Array): number { + const marker = encoder.encode('PACK') + outer: for (let i = 0; i <= data.length - marker.length; i++) { + for (let j = 0; j < marker.length; j++) { + if (data[i + j] !== marker[j]) continue outer + } + return i + } + return -1 +} + +export function encodeRefDeltaBase(baseSha: string): Uint8Array { + return hexToBytes(baseSha) +} + +function parsePktLines(data: Uint8Array): Array { + const packets = Array() + let pos = 0 + while (pos + 4 <= data.length) { + const length = Number.parseInt(decoder.decode(data.subarray(pos, pos + 4)), 16) + pos += 4 + if (length === 0) { + packets.push(null) + continue + } + const payloadLength = length - 4 + packets.push(data.subarray(pos, pos + payloadLength)) + pos += payloadLength + } + return packets +}