Press n or j to go to the next uncovered block, b, p or k for the previous block.
| 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 | 2x 2x 2x 2x 2x 2x 2x 1x 2x 2x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x | /**
* Fetch every Zeroheight page as a `ZhPage`: MCP for visible content, REST
* API for tab order and hidden pages/tabs. Shared by `zh:audit` and `zh:pull`.
*/
import { stripSignedUrls } from './hash.js';
import { mapPool } from './http.js';
import { flattenListPages, normName, sortedTabs, splitMcpMarkdown, splitTabs } from './pages.js';
/** @typedef {(msg: string) => void} Log */
/**
* Merge MCP markdown (visible content) with the API page (tab order, hidden tabs).
* @param {import('./pages.js').ZhListedPage} listed
* @param {string} md
* @param {import('./api-client.js').ApiPage} api
* @param {Log} log
* @returns {import('./audit.js').ZhPage}
*/
export function visiblePage(listed, md, api, log) {
const { body } = splitMcpMarkdown(md);
const apiTabs = sortedTabs(api);
const sections = splitTabs(
body,
apiTabs.filter((t) => !t.hidden).map((t) => t.name),
);
const tabs = apiTabs.map((t) => {
const fromMcp = !t.hidden ? sections.get(normName(t.name)) : undefined;
if (!t.hidden && fromMcp === undefined)
log(` ! ${listed.name} > ${t.name}: tab not found in MCP output, using API content`);
return {
name: t.name.trim(),
uid: t.uid,
order: t.order,
hidden: t.hidden,
// Keep the signed URLs: the pull downloads them. `hashContent` strips them.
content: fromMcp ?? t.content ?? '',
};
});
return {
uid: listed.uid,
id: listed.id,
name: listed.name.trim(),
categoryPath: listed.categoryPath,
hidden: false,
source: 'mcp',
raw: md,
content: tabs.length ? '' : body,
tabs,
};
}
/**
* Hidden pages aren't served by the MCP, so content comes from the API.
* @param {import('./api-client.js').ApiPage} api
* @returns {import('./audit.js').ZhPage}
*/
export function hiddenPage(api) {
const tabs = sortedTabs(api).map((t) => ({
name: t.name.trim(),
uid: t.uid,
order: t.order,
hidden: true,
content: stripSignedUrls(t.content ?? ''),
}));
const content = stripSignedUrls(api.content ?? '');
return {
uid: api.uid,
id: api.id,
name: api.name.trim(),
categoryPath: [],
hidden: true,
source: 'api',
raw: JSON.stringify({ introduction: api.introduction ?? '', content, tabs }),
content: tabs.length ? '' : content,
tabs,
};
}
/**
* @param {{
* mcp: ReturnType<typeof import('./mcp-client.js').createMcpClient>,
* api: ReturnType<typeof import('./api-client.js').createApiClient>,
* only?: Set<string> | null,
* includeHidden?: boolean,
* log: Log,
* }} opts
* @returns {Promise<{ releaseId: number | null, zhPages: import('./audit.js').ZhPage[] }>}
*/
export async function fetchZhPages({ mcp, api, only = null, includeHidden = true, log }) {
log('Listing pages…');
const listed = flattenListPages(await mcp.listPages());
const releaseId = listed[0]?.releaseId ?? null;
const visibleIds = new Set(listed.map((p) => p.id));
const apiList = await api.listPages();
const visible = listed.filter((p) => !only || only.has(p.uid));
log(`Fetching ${apiList.length} pages from the API and ${visible.length} from the MCP…`);
let apiDone = 0;
let mcpDone = 0;
// The two sources are independent, so run both pools at once.
const [apiPages, mdList] = await Promise.all([
mapPool(apiList, 4, async (p) => {
const page = await api.getPage(p.id);
Iif (++apiDone % 25 === 0) log(` API ${apiDone}/${apiList.length}`);
return page;
}),
mapPool(visible, 4, async (p) => {
const md = await mcp.getPage(p.id);
Iif (++mcpDone % 25 === 0) log(` MCP ${mcpDone}/${visible.length}`);
return md;
}),
]);
const apiById = new Map(apiPages.map((p) => [p.id, p]));
/** @type {import('./audit.js').ZhPage[]} */
const zhPages = visible.map((p, i) => {
const apiPage = apiById.get(p.id);
Iif (!apiPage) throw new Error(`Page ${p.uid} (${p.name}) is in the MCP list but not the API`);
return visiblePage(p, mdList[i], apiPage, log);
});
Eif (includeHidden) {
for (const page of apiPages) {
Eif (visibleIds.has(page.id) || (only && !only.has(page.uid))) continue;
zhPages.push(hiddenPage(page));
}
}
return { releaseId, zhPages };
}
|