Index chain data

import { appendFile, readFile, writeFile } from "node:fs/promises";
import { createPublicClient, erc20Abi, formatUnits, http, parseEventLogs, type Hex } from "viem";
import { botChainTestnet } from "@uzolabs/sdk/chains";
import { getAddresses } from "@uzolabs/sdk/contracts";

const BOTSCAN = botChainTestnet.blockExplorers.default.url;
const client = createPublicClient({ chain: botChainTestnet, transport: http() });
const { usdt } = getAddresses(botChainTestnet.id);
const TRANSFER = "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef";

const START_BLOCK = 25_399_000n; // Where to start the first time. Use your contract's deployment block.
const WINDOW = 5_000n; // Blocks per request. Keep it small enough to stay under 1,000 logs.
const CONFIRMATIONS = 12n; // Stay this many blocks behind the head. An example value; choose your own.
const STATE_FILE = "state.json";
const OUT_FILE = "transfers.jsonl";

type RawLog = { address: Hex; topics: (Hex | null)[]; data: Hex; blockNumber: Hex; transactionHash: Hex; transactionIndex: Hex; logIndex: Hex };

async function loadNextBlock(): Promise<bigint> {
  try {
    return BigInt(JSON.parse(await readFile(STATE_FILE, "utf8")).nextBlock);
  } catch {
    return START_BLOCK;
  }
}

async function fetchLogs(fromBlock: bigint, toBlock: bigint): Promise<RawLog[]> {
  const url = `${BOTSCAN}/api?module=logs&action=getLogs&address=${usdt}&topic0=${TRANSFER}&fromBlock=${fromBlock}&toBlock=${toBlock}`;
  const response = await fetch(url);
  if (!response.ok) throw new Error(`BOTScan answered ${response.status}`);
  const body = (await response.json()) as { status: string; message: string; result: RawLog[] | null };
  if (body.result === null) throw new Error(body.message);
  return body.result; // An empty array means no logs in this range.
}

// One pass: index every confirmed block since the last run, one window at a time.
const head = (await client.getBlockNumber()) - CONFIRMATIONS;
let from = await loadNextBlock();
while (from <= head) {
  const to = from + WINDOW - 1n < head ? from + WINDOW - 1n : head;
  const raw = await fetchLogs(from, to);
  if (raw.length >= 1000) throw new Error(`Window ${from}-${to} hit the 1,000 log cap. Lower WINDOW.`);

  const logs = parseEventLogs({
    abi: erc20Abi,
    eventName: "Transfer",
    logs: raw.map((log) => ({ ...log, topics: log.topics.filter((t) => t !== null) as [Hex, ...Hex[]], blockHash: null, removed: false })),
  });
  const lines = logs.map((log) =>
    JSON.stringify({ block: Number(log.blockNumber), tx: log.transactionHash, from: log.args.from, to: log.args.to, value: formatUnits(log.args.value, 6) }),
  );
  if (lines.length > 0) await appendFile(OUT_FILE, lines.join("\n") + "\n");

  // Save progress only after the data is written, so a crash repeats a window instead of skipping one.
  await writeFile(STATE_FILE, JSON.stringify({ nextBlock: String(to + 1n) }));
  console.log(`Blocks ${from}-${to}: ${logs.length} transfers`);
  from = to + 1n;
}
console.log(`Up to date at block ${head}`);