Two-phase backlog scan with self-healing found-count tracking

Split each channel's backlog scan into a find phase (settles the
found count first) and a react phase, so the progress display no
longer grows both numbers in lockstep. Found messages are logged in
a dedicated, message_id-deduped table and recovered across restarts
independent of the resume checkpoint, so nothing found is silently
abandoned if the script stops mid-react.

Also drops the separately-maintained messages_found counter, which
proved prone to drift under repeated interruptions/rescans, in favor
of computing it live from the dedup table - self-healing regardless
of how many times a channel gets rescanned.
This commit is contained in:
2026-08-01 18:46:28 +02:00
parent 41b6b3f234
commit ab0afa149a
6 changed files with 136 additions and 32 deletions

View File

@@ -1,11 +1,12 @@
import asyncio
import datetime
import logging
from typing import Awaitable, Callable
from typing import Awaitable, Callable, Optional
import discord
from bot.discovery import Target
from bot.display import StatsDisplay
from bot.storage import Storage
log = logging.getLogger(__name__)
@@ -18,45 +19,64 @@ async def scan_channel_backlog(
channel: discord.abc.Messageable,
target: Target,
friend_id: int,
emoji: str,
cutoff: datetime.datetime,
react_fn: ReactFn,
display: Optional[StatsDisplay] = None,
) -> None:
"""Scan a channel's history once, resumably.
"""Scan a channel's history once, resumably, in two phases.
First run: walk forward from `cutoff` (now - days_back) to the present.
Subsequent runs: only diff forward from the newest message already seen,
since the days_back window only ever moves forward in time.
Phase 1 finds every message from the friend in the window first (so the
"found" count settles before any reacting starts, rather than climbing
in lockstep with "reacted"), permanently logging each one (deduped by
message ID) so re-scanning it later never inflates the count. Phase 2
then reacts to each one in order.
Before Phase 1 even starts, any message previously logged as found for
this channel but not yet reacted to (e.g. the script was stopped midway
through Phase 2 last time) is recovered from that same log and queued
up again — this is independent of the newest_id_seen checkpoint below,
so nothing found is ever silently abandoned even if the checkpoint has
already moved past it.
"""
channel_id = getattr(channel, "id", None)
if channel_id is None:
return
found_messages = []
pending_ids = await asyncio.to_thread(storage.get_pending_found_message_ids, channel_id, emoji)
for message_id in pending_ids:
try:
found_messages.append(await channel.fetch_message(int(message_id)))
except discord.NotFound:
pass # message (or its channel access) is gone since it was found
except discord.HTTPException:
log.exception("Failed to refetch pending message %s in channel %s", message_id, channel_id)
state = await asyncio.to_thread(storage.get_scan_state, channel_id)
newest_id_seen = int(state["newest_id_seen"]) if state and state.get("newest_id_seen") else None
after = discord.Object(id=newest_id_seen) if newest_id_seen else cutoff
checkpoint_every = 25
seen_since_checkpoint = 0
try:
async for message in channel.history(after=after, oldest_first=True, limit=None):
if newest_id_seen is None or message.id > newest_id_seen:
newest_id_seen = message.id
await react_fn(message, target)
seen_since_checkpoint += 1
if seen_since_checkpoint >= checkpoint_every:
seen_since_checkpoint = 0
if message.author.id == friend_id:
found_messages.append(message)
await asyncio.to_thread(
storage.set_scan_state, channel_id, target.target_id, newest_id_seen, False
storage.record_message_found, message.id, channel_id, target.target_id, friend_id
)
if display is not None:
await display.refresh()
await asyncio.to_thread(
storage.set_scan_state, channel_id, target.target_id, newest_id_seen, True
)
await asyncio.to_thread(storage.set_scan_state, channel_id, target.target_id, newest_id_seen, True)
except discord.Forbidden:
log.warning("No access to channel %s (%s), skipping", channel_id, getattr(channel, "name", ""))
return
except discord.HTTPException:
log.exception("HTTP error scanning channel %s", channel_id)
return
for message in found_messages:
await react_fn(message, target)