Two-phase backlog scan with self-healing found-count tracking
Split each channel's backlog scan into a find phase (settles the found count first) and a react phase, so the progress display no longer grows both numbers in lockstep. Found messages are logged in a dedicated, message_id-deduped table and recovered across restarts independent of the resume checkpoint, so nothing found is silently abandoned if the script stops mid-react. Also drops the separately-maintained messages_found counter, which proved prone to drift under repeated interruptions/rescans, in favor of computing it live from the dedup table - self-healing regardless of how many times a channel gets rescanned.
This commit is contained in:
@@ -1,11 +1,12 @@
|
||||
import asyncio
|
||||
import datetime
|
||||
import logging
|
||||
from typing import Awaitable, Callable
|
||||
from typing import Awaitable, Callable, Optional
|
||||
|
||||
import discord
|
||||
|
||||
from bot.discovery import Target
|
||||
from bot.display import StatsDisplay
|
||||
from bot.storage import Storage
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
@@ -18,45 +19,64 @@ async def scan_channel_backlog(
|
||||
channel: discord.abc.Messageable,
|
||||
target: Target,
|
||||
friend_id: int,
|
||||
emoji: str,
|
||||
cutoff: datetime.datetime,
|
||||
react_fn: ReactFn,
|
||||
display: Optional[StatsDisplay] = None,
|
||||
) -> None:
|
||||
"""Scan a channel's history once, resumably.
|
||||
"""Scan a channel's history once, resumably, in two phases.
|
||||
|
||||
First run: walk forward from `cutoff` (now - days_back) to the present.
|
||||
Subsequent runs: only diff forward from the newest message already seen,
|
||||
since the days_back window only ever moves forward in time.
|
||||
Phase 1 finds every message from the friend in the window first (so the
|
||||
"found" count settles before any reacting starts, rather than climbing
|
||||
in lockstep with "reacted"), permanently logging each one (deduped by
|
||||
message ID) so re-scanning it later never inflates the count. Phase 2
|
||||
then reacts to each one in order.
|
||||
|
||||
Before Phase 1 even starts, any message previously logged as found for
|
||||
this channel but not yet reacted to (e.g. the script was stopped midway
|
||||
through Phase 2 last time) is recovered from that same log and queued
|
||||
up again — this is independent of the newest_id_seen checkpoint below,
|
||||
so nothing found is ever silently abandoned even if the checkpoint has
|
||||
already moved past it.
|
||||
"""
|
||||
channel_id = getattr(channel, "id", None)
|
||||
if channel_id is None:
|
||||
return
|
||||
|
||||
found_messages = []
|
||||
pending_ids = await asyncio.to_thread(storage.get_pending_found_message_ids, channel_id, emoji)
|
||||
for message_id in pending_ids:
|
||||
try:
|
||||
found_messages.append(await channel.fetch_message(int(message_id)))
|
||||
except discord.NotFound:
|
||||
pass # message (or its channel access) is gone since it was found
|
||||
except discord.HTTPException:
|
||||
log.exception("Failed to refetch pending message %s in channel %s", message_id, channel_id)
|
||||
|
||||
state = await asyncio.to_thread(storage.get_scan_state, channel_id)
|
||||
newest_id_seen = int(state["newest_id_seen"]) if state and state.get("newest_id_seen") else None
|
||||
|
||||
after = discord.Object(id=newest_id_seen) if newest_id_seen else cutoff
|
||||
|
||||
checkpoint_every = 25
|
||||
seen_since_checkpoint = 0
|
||||
|
||||
try:
|
||||
async for message in channel.history(after=after, oldest_first=True, limit=None):
|
||||
if newest_id_seen is None or message.id > newest_id_seen:
|
||||
newest_id_seen = message.id
|
||||
|
||||
await react_fn(message, target)
|
||||
|
||||
seen_since_checkpoint += 1
|
||||
if seen_since_checkpoint >= checkpoint_every:
|
||||
seen_since_checkpoint = 0
|
||||
if message.author.id == friend_id:
|
||||
found_messages.append(message)
|
||||
await asyncio.to_thread(
|
||||
storage.set_scan_state, channel_id, target.target_id, newest_id_seen, False
|
||||
storage.record_message_found, message.id, channel_id, target.target_id, friend_id
|
||||
)
|
||||
if display is not None:
|
||||
await display.refresh()
|
||||
|
||||
await asyncio.to_thread(
|
||||
storage.set_scan_state, channel_id, target.target_id, newest_id_seen, True
|
||||
)
|
||||
await asyncio.to_thread(storage.set_scan_state, channel_id, target.target_id, newest_id_seen, True)
|
||||
except discord.Forbidden:
|
||||
log.warning("No access to channel %s (%s), skipping", channel_id, getattr(channel, "name", ""))
|
||||
return
|
||||
except discord.HTTPException:
|
||||
log.exception("HTTP error scanning channel %s", channel_id)
|
||||
return
|
||||
|
||||
for message in found_messages:
|
||||
await react_fn(message, target)
|
||||
|
||||
Reference in New Issue
Block a user