commit d38bceff4a0824695b93b2bebc22a9e73c6b8fd6 from: lex0de date: Wed Sep 2 20:02:48 2026 UTC Fix Lobsters feed polling commit - 90ffadc3bd024e8f62fa22e9e322923b8217e391 commit + d38bceff4a0824695b93b2bebc22a9e73c6b8fd6 blob - 330f390fdae9defea22c45d0327ae4627ac011f4 blob + 316d1678ba5d5811bcadd191187443b5b8f12c5c --- crybot.pl +++ crybot.pl @@ -61,7 +61,7 @@ my %config = ( { key => 'lobsters', label => $ENV{CRYBOT_LOBSTERS_LABEL} // 'Lobsters', - url => $ENV{CRYBOT_LOBSTERS_FEED} // 'https://lobste.rs/rss', + url => $ENV{CRYBOT_LOBSTERS_FEED} // 'https://lobste.rs/newest.rss', }, ], ); @@ -99,7 +99,7 @@ my %COMMANDS = ( help => \&cmd_help, ); -main(); +main() unless caller; sub main { my $sock = connect_to_server(\%config); @@ -421,34 +421,22 @@ sub poll_feeds { my $key = $feed->{key} // $feed_url; my $last_guid = $state->{$key}{last_guid}; + my ( $cursor_status, $cursor_guid, $announce ) = + select_feed_updates( \@items, $last_guid ); - unless ( defined $last_guid ) { - my $initial = $items[0]; - my $guid = $initial->{guid} // $initial->{link}; - if ($guid) { - $state->{$key}{last_guid} = $guid; - $state_changed = 1; - } - next; + if ( defined $cursor_guid && + ( !defined $last_guid || $cursor_guid ne $last_guid ) ) { + $state->{$key}{last_guid} = $cursor_guid; + $state_changed = 1; } - my @announce; - for my $item (@items) { - my $guid = $item->{guid} // $item->{link}; - next unless $guid; - last if defined $last_guid && $guid eq $last_guid; - push @announce, $item; + if ( $cursor_status eq 'missing' ) { + debug_log( $config, + "[feed] saved cursor is absent from $feed_url; rebaselining" ); } - next unless @announce; - @announce = reverse @announce; - - for my $item (@announce) { + for my $item ( @{$announce} ) { my $guid = $item->{guid} // $item->{link}; - if ($guid) { - $state->{$key}{last_guid} = $guid; - $state_changed = 1; - } next unless $send_output; @@ -468,6 +456,32 @@ sub poll_feeds { if $state_changed; } +sub select_feed_updates { + my ( $items, $last_guid ) = @_; + return ( 'empty', undef, [] ) + unless ref $items eq 'ARRAY' && @{$items}; + + my $newest_guid = $items->[0]{guid} // $items->[0]{link}; + return ( 'empty', undef, [] ) unless defined $newest_guid; + return ( 'initial', $newest_guid, [] ) + unless defined $last_guid && length $last_guid; + + my @announce; + my $cursor_found = 0; + for my $item ( @{$items} ) { + my $guid = $item->{guid} // $item->{link}; + next unless defined $guid; + if ( $guid eq $last_guid ) { + $cursor_found = 1; + last; + } + push @announce, $item; + } + + return ( 'missing', $newest_guid, [] ) unless $cursor_found; + return ( 'current', $newest_guid, [ reverse @announce ] ); +} + sub parse_rss_items { my ($xml) = @_; return unless defined $xml; blob - /dev/null blob + 7eb77fbc249364e48a0ebd2e7a82bee57b5c136c (mode 644) --- /dev/null +++ t/feed_polling.t @@ -0,0 +1,91 @@ +# Crybot - Developed by lex0de (lex0de@tuta.com) +# cryobot/t/feed_polling.t + +use v5.24; +use strict; +use warnings; + +use FindBin qw($Bin); +use Test::More; + +my $script = "$Bin/../crybot.pl"; +my $loaded = do $script; +die "Unable to load $script: $@ $!" unless defined $loaded; + +sub feed_item { + my ( $guid, $title ) = @_; + return { + guid => $guid, + link => "https://example.invalid/$guid", + title => $title, + }; +} + +subtest 'empty feed does not change the cursor' => sub { + my ( $status, $cursor, $updates ) = select_feed_updates( [], 'story-1' ); + is( $status, 'empty', 'empty status returned' ); + ok( !defined $cursor, 'cursor remains undefined' ); + is_deeply( $updates, [], 'nothing announced' ); +}; + +subtest 'first poll establishes a silent baseline' => sub { + my @items = ( feed_item( 'story-2', 'Second' ), + feed_item( 'story-1', 'First' ) ); + my ( $status, $cursor, $updates ) = select_feed_updates( \@items, undef ); + is( $status, 'initial', 'initial status returned' ); + is( $cursor, 'story-2', 'newest item becomes cursor' ); + is_deeply( $updates, [], 'initial feed is not announced' ); +}; + +subtest 'unchanged feed produces no announcements' => sub { + my @items = ( feed_item( 'story-2', 'Second' ), + feed_item( 'story-1', 'First' ) ); + my ( $status, $cursor, $updates ) = + select_feed_updates( \@items, 'story-2' ); + is( $status, 'current', 'current status returned' ); + is( $cursor, 'story-2', 'cursor remains current' ); + is_deeply( $updates, [], 'nothing announced' ); +}; + +subtest 'new entries are announced oldest first' => sub { + my @items = ( + feed_item( 'story-4', 'Fourth' ), + feed_item( 'story-3', 'Third' ), + feed_item( 'story-2', 'Second' ), + feed_item( 'story-1', 'First' ), + ); + my ( $status, $cursor, $updates ) = + select_feed_updates( \@items, 'story-2' ); + is( $status, 'current', 'current status returned' ); + is( $cursor, 'story-4', 'cursor advances to newest item' ); + is_deeply( [ map { $_->{guid} } @{$updates} ], + [ 'story-3', 'story-4' ], 'announcements are chronological' ); +}; + +subtest 'missing cursor rebaselines without replay' => sub { + my @items = ( + feed_item( 'ranked-3', 'Third ranked story' ), + feed_item( 'ranked-1', 'First ranked story' ), + feed_item( 'ranked-2', 'Second ranked story' ), + ); + my ( $status, $cursor, $updates ) = + select_feed_updates( \@items, 'removed-story' ); + is( $status, 'missing', 'missing status returned' ); + is( $cursor, 'ranked-3', 'cursor advances to current head' ); + is_deeply( $updates, [], 'reordered feed is not replayed' ); +}; + +subtest 'link is accepted when guid is absent' => sub { + my @items = ( + { link => 'https://example.invalid/new', title => 'New' }, + { link => 'https://example.invalid/old', title => 'Old' }, + ); + my ( $status, $cursor, $updates ) = + select_feed_updates( \@items, 'https://example.invalid/old' ); + is( $status, 'current', 'current status returned' ); + is( $cursor, 'https://example.invalid/new', 'link becomes cursor' ); + is_deeply( [ map { $_->{link} } @{$updates} ], + ['https://example.invalid/new'], 'link-only item is announced' ); +}; + +done_testing;