MDEV-34614 mysqlbinlog warn on EOF before GTID in --stop-position

This commit adds warnings for `--stop-position` GTIDs
that were not reached at EOF, mainly as a follow-up to
MDEV-27037 Mysqlbinlog should output a warning if EOF is found before its stop condition
(#3400).

`--stop-position` warnings inform possible mistakes in the input,
especially for progress reporting of scripts/wrappers.
MDEV-34614 enhances MDEV-27037 with individualized GTID validation, for
GTID range selection weren’t in all versions that MDEV-27037 targeted.

The `Gtid_event_filter` family provides the the warning
mechanism polymorphically and through the new public method
`verify_completed_state`. It’s uncommon to commit a new public
API targeting a non-latest version, but this design is not only
hierarchical but also extensible (e.g., to `--ignore-server-ids`).

client/mysqlbinlog.cc

@@ -3489,7 +3489,7 @@ static Exit_status dump_local_log_entries(PRINT_EVENT_INFO *print_event_info,
 
       /*
         Emit a warning in the event that we finished processing input
-        before reaching the boundary indicated by --stop-position.
+        before reaching the boundary indicated by --stop-position index.
       */
       if (((longlong)stop_position != stop_position_default) &&
           stop_position > my_b_tell(file))
@@ -3511,6 +3511,23 @@ static Exit_status dump_local_log_entries(PRINT_EVENT_INFO *print_event_info,
                   "before end of input", stop_datetime_str);
       }
 
+      /*
+        Emit warning(s) in the event that we finished processing input
+        before reaching all boundaries indicated by --stop-position GTID(s).
+        When implemented, replace `position_gtid_filter` in the condition with
+        `gtid_event_filter`, and it will also warn if we finished
+        processing before reaching all --start-position GTID(s)
+        and touching all --ignore-domain/server-ids.
+      */
+      // Gtid_event_filter::has_finished() isn't necessarily a O(1) happy path.
+      if (position_gtid_filter && position_gtid_filter->verify_completed_state())
+      {
+        // The condition covers the warnings but not this step -
+        if (result_file)
+          fflush(result_file);
+        retval = OK_STOP;
+      }
+
       goto end;
     }
     last_ev_when= ev->when;

mysql-test/suite/binlog/r/binlog_mysqlbinlog_warn_stop_gtid.result

@@ -0,0 +1,33 @@
+
+# MDEV-34614 mysqlbinlog warn on EOF before GTID in --stop-position
+
+RESET MASTER;
+SET @@session.gtid_domain_id= 0;
+SET @@session.server_id= 1;
+SET @@session.gtid_seq_no= 1;
+CREATE TABLE t1(a int);
+INSERT INTO t1 VALUES (1), (2);
+SET @@session.gtid_domain_id= 1;
+CREATE TABLE t2(a int);
+INSERT INTO t2 VALUES (3);
+FLUSH LOGS;
+Case: Default, must not see warning.
+# MYSQL_BINLOG --short-form MYSQLD_DATADIR/master-bin.000001 --result-file=ignored_output_file
+Case: Stop GTID at EOF, must not see warning.
+# MYSQL_BINLOG --short-form --skip-gtid-strict-mode --stop-position=0-1-2 MYSQLD_DATADIR/master-bin.000001 --result-file=ignored_output_file
+Case: Stop GTID after EOF, must see warning.
+# MYSQL_BINLOG --short-form --skip-gtid-strict-mode --stop-position=1-1-9 MYSQLD_DATADIR/master-bin.000001 --result-file=ignored_output_file
+WARNING: Did not reach stop position 1-1-9 before end of input
+Case: Stop GTID nonexistent, must see warning.
+# MYSQL_BINLOG --short-form --skip-gtid-strict-mode --stop-position=9-1-1 MYSQLD_DATADIR/master-bin.000001 --result-file=ignored_output_file
+WARNING: Did not reach stop position 9-1-1 before end of input
+Case: Stop GTID at EOF + after EOF, must see only one warning.
+# MYSQL_BINLOG --short-form --skip-gtid-strict-mode --stop-position=0-1-2,1-1-9 MYSQLD_DATADIR/master-bin.000001 --result-file=ignored_output_file
+WARNING: Did not reach stop position 1-1-9 before end of input
+Case: Stop GTID after EOF + nonexistent, must see two warnings.
+# MYSQL_BINLOG --short-form --skip-gtid-strict-mode --stop-position=1-1-9,9-1-1 MYSQLD_DATADIR/master-bin.000001 --result-file=ignored_output_file
+WARNING: Did not reach stop position 9-1-1 before end of input
+WARNING: Did not reach stop position 1-1-9 before end of input
+DROP TABLE t1;
+DROP TABLE t2;
+# End of binlog_mysqlbinlog_warn_stop_gtid.test

mysql-test/suite/binlog/t/binlog_mysqlbinlog_warn_stop_gtid.test

@@ -0,0 +1,50 @@
+--echo
+--echo # MDEV-34614 mysqlbinlog warn on EOF before GTID in --stop-position
+--echo
+
+--source include/have_log_bin.inc
+
+--let ignored_output_file= $MYSQLTEST_VARDIR/tmp/warn_pos_test_file.out
+
+RESET MASTER;
+SET @@session.gtid_domain_id= 0;
+SET @@session.server_id= 1;
+SET @@session.gtid_seq_no= 1;
+CREATE TABLE t1(a int);
+INSERT INTO t1 VALUES (1), (2);
+SET @@session.gtid_domain_id= 1;
+CREATE TABLE t2(a int);
+INSERT INTO t2 VALUES (3);
+--let MYSQLD_DATADIR= `SELECT @@datadir;`
+FLUSH LOGS;
+
+--echo Case: Default, must not see warning.
+--echo # MYSQL_BINLOG --short-form MYSQLD_DATADIR/master-bin.000001 --result-file=ignored_output_file
+--exec $MYSQL_BINLOG --short-form $MYSQLD_DATADIR/master-bin.000001 --result-file=$ignored_output_file 2>&1
+
+--echo Case: Stop GTID at EOF, must not see warning.
+--echo # MYSQL_BINLOG --short-form --skip-gtid-strict-mode --stop-position=0-1-2 MYSQLD_DATADIR/master-bin.000001 --result-file=ignored_output_file
+--exec $MYSQL_BINLOG --short-form --skip-gtid-strict-mode --stop-position=0-1-2 $MYSQLD_DATADIR/master-bin.000001 --result-file=$ignored_output_file 2>&1
+
+--echo Case: Stop GTID after EOF, must see warning.
+--echo # MYSQL_BINLOG --short-form --skip-gtid-strict-mode --stop-position=1-1-9 MYSQLD_DATADIR/master-bin.000001 --result-file=ignored_output_file
+--exec $MYSQL_BINLOG --short-form --skip-gtid-strict-mode --stop-position=1-1-9 $MYSQLD_DATADIR/master-bin.000001 --result-file=$ignored_output_file 2>&1
+
+--echo Case: Stop GTID nonexistent, must see warning.
+--echo # MYSQL_BINLOG --short-form --skip-gtid-strict-mode --stop-position=9-1-1 MYSQLD_DATADIR/master-bin.000001 --result-file=ignored_output_file
+--exec $MYSQL_BINLOG --short-form --skip-gtid-strict-mode --stop-position=9-1-1 $MYSQLD_DATADIR/master-bin.000001 --result-file=$ignored_output_file 2>&1
+
+--echo Case: Stop GTID at EOF + after EOF, must see only one warning.
+--echo # MYSQL_BINLOG --short-form --skip-gtid-strict-mode --stop-position=0-1-2,1-1-9 MYSQLD_DATADIR/master-bin.000001 --result-file=ignored_output_file
+--exec $MYSQL_BINLOG --short-form --skip-gtid-strict-mode --stop-position=0-1-2,1-1-9 $MYSQLD_DATADIR/master-bin.000001 --result-file=$ignored_output_file 2>&1
+
+--echo Case: Stop GTID after EOF + nonexistent, must see two warnings.
+--echo # MYSQL_BINLOG --short-form --skip-gtid-strict-mode --stop-position=1-1-9,9-1-1 MYSQLD_DATADIR/master-bin.000001 --result-file=ignored_output_file
+--exec $MYSQL_BINLOG --short-form --skip-gtid-strict-mode --stop-position=1-1-9,9-1-1 $MYSQLD_DATADIR/master-bin.000001 --result-file=$ignored_output_file 2>&1
+
+DROP TABLE t1;
+DROP TABLE t2;
+
+--remove_file $ignored_output_file
+
+--echo # End of binlog_mysqlbinlog_warn_stop_gtid.test

sql/rpl_gtid.cc

@@ -3526,6 +3526,32 @@ my_bool Window_gtid_event_filter::has_finished()
   return m_has_stop ? m_has_passed : FALSE;
 }
 
+my_bool Window_gtid_event_filter::verify_completed_state()
+{
+  /*TODO: validate `--start-position` (::m_start)
+    Normally with ::m_has_start, the states goes from inactive to ::m_is_active,
+    then to ::m_has_passed if ::m_has_stop (::has_finished()).
+    However, the filter can finish at ::m_has_passed without ever becoming
+    ::m_is_active if ::is_range_invalid(), complicating checks.
+    Ideally, ::is_range_invalid() should warn outside of
+    `--gtid-strict-mode` as well, but that's its own distinct discussion.
+  */
+  if (m_has_stop && !m_has_passed)
+  {
+    /*XXX:
+      `mysqlbinlog.cc`'s `static warning()` family is all duplicate
+      code of the Binlog_gtid_state_validator::warn() family, except
+      it specifies stderr and has a `fflush(result_file)` step.
+    */
+    Binlog_gtid_state_validator::warn(stderr,
+      "Did not reach stop position %u-%u-%llu before end of input",
+      PARAM_GTID(m_stop)
+    );
+    return TRUE;
+  }
+  return FALSE;
+}
+
 void free_u32_gtid_filter_element(void *p)
 {
   gtid_filter_element<uint32> *gfe= (gtid_filter_element<uint32> *) p;
@@ -3608,6 +3634,31 @@ my_bool Id_delegating_gtid_event_filter<T>::has_finished()
          m_num_completed_filters == m_num_stateful_filters;
 }
 
+/**
+  Iteration block for Id_delegating_gtid_event_filter::verify_completed_state()
+*/
+static my_bool
+verify_subfilter_completed_state(void *entry, void *is_any_incomplete)
+{
+  if (entry && reinterpret_cast<
+    gtid_filter_element<decltype(rpl_gtid::domain_id)> *
+  >(entry)->filter->verify_completed_state())
+    *(my_bool *)is_any_incomplete= TRUE;
+  return FALSE; // do not terminate early
+}
+
+template <typename T>
+my_bool Id_delegating_gtid_event_filter<T>::verify_completed_state()
+{
+  if (has_finished()) // fast happy path
+    return FALSE;
+  // If a user-defined filters is not deactivated, it may not be complete.
+  my_bool is_any_incomplete= FALSE;
+  my_hash_iterate(&m_filters_by_id_hash,
+                  verify_subfilter_completed_state, &is_any_incomplete);
+  return is_any_incomplete;
+}
+
 template <typename T>
 my_bool Id_delegating_gtid_event_filter<T>::exclude(rpl_gtid *gtid)
 {
@@ -4068,3 +4119,17 @@ my_bool Intersecting_gtid_event_filter::has_finished()
   }
   return FALSE;
 }
+
+my_bool Intersecting_gtid_event_filter::verify_completed_state()
+{
+  my_bool is_any_incomplete= FALSE;
+  Gtid_event_filter *subfilter;
+  for (size_t i= 0; i < m_filters.elements; i++)
+  {
+    subfilter= *(Gtid_event_filter **)dynamic_array_ptr(&m_filters, i);
+    DBUG_ASSERT(subfilter);
+    if (subfilter->verify_completed_state())
+      is_any_incomplete= TRUE;
+  }
+  return is_any_incomplete;
+}

sql/rpl_gtid.h

@@ -568,7 +568,25 @@ class Gtid_event_filter
 
     Returns TRUE when completed, and FALSE when the filter has not finished.
   */
-  virtual my_bool has_finished() = 0;
+  virtual my_bool has_finished() { return FALSE; }
+
+  /**
+    Check that this filter implementation is at a *completely* final state,
+    and warn if it is not.
+
+    For a filter that can maintain their own state,
+    this not only validates if the filter ::has_finished(),
+    but may also print specific warnings for its variety of non-finished states.
+
+    For a filter that manage multiple subfilters, this should iterate through
+    *all* subfilters to have each self-report if it wasn't fully effective.
+    This cumulative result may not correlate with the ::has_finished() state.
+
+    @return
+      FALSE if the filter is at a completed state, or TRUE if it warned
+      its state (This scheme is the opposite of ::has_finished()!)
+  */
+  virtual my_bool verify_completed_state() { return FALSE; }
 };
 
 /*
@@ -585,7 +603,6 @@ class Accept_all_gtid_filter : public Gtid_event_filter
   ~Accept_all_gtid_filter() {}
   my_bool exclude(rpl_gtid *) override { return FALSE; }
   uint32 get_filter_type() override { return ACCEPT_ALL_GTID_FILTER_TYPE; }
-  my_bool has_finished() override { return FALSE; }
 };
 
 /*
@@ -598,7 +615,6 @@ class Reject_all_gtid_filter : public Gtid_event_filter
   ~Reject_all_gtid_filter() {}
   my_bool exclude(rpl_gtid *) override { return TRUE; }
   uint32 get_filter_type() override { return REJECT_ALL_GTID_FILTER_TYPE; }
-  my_bool has_finished() override { return FALSE; }
 };
 
 /*
@@ -618,6 +634,7 @@ class Window_gtid_event_filter : public Gtid_event_filter
 
   my_bool exclude(rpl_gtid*) override;
   my_bool has_finished() override;
+  my_bool verify_completed_state() override;
 
   /*
     Set the GTID that begins this window (exclusive)
@@ -692,7 +709,7 @@ class Window_gtid_event_filter : public Gtid_event_filter
 
   /*
     m_has_passed : Indicates whether or not the program is currently reading
-                   events from within this window.
+                   events from beyond this window.
    */
   my_bool m_has_passed;
 
@@ -729,6 +746,7 @@ class Id_delegating_gtid_event_filter : public Gtid_event_filter
 
   my_bool exclude(rpl_gtid *gtid) override;
   my_bool has_finished() override;
+  my_bool verify_completed_state() override;
   void set_default_filter(Gtid_event_filter *default_filter);
 
   uint32 get_filter_type() override { return DELEGATING_GTID_FILTER_TYPE; }
@@ -910,6 +928,7 @@ class Intersecting_gtid_event_filter : public Gtid_event_filter
   */
   my_bool has_finished() override;
 
+  my_bool verify_completed_state() override;
   uint32 get_filter_type() override { return INTERSECTING_GTID_FILTER_TYPE; }
 
   /*