diff --git a/tap_github/client.py b/tap_github/client.py index c63332b..bb3f2e3 100644 --- a/tap_github/client.py +++ b/tap_github/client.py @@ -90,7 +90,23 @@ def get_records(self, context: Context | None) -> Iterable[dict[str, Any]]: if context is not None and "org" in context: self.authenticator.set_organization(context["org"]) - yield from super().get_records(context) + try: + yield from super().get_records(context) + except Exception as exc: + # singer_sdk's own Stream.sync() catch-all only logs "an unhandled + # error occurred while syncing ''" with no detail on why. + # Surface the real cause here before it propagates - + # FatalAPIError/RetriableAPIError already carry a rich message + # (HTTP status, GitHub-Request-Id, ...) that would otherwise be + # lost. + self.logger.error( + "%s failed for context=%s: %s: %s", + self.name, + context, + type(exc).__name__, + exc, + ) + raise def _throttle_before_request(self) -> None: """Sleep if we're currently backing off from a secondary rate limit.""" diff --git a/tap_github/repository_streams.py b/tap_github/repository_streams.py index 0d2c6c5..2dbafb2 100644 --- a/tap_github/repository_streams.py +++ b/tap_github/repository_streams.py @@ -307,7 +307,12 @@ def _get_records_round_robin(self, context: Context) -> Iterable[dict[str, Any]] all_records = list(super().get_records(context)) if not all_records: - self.logger.info(f"[{self.name}:{partition_label}] round-robin summary: 0 repos returned by the API.") + # warning, not info: prod runs at warning level, and this + # summary is the only per-run signal of enumeration progress. + self.logger.warning( + f"[{self.name}:{partition_label}] round-robin summary: " + "0 repos returned by the API." + ) return all_records.sort(key=lambda record: record["id"]) @@ -342,7 +347,10 @@ def _get_records_round_robin(self, context: Context) -> Iterable[dict[str, Any]] self._write_state_message() finally: full_lap = processed >= len(ordered_records) - self.logger.info( + # warning, not info: prod runs at warning level, and this is the + # only per-run signal of enumeration progress - without it there + # is no visibility into how far a run got before crashing. + self.logger.warning( f"[{self.name}:{partition_label}] round-robin summary: " f"processed {processed}/{len(ordered_records)} repos in " f"{time.monotonic() - started_at:.1f}s "