[Bug]: Bigtable to BigQuery streaming template - many errors on bigtable api audit log

[giogit]

Related Template(s)

bigtable_change_streams_to_bigquery

Template Version

2024-12-03-00_rc00

What happened?

The Dataflow job transfers data correctly and no errors are logged on client side.
But on the Bigtable audit logs, thousands of errors (per minute) are constantly written:
methodName: "google.bigtable.v2.Bigtable.ReadChangeStream"
message: "The operation was cancelled."

Relevant log output

{
  "protoPayload": {
    "@type": "type.googleapis.com/google.cloud.audit.AuditLog",
    "status": {
      "code": 1,
      "message": "Request cancelled; terminating connection."
    },
    "authenticationInfo": {
      "principalEmail": "[email protected]",
      "serviceAccountDelegationInfo": [
        {
          "firstPartyPrincipal": {
            "principalEmail": "[email protected]"
          }
        }
      ],
      "principalSubject": "serviceAccount:[email protected]"
    },
    "requestMetadata": {
      "callerIp": "gce-internal-ip",
      "callerSuppliedUserAgent": "bigtable-java/2.31.0 grpc-java-netty/1.60.0,gzip(gfe)",
      "requestAttributes": {
        "time": "2024-12-16T12:45:04.410658229Z",
        "auth": {}
      },
      "destinationAttributes": {}
    },
    "serviceName": "bigtable.googleapis.com",
    "methodName": "google.bigtable.v2.Bigtable.ReadChangeStream",
    "authorizationInfo": [
      {
        "resource": "projects/.../instances/newledge-metrics/tables/data",
        "permission": "bigtable.tables.readRows",
        "granted": true,
        "resourceAttributes": {
          "service": "bigtableadmin.googleapis.com",
          "name": "projects/.../instances/newledge-metrics/tables/data",
          "type": "bigtableadmin.googleapis.com/Table"
        },
        "permissionType": "DATA_READ"
      }
    ],
    "resourceName": "projects/.../instances/newledge-metrics/tables/data",
    "request": {
      "@type": "type.googleapis.com/google.bigtable.v2.ReadChangeStreamRequest",
      "continuationTokens": {
        "tokens": [
          {
            "token": "01WHVNTmJfKzhKU1VlaUVpatUAAAArmz0ihIwFFP86lOCO14HkNyR8puk0WvLqgNdRACXN03DYR3XImDajKO7qYFjC0eTpKrL5hf0YySlXvhKFvNqtCL37oYlSsXePzSEmNkWq4W0_IEvmVpWcMiNPMY9eQGwaR_TzGmHxqwRvDIUJzaMGxb-DxpN0WDOd39qtp8ZKfQLM3aNWw0PC0dFsIwhBGpNKzl_ude_nWPF1c7fEYT7p7PPFqX1PwRDamwJBsN0SnynT_Qcx5qYGwqFhrJ55R7IhUGxQeNJFqQHltw8vVFOGThF9wL6PSMh6t9WWQGJXzQ",
            "partition": {
              "rowRange": {
                "startKeyClosed": "VVU0MDpkZTo9MDAwOnJhdzpyMg==",
                "endKeyOpen": "VVU0MDpkZTo9MDAwOnJhdzpz"
              }
            }
          }
        ]
      },
      "partition": {
        "rowRange": {
          "startKeyClosed": "VVU0MDpkZTo9MDAwOnJhdzpyMg==",
          "endKeyOpen": "VVU0MDpkZTo9MDAwOnJhdzpz"
        }
      },
      "heartbeatDuration": "1s",
      "tableName": "projects/.../instances/newledge-metrics/tables/data",
      "appProfileId": "cr-change-stream-app-profile"
    }
  },
  "insertId": "1sdppy7e1lx5n",
  "resource": {
    "type": "audited_resource",
    "labels": {
      "method": "google.bigtable.v2.Bigtable.ReadChangeStream",
      "service": "bigtable.googleapis.com",
      "project_id": "..."
    }
  },
  "timestamp": "2024-12-16T12:45:04.410622637Z",
  "severity": "ERROR",
  "logName": "projects/.../logs/cloudaudit.googleapis.com%2Fdata_access",
  "operation": {
    "id": "5418149984605994989",
    "producer": "bigtable.googleapis.com",
    "first": true
  },
  "receiveTimestamp": "2024-12-16T12:45:09.816061413Z"
}