"app-id"public static final java.lang.StringAUTH_DEFAULT_REFRESH_ENABLED"auth.default-refresh-enabled"AUTH_SESSION_TIMEOUT_MS"auth.session-timeout-ms"public static final booleanAUTH_DEFAULT_REFRESH_ENABLED_DEFAULTfalsepublic static final java.lang.String
+CACHE_CASE_SENSITIVE"cache.case-sensitive"public static final java.lang.StringAUTH_SESSION_TIMEOUT_MS"auth.session-timeout-ms"public static final boolean
+CACHE_CASE_SENSITIVE_DEFAULTtrue"client.pool.cache.eviction-interval-ms"public static final java.lang.StringCLIENT_POOL_CACHE_KEYS"client-pool-cache-keys"public static final java.lang.StringCLIENT_POOL_SIZE"clients"public static final intCLIENT_POOL_SIZE_DEFAULT2public static final java.lang.StringFILE_IO_IMPL"io-impl"public static final java.lang.StringIO_MANIFEST_CACHE_ENABLED"io.manifest.cache-enabled"public static final booleanIO_MANIFEST_CACHE_ENABLED_DEFAULTfalsepublic static final java.lang.StringIO_MANIFEST_CACHE_EXPIRATION_INTERVAL_MS"io.manifest.cache.expiration-interval-ms"public static final java.lang.StringIO_MANIFEST_CACHE_MAX_CONTENT_LENGTH"io.manifest.cache.max-content-length"public static final longIO_MANIFEST_CACHE_MAX_CONTENT_LENGTH_DEFAULT8388608Lpublic static final java.lang.StringIO_MANIFEST_CACHE_MAX_TOTAL_BYTES"io.manifest.cache.max-total-bytes"public static final longIO_MANIFEST_CACHE_MAX_TOTAL_BYTES_DEFAULT104857600Lpublic static final java.lang.StringLOCK_ACQUIRE_INTERVAL_MS"lock.acquire-interval-ms"public static final java.lang.StringLOCK_ACQUIRE_TIMEOUT_MS"lock.acquire-timeout-ms"public static final java.lang.StringLOCK_HEARTBEAT_INTERVAL_MS"lock.heartbeat-interval-ms"public static final java.lang.StringLOCK_HEARTBEAT_THREADS"lock.heartbeat-threads"public static final intLOCK_HEARTBEAT_THREADS_DEFAULT4public static final java.lang.StringLOCK_HEARTBEAT_TIMEOUT_MS"lock.heartbeat-timeout-ms"public static final java.lang.StringLOCK_IMPL"lock-impl"public static final java.lang.StringLOCK_TABLE"lock.table"public static final java.lang.StringMETRICS_REPORTER_IMPL"metrics-reporter-impl"public static final java.lang.StringTABLE_DEFAULT_PREFIX"table-default."public static final java.lang.StringTABLE_OVERRIDE_PREFIX"table-override."public static final java.lang.StringURI"uri"public static final java.lang.StringUSER"user"public static final java.lang.String| Modifier and Type | +Constant Field | +Value | +
|---|---|---|
+
+public static final java.lang.String |
+DELETE_FILE_PATH |
+"delete_file_path" |
+
+
+public static final java.lang.String |
+PARTITION |
+"partition" |
+
+
+public static final java.lang.String |
+SPEC_ID |
+"spec_id" |
+
| Modifier and Type | @@ -928,6 +968,20 @@|||||
|---|---|---|---|---|---|
+
+public static final java.lang.String |
+ADAPTIVE_SPLIT_SIZE_ENABLED |
+"read.split.adaptive-size.enabled" |
+|||
+
+public static final boolean |
+ADAPTIVE_SPLIT_SIZE_ENABLED_DEFAULT |
+true |
+|||
public static final java.lang.String |
@@ -1089,194 +1143,208 @@ "current-snapshot-timestamp-ms" |
||||
+
+public static final java.lang.String |
+DATA_PLANNING_MODE |
+"read.data-planning-mode" |
+|||
public static final java.lang.String |
DEFAULT_FILE_FORMAT |
"write.format.default" |
|||
public static final java.lang.String |
DEFAULT_FILE_FORMAT_DEFAULT |
"parquet" |
|||
public static final java.lang.String |
DEFAULT_NAME_MAPPING |
"schema.name-mapping.default" |
|||
public static final java.lang.String |
DEFAULT_PARTITION_SPEC |
"default-partition-spec" |
|||
public static final java.lang.String |
DEFAULT_SORT_ORDER |
"default-sort-order" |
|||
public static final java.lang.String |
DEFAULT_WRITE_METRICS_MODE |
"write.metadata.metrics.default" |
|||
public static final java.lang.String |
DEFAULT_WRITE_METRICS_MODE_DEFAULT |
"truncate(16)" |
|||
public static final java.lang.String |
DELETE_AVRO_COMPRESSION |
"write.delete.avro.compression-codec" |
|||
public static final java.lang.String |
DELETE_AVRO_COMPRESSION_LEVEL |
"write.delete.avro.compression-level" |
|||
public static final java.lang.String |
DELETE_DEFAULT_FILE_FORMAT |
"write.delete.format.default" |
|||
public static final java.lang.String |
DELETE_DISTRIBUTION_MODE |
"write.delete.distribution-mode" |
|||
public static final java.lang.String |
DELETE_ISOLATION_LEVEL |
"write.delete.isolation-level" |
|||
public static final java.lang.String |
DELETE_ISOLATION_LEVEL_DEFAULT |
"serializable" |
|||
public static final java.lang.String |
DELETE_MODE |
"write.delete.mode" |
|||
public static final java.lang.String |
DELETE_ORC_BLOCK_SIZE_BYTES |
"write.delete.orc.block-size-bytes" |
|||
public static final java.lang.String |
DELETE_ORC_COMPRESSION |
"write.delete.orc.compression-codec" |
|||
public static final java.lang.String |
DELETE_ORC_COMPRESSION_STRATEGY |
"write.delete.orc.compression-strategy" |
|||
public static final java.lang.String |
DELETE_ORC_STRIPE_SIZE_BYTES |
"write.delete.orc.stripe-size-bytes" |
|||
public static final java.lang.String |
DELETE_ORC_WRITE_BATCH_SIZE |
"write.delete.orc.vectorized.batch-size" |
|||
public static final java.lang.String |
DELETE_PARQUET_COMPRESSION |
"write.delete.parquet.compression-codec" |
|||
public static final java.lang.String |
DELETE_PARQUET_COMPRESSION_LEVEL |
"write.delete.parquet.compression-level" |
|||
public static final java.lang.String |
DELETE_PARQUET_DICT_SIZE_BYTES |
"write.delete.parquet.dict-size-bytes" |
|||
public static final java.lang.String |
DELETE_PARQUET_PAGE_ROW_LIMIT |
"write.delete.parquet.page-row-limit" |
|||
public static final java.lang.String |
DELETE_PARQUET_PAGE_SIZE_BYTES |
"write.delete.parquet.page-size-bytes" |
|||
public static final java.lang.String |
DELETE_PARQUET_ROW_GROUP_CHECK_MAX_RECORD_COUNT |
"write.delete.parquet.row-group-check-max-record-count" |
|||
public static final java.lang.String |
DELETE_PARQUET_ROW_GROUP_CHECK_MIN_RECORD_COUNT |
"write.delete.parquet.row-group-check-min-record-count" |
|||
public static final java.lang.String |
DELETE_PARQUET_ROW_GROUP_SIZE_BYTES |
"write.delete.parquet.row-group-size-bytes" |
|||
+
+public static final java.lang.String |
+DELETE_PLANNING_MODE |
+"read.delete-planning-mode" |
+|||
@@ -1327,425 +1395,425 @@ org.apache.* | true |
||||
| + |
public static final java.lang.String |
-HMS_TABLE_OWNER |
-"hive.metastore.table.owner" |
+HIVE_LOCK_ENABLED |
+"engine.hive.lock-enabled" |
+
+public static final boolean |
+HIVE_LOCK_ENABLED_DEFAULT |
+true |
+|||
public static final java.lang.String |
MANIFEST_LISTS_ENABLED |
"write.manifest-lists.enabled" |
|||
public static final boolean |
MANIFEST_LISTS_ENABLED_DEFAULT |
true |
|||
public static final java.lang.String |
MANIFEST_MERGE_ENABLED |
"commit.manifest-merge.enabled" |
|||
public static final boolean |
MANIFEST_MERGE_ENABLED_DEFAULT |
true |
|||
public static final java.lang.String |
MANIFEST_MIN_MERGE_COUNT |
"commit.manifest.min-count-to-merge" |
|||
public static final int |
MANIFEST_MIN_MERGE_COUNT_DEFAULT |
100 |
|||
public static final java.lang.String |
MANIFEST_TARGET_SIZE_BYTES |
"commit.manifest.target-size-bytes" |
|||
public static final long |
MANIFEST_TARGET_SIZE_BYTES_DEFAULT |
8388608L |
|||
public static final java.lang.String |
MAX_REF_AGE_MS |
"history.expire.max-ref-age-ms" |
|||
public static final long |
MAX_REF_AGE_MS_DEFAULT |
9223372036854775807L |
|||
public static final java.lang.String |
MAX_SNAPSHOT_AGE_MS |
"history.expire.max-snapshot-age-ms" |
|||
public static final long |
MAX_SNAPSHOT_AGE_MS_DEFAULT |
432000000L |
|||
-
-public static final java.lang.String |
-MERGE_CARDINALITY_CHECK_ENABLED |
-"write.merge.cardinality-check.enabled" |
-|||
-
-public static final boolean |
-MERGE_CARDINALITY_CHECK_ENABLED_DEFAULT |
-true |
-|||
public static final java.lang.String |
MERGE_DISTRIBUTION_MODE |
"write.merge.distribution-mode" |
|||
public static final java.lang.String |
MERGE_ISOLATION_LEVEL |
"write.merge.isolation-level" |
|||
public static final java.lang.String |
MERGE_ISOLATION_LEVEL_DEFAULT |
"serializable" |
|||
public static final java.lang.String |
MERGE_MODE |
"write.merge.mode" |
|||
public static final java.lang.String |
METADATA_COMPRESSION |
"write.metadata.compression-codec" |
|||
public static final java.lang.String |
METADATA_COMPRESSION_DEFAULT |
"none" |
|||
public static final java.lang.String |
METADATA_DELETE_AFTER_COMMIT_ENABLED |
"write.metadata.delete-after-commit.enabled" |
|||
public static final boolean |
METADATA_DELETE_AFTER_COMMIT_ENABLED_DEFAULT |
false |
|||
public static final java.lang.String |
METADATA_PREVIOUS_VERSIONS_MAX |
"write.metadata.previous-versions-max" |
|||
public static final int |
METADATA_PREVIOUS_VERSIONS_MAX_DEFAULT |
100 |
|||
public static final java.lang.String |
METADATA_SPLIT_SIZE |
"read.split.metadata-target-size" |
|||
public static final long |
METADATA_SPLIT_SIZE_DEFAULT |
33554432L |
|||
public static final java.lang.String |
METRICS_MAX_INFERRED_COLUMN_DEFAULTS |
"write.metadata.metrics.max-inferred-column-defaults" |
|||
public static final int |
METRICS_MAX_INFERRED_COLUMN_DEFAULTS_DEFAULT |
100 |
|||
public static final java.lang.String |
METRICS_MODE_COLUMN_CONF_PREFIX |
"write.metadata.metrics.column." |
|||
public static final java.lang.String |
MIN_SNAPSHOTS_TO_KEEP |
"history.expire.min-snapshots-to-keep" |
|||
public static final int |
MIN_SNAPSHOTS_TO_KEEP_DEFAULT |
1 |
|||
public static final java.lang.String |
OBJECT_STORE_ENABLED |
"write.object-storage.enabled" |
|||
public static final boolean |
OBJECT_STORE_ENABLED_DEFAULT |
false |
|||
public static final java.lang.String |
OBJECT_STORE_PATH |
"write.object-storage.path" |
|||
public static final java.lang.String |
ORC_BATCH_SIZE |
"read.orc.vectorization.batch-size" |
|||
public static final int |
ORC_BATCH_SIZE_DEFAULT |
5000 |
|||
public static final java.lang.String |
ORC_BLOCK_SIZE_BYTES |
"write.orc.block-size-bytes" |
|||
public static final long |
ORC_BLOCK_SIZE_BYTES_DEFAULT |
268435456L |
|||
public static final java.lang.String |
ORC_BLOOM_FILTER_COLUMNS |
"write.orc.bloom.filter.columns" |
|||
public static final java.lang.String |
ORC_BLOOM_FILTER_COLUMNS_DEFAULT |
"" |
|||
public static final java.lang.String |
ORC_BLOOM_FILTER_FPP |
"write.orc.bloom.filter.fpp" |
|||
public static final double |
ORC_BLOOM_FILTER_FPP_DEFAULT |
0.05 |
|||
public static final java.lang.String |
ORC_COMPRESSION |
"write.orc.compression-codec" |
|||
public static final java.lang.String |
ORC_COMPRESSION_DEFAULT |
"zlib" |
|||
public static final java.lang.String |
ORC_COMPRESSION_STRATEGY |
"write.orc.compression-strategy" |
|||
public static final java.lang.String |
ORC_COMPRESSION_STRATEGY_DEFAULT |
"speed" |
|||
public static final java.lang.String |
ORC_STRIPE_SIZE_BYTES |
"write.orc.stripe-size-bytes" |
|||
public static final long |
ORC_STRIPE_SIZE_BYTES_DEFAULT |
67108864L |
|||
public static final java.lang.String |
ORC_VECTORIZATION_ENABLED |
"read.orc.vectorization.enabled" |
|||
public static final boolean |
ORC_VECTORIZATION_ENABLED_DEFAULT |
false |
|||
public static final java.lang.String |
ORC_WRITE_BATCH_SIZE |
"write.orc.vectorized.batch-size" |
|||
public static final int |
ORC_WRITE_BATCH_SIZE_DEFAULT |
1024 |
|||
public static final java.lang.String |
PARQUET_BATCH_SIZE |
"read.parquet.vectorization.batch-size" |
|||
public static final int |
PARQUET_BATCH_SIZE_DEFAULT |
5000 |
|||
public static final java.lang.String |
PARQUET_BLOOM_FILTER_COLUMN_ENABLED_PREFIX |
"write.parquet.bloom-filter-enabled.column." |
|||
public static final java.lang.String |
PARQUET_BLOOM_FILTER_MAX_BYTES |
"write.parquet.bloom-filter-max-bytes" |
|||
public static final int |
PARQUET_BLOOM_FILTER_MAX_BYTES_DEFAULT |
1048576 |
|||
public static final java.lang.String |
PARQUET_COMPRESSION |
"write.parquet.compression-codec" |
|||
public static final java.lang.String |
PARQUET_COMPRESSION_DEFAULT |
"gzip" |
|||
+
+public static final java.lang.String |
+PARQUET_COMPRESSION_DEFAULT_SINCE_1_4_0 |
+"zstd" |
+|||
@@ -1887,202 +1955,209 @@ org.apache.* | false |
||||
+
+public static final java.lang.String |
+SPARK_WRITE_ADVISORY_PARTITION_SIZE_BYTES |
+"write.spark.advisory-partition-size-bytes" |
+|||
public static final java.lang.String |
SPARK_WRITE_PARTITIONED_FANOUT_ENABLED |
"write.spark.fanout.enabled" |
|||
public static final boolean |
SPARK_WRITE_PARTITIONED_FANOUT_ENABLED_DEFAULT |
false |
|||
public static final java.lang.String |
SPLIT_LOOKBACK |
"read.split.planning-lookback" |
|||
public static final int |
SPLIT_LOOKBACK_DEFAULT |
10 |
|||
public static final java.lang.String |
SPLIT_OPEN_FILE_COST |
"read.split.open-file-cost" |
|||
public static final long |
SPLIT_OPEN_FILE_COST_DEFAULT |
4194304L |
|||
public static final java.lang.String |
SPLIT_SIZE |
"read.split.target-size" |
|||
public static final long |
SPLIT_SIZE_DEFAULT |
134217728L |
|||
public static final java.lang.String |
UPDATE_DISTRIBUTION_MODE |
"write.update.distribution-mode" |
|||
public static final java.lang.String |
UPDATE_ISOLATION_LEVEL |
"write.update.isolation-level" |
|||
public static final java.lang.String |
UPDATE_ISOLATION_LEVEL_DEFAULT |
"serializable" |
|||
public static final java.lang.String |
UPDATE_MODE |
"write.update.mode" |
|||
public static final java.lang.String |
UPSERT_ENABLED |
"write.upsert.enabled" |
|||
public static final boolean |
UPSERT_ENABLED_DEFAULT |
false |
|||
public static final java.lang.String |
UUID |
"uuid" |
|||
public static final java.lang.String |
WRITE_AUDIT_PUBLISH_ENABLED |
"write.wap.enabled" |
|||
public static final java.lang.String |
WRITE_AUDIT_PUBLISH_ENABLED_DEFAULT |
"false" |
|||
public static final java.lang.String |
WRITE_DATA_LOCATION |
"write.data.path" |
|||
public static final java.lang.String |
WRITE_DISTRIBUTION_MODE |
"write.distribution-mode" |
|||
public static final java.lang.String |
WRITE_DISTRIBUTION_MODE_HASH |
"hash" |
|||
public static final java.lang.String |
WRITE_DISTRIBUTION_MODE_NONE |
"none" |
|||
public static final java.lang.String |
WRITE_DISTRIBUTION_MODE_RANGE |
"range" |
|||
public static final java.lang.String |
WRITE_FOLDER_STORAGE_LOCATION |
"write.folder-storage.path" |
|||
public static final java.lang.String |
WRITE_LOCATION_PROVIDER_IMPL |
"write.location-provider.impl" |
|||
public static final java.lang.String |
WRITE_METADATA_LOCATION |
"write.metadata.path" |
|||
public static final java.lang.String |
WRITE_PARTITION_SUMMARY_LIMIT |
"write.summary.partition-limit" |
|||
public static final int |
WRITE_PARTITION_SUMMARY_LIMIT_DEFAULT |
0 |
|||
public static final java.lang.String |
WRITE_TARGET_FILE_SIZE_BYTES |
"write.target-file-size-bytes" |
|||
public static final long |
@@ -2197,7 +2272,7 @@ MAX_CONCURRENT_FILE_GROUP_REWRITES_DEFAULT |
-1 |
+5 |
||
@@ -2272,6 +2347,182 @@ org.apache.* |
| Modifier and Type | +Constant Field | +Value | +
|---|---|---|
+
+public static final java.lang.String |
+MAX_CONCURRENT_FILE_GROUP_REWRITES |
+"max-concurrent-file-group-rewrites" |
+
+
+public static final int |
+MAX_CONCURRENT_FILE_GROUP_REWRITES_DEFAULT |
+5 |
+
+
+public static final java.lang.String |
+PARTIAL_PROGRESS_ENABLED |
+"partial-progress.enabled" |
+
+
+public static final boolean |
+PARTIAL_PROGRESS_ENABLED_DEFAULT |
+false |
+
+
+public static final java.lang.String |
+PARTIAL_PROGRESS_MAX_COMMITS |
+"partial-progress.max-commits" |
+
+
+public static final int |
+PARTIAL_PROGRESS_MAX_COMMITS_DEFAULT |
+10 |
+
+
+public static final java.lang.String |
+REWRITE_JOB_ORDER |
+"rewrite-job-order" |
+
| Modifier and Type | +Constant Field | +Value | +
|---|---|---|
+
+public static final java.lang.String |
+DELETE_FILE_THRESHOLD |
+"delete-file-threshold" |
+
+
+public static final int |
+DELETE_FILE_THRESHOLD_DEFAULT |
+2147483647 |
+
| Modifier and Type | +Constant Field | +Value | +
|---|---|---|
+
+public static final java.lang.String |
+MAX_FILE_GROUP_SIZE_BYTES |
+"max-file-group-size-bytes" |
+
+
+public static final long |
+MAX_FILE_GROUP_SIZE_BYTES_DEFAULT |
+107374182400L |
+
+
+public static final java.lang.String |
+MAX_FILE_SIZE_BYTES |
+"max-file-size-bytes" |
+
+
+public static final double |
+MAX_FILE_SIZE_DEFAULT_RATIO |
+1.8 |
+
+
+public static final java.lang.String |
+MIN_FILE_SIZE_BYTES |
+"min-file-size-bytes" |
+
+
+public static final double |
+MIN_FILE_SIZE_DEFAULT_RATIO |
+0.75 |
+
+
+public static final java.lang.String |
+MIN_INPUT_FILES |
+"min-input-files" |
+
+
+public static final int |
+MIN_INPUT_FILES_DEFAULT |
+5 |
+
+
+public static final java.lang.String |
+REWRITE_ALL |
+"rewrite-all" |
+
+
+public static final boolean |
+REWRITE_ALL_DEFAULT |
+false |
+
+
+public static final java.lang.String |
+TARGET_FILE_SIZE_BYTES |
+"target-file-size-bytes" |
+
| Modifier and Type | +Constant Field | +Value | +
|---|---|---|
+
+protected static final java.lang.String |
+CLIENT_CREDENTIAL_PROVIDER_PREFIX |
+"client.credentials-provider." |
+
+
+public static final java.lang.String |
+CLIENT_CREDENTIALS_PROVIDER |
+"client.credentials-provider" |
+
+
+public static final java.lang.String |
+CLIENT_REGION |
+"client.region" |
+
| Modifier and Type | +Constant Field | +Value | |||
|---|---|---|---|---|---|
| + |
public static final java.lang.String |
-HTTP_CLIENT_APACHE_USE_IDLE_CONNECTION_REAPER_ENABLED |
-"http-client.apache.use-idle-connection-reaper-enabled" |
+APACHE_CONNECTION_ACQUISITION_TIMEOUT_MS |
+"http-client.apache.connection-acquisition-timeout-ms" |
| + |
public static final java.lang.String |
-HTTP_CLIENT_TYPE |
-"http-client.type" |
+APACHE_CONNECTION_MAX_IDLE_TIME_MS |
+"http-client.apache.connection-max-idle-time-ms" |
| + |
public static final java.lang.String |
-HTTP_CLIENT_TYPE_APACHE |
-"apache" |
+APACHE_CONNECTION_TIME_TO_LIVE_MS |
+"http-client.apache.connection-time-to-live-ms" |
| + |
public static final java.lang.String |
-HTTP_CLIENT_TYPE_DEFAULT |
-"urlconnection" |
+APACHE_CONNECTION_TIMEOUT_MS |
+"http-client.apache.connection-timeout-ms" |
| + |
public static final java.lang.String |
-HTTP_CLIENT_TYPE_URLCONNECTION |
-"urlconnection" |
+APACHE_EXPECT_CONTINUE_ENABLED |
+"http-client.apache.expect-continue-enabled" |
| + |
public static final java.lang.String |
-HTTP_CLIENT_URLCONNECTION_CONNECTION_TIMEOUT_MS |
-"http-client.urlconnection.connection-timeout-ms" |
+APACHE_MAX_CONNECTIONS |
+"http-client.apache.max-connections" |
| + |
public static final java.lang.String |
-HTTP_CLIENT_URLCONNECTION_SOCKET_TIMEOUT_MS |
-"http-client.urlconnection.socket-timeout-ms" |
+APACHE_SOCKET_TIMEOUT_MS |
+"http-client.apache.socket-timeout-ms" |
| + |
public static final java.lang.String |
-LAKE_FORMATION_DB_NAME |
-"lakeformation.db-name" |
+APACHE_TCP_KEEP_ALIVE_ENABLED |
+"http-client.apache.tcp-keep-alive-enabled" |
| + |
public static final java.lang.String |
-LAKE_FORMATION_TABLE_NAME |
-"lakeformation.table-name" |
+APACHE_USE_IDLE_CONNECTION_REAPER_ENABLED |
+"http-client.apache.use-idle-connection-reaper-enabled" |
| + |
public static final java.lang.String |
-REST_ACCESS_KEY_ID |
-"rest.access-key-id" |
+CLIENT_TYPE |
+"http-client.type" |
| + |
public static final java.lang.String |
-REST_SECRET_ACCESS_KEY |
-"rest.secret-access-key" |
+CLIENT_TYPE_APACHE |
+"apache" |
| + |
public static final java.lang.String |
-REST_SESSION_TOKEN |
-"rest.session-token" |
+CLIENT_TYPE_DEFAULT |
+"apache" |
| + |
public static final java.lang.String |
-REST_SIGNER_REGION |
-"rest.signing-region" |
+CLIENT_TYPE_URLCONNECTION |
+"urlconnection" |
| + |
public static final java.lang.String |
-REST_SIGNING_NAME |
-"rest.signing-name" |
+URLCONNECTION_CONNECTION_TIMEOUT_MS |
+"http-client.urlconnection.connection-timeout-ms" |
| + |
public static final java.lang.String |
-REST_SIGNING_NAME_DEFAULT |
-"execute-api" |
+URLCONNECTION_SOCKET_TIMEOUT_MS |
+"http-client.urlconnection.socket-timeout-ms" |
-
-public static final java.lang.String |
-S3_ACCELERATION_ENABLED |
-"s3.acceleration-enabled" |
+
| Modifier and Type | +Constant Field | +Value | |||
|---|---|---|---|---|---|
-
-public static final boolean |
-S3_ACCELERATION_ENABLED_DEFAULT |
-false |
-|||
| + |
public static final java.lang.String |
-S3_ACCESS_POINTS_PREFIX |
-"s3.access-points." |
+LF_AUTHORIZED_CALLER |
+"LakeFormationAuthorizedCaller" |
+
| Modifier and Type | +Constant Field | +Value | |||
|---|---|---|---|---|---|
| + |
public static final java.lang.String |
-S3_CHECKSUM_ENABLED |
-"s3.checksum-enabled" |
+ACCELERATION_ENABLED |
+"s3.acceleration-enabled" |
| + |
public static final boolean |
-S3_CHECKSUM_ENABLED_DEFAULT |
+ACCELERATION_ENABLED_DEFAULT |
false |
|
| + |
public static final java.lang.String |
-S3_DELETE_ENABLED |
-"s3.delete-enabled" |
+ACCESS_KEY_ID |
+"s3.access-key-id" |
| + |
-public static final boolean |
-S3_DELETE_ENABLED_DEFAULT |
-true |
+ACCESS_POINTS_PREFIX |
+"s3.access-points." |
| + |
public static final java.lang.String |
-S3_DELETE_TAGS_PREFIX |
-"s3.delete.tags." |
+ACL |
+"s3.acl" |
| + |
public static final java.lang.String |
-S3_DUALSTACK_ENABLED |
-"s3.dualstack-enabled" |
+CHECKSUM_ENABLED |
+"s3.checksum-enabled" |
| + |
public static final boolean |
-S3_DUALSTACK_ENABLED_DEFAULT |
+CHECKSUM_ENABLED_DEFAULT |
false |
|
| + |
public static final java.lang.String |
-S3_PRELOAD_CLIENT_ENABLED |
-"s3.preload-client-enabled" |
+CLIENT_FACTORY |
+"s3.client-factory-impl" |
| + |
-public static final boolean |
-S3_PRELOAD_CLIENT_ENABLED_DEFAULT |
-false |
+DELETE_BATCH_SIZE |
+"s3.delete.batch-size" |
| + |
+
+public static final int |
+DELETE_BATCH_SIZE_DEFAULT |
+250 |
+||
+
+public static final int |
+DELETE_BATCH_SIZE_MAX |
+1000 |
+|||
public static final java.lang.String |
-S3_REMOTE_SIGNING_ENABLED |
-"s3.remote-signing-enabled" |
+DELETE_ENABLED |
+"s3.delete-enabled" |
|
| + |
public static final boolean |
-S3_REMOTE_SIGNING_ENABLED_DEFAULT |
-false |
+DELETE_ENABLED_DEFAULT |
+true |
| + |
public static final java.lang.String |
-S3_TAG_ICEBERG_NAMESPACE |
-"iceberg.namespace" |
+DELETE_TAGS_PREFIX |
+"s3.delete.tags." |
| + |
public static final java.lang.String |
-S3_TAG_ICEBERG_TABLE |
-"iceberg.table" |
+DELETE_THREADS |
+"s3.delete.num-threads" |
| + |
public static final java.lang.String |
-S3_USE_ARN_REGION_ENABLED |
-"s3.use-arn-region-enabled" |
+DUALSTACK_ENABLED |
+"s3.dualstack-enabled" |
| + |
public static final boolean |
-S3_USE_ARN_REGION_ENABLED_DEFAULT |
+DUALSTACK_ENABLED_DEFAULT |
false |
|
| + |
public static final java.lang.String |
-S3_WRITE_NAMESPACE_TAG_ENABLED |
-"s3.write.namespace-tag-enabled" |
+ENDPOINT |
+"s3.endpoint" |
| + |
-public static final boolean |
-S3_WRITE_NAMESPACE_TAG_ENABLED_DEFAULT |
-false |
+MULTIPART_SIZE |
+"s3.multipart.part-size-bytes" |
| + |
-public static final java.lang.String |
-S3_WRITE_TABLE_TAG_ENABLED |
-"s3.write.table-tag-enabled" |
+MULTIPART_SIZE_DEFAULT |
+33554432 |
| + |
-public static final boolean |
-S3_WRITE_TABLE_TAG_ENABLED_DEFAULT |
-false |
+MULTIPART_SIZE_MIN |
+5242880 |
| + |
public static final java.lang.String |
-S3_WRITE_TAGS_PREFIX |
-"s3.write.tags." |
+MULTIPART_THRESHOLD_FACTOR |
+"s3.multipart.threshold" |
| + |
-public static final java.lang.String |
-S3FILEIO_ACCESS_KEY_ID |
-"s3.access-key-id" |
+MULTIPART_THRESHOLD_FACTOR_DEFAULT |
+1.5 |
| + |
public static final java.lang.String |
-S3FILEIO_ACL |
-"s3.acl" |
+MULTIPART_UPLOAD_THREADS |
+"s3.multipart.num-threads" |
| + |
public static final java.lang.String |
-S3FILEIO_DELETE_BATCH_SIZE |
-"s3.delete.batch-size" |
+PATH_STYLE_ACCESS |
+"s3.path-style-access" |
| + |
-public static final int |
-S3FILEIO_DELETE_BATCH_SIZE_DEFAULT |
-250 |
+PATH_STYLE_ACCESS_DEFAULT |
+false |
| + |
-public static final int |
-S3FILEIO_DELETE_BATCH_SIZE_MAX |
-1000 |
+PRELOAD_CLIENT_ENABLED |
+"s3.preload-client-enabled" |
| + |
-public static final java.lang.String |
-S3FILEIO_DELETE_THREADS |
-"s3.delete.num-threads" |
+PRELOAD_CLIENT_ENABLED_DEFAULT |
+false |
| + |
public static final java.lang.String |
-S3FILEIO_ENDPOINT |
-"s3.endpoint" |
+REMOTE_SIGNING_ENABLED |
+"s3.remote-signing-enabled" |
| + |
-public static final java.lang.String |
-S3FILEIO_MULTIPART_SIZE |
-"s3.multipart.part-size-bytes" |
+REMOTE_SIGNING_ENABLED_DEFAULT |
+false |
| + |
-public static final int |
-S3FILEIO_MULTIPART_SIZE_DEFAULT |
-33554432 |
+S3_TAG_ICEBERG_NAMESPACE |
+"iceberg.namespace" |
| + |
-public static final int |
-S3FILEIO_MULTIPART_SIZE_MIN |
-5242880 |
+S3_TAG_ICEBERG_TABLE |
+"iceberg.table" |
| + |
public static final java.lang.String |
-S3FILEIO_MULTIPART_THRESHOLD_FACTOR |
-"s3.multipart.threshold" |
+SECRET_ACCESS_KEY |
+"s3.secret-access-key" |
| + |
-public static final double |
-S3FILEIO_MULTIPART_THRESHOLD_FACTOR_DEFAULT |
-1.5 |
+SESSION_TOKEN |
+"s3.session-token" |
| + |
public static final java.lang.String |
-S3FILEIO_MULTIPART_UPLOAD_THREADS |
-"s3.multipart.num-threads" |
+SSE_KEY |
+"s3.sse.key" |
| + |
public static final java.lang.String |
-S3FILEIO_PATH_STYLE_ACCESS |
-"s3.path-style-access" |
+SSE_MD5 |
+"s3.sse.md5" |
| + |
-public static final boolean |
-S3FILEIO_PATH_STYLE_ACCESS_DEFAULT |
-false |
+SSE_TYPE |
+"s3.sse.type" |
| + |
public static final java.lang.String |
-S3FILEIO_SECRET_ACCESS_KEY |
-"s3.secret-access-key" |
+SSE_TYPE_CUSTOM |
+"custom" |
| + |
public static final java.lang.String |
-S3FILEIO_SESSION_TOKEN |
-"s3.session-token" |
+SSE_TYPE_KMS |
+"kms" |
| + |
public static final java.lang.String |
-S3FILEIO_SSE_KEY |
-"s3.sse.key" |
+SSE_TYPE_NONE |
+"none" |
| + |
public static final java.lang.String |
-S3FILEIO_SSE_MD5 |
-"s3.sse.md5" |
+SSE_TYPE_S3 |
+"s3" |
| + |
public static final java.lang.String |
-S3FILEIO_SSE_TYPE |
-"s3.sse.type" |
+STAGING_DIRECTORY |
+"s3.staging-dir" |
| + |
public static final java.lang.String |
-S3FILEIO_SSE_TYPE_CUSTOM |
-"custom" |
+USE_ARN_REGION_ENABLED |
+"s3.use-arn-region-enabled" |
| + |
+
+public static final boolean |
+USE_ARN_REGION_ENABLED_DEFAULT |
+false |
+||
public static final java.lang.String |
-S3FILEIO_SSE_TYPE_KMS |
-"kms" |
+WRITE_NAMESPACE_TAG_ENABLED |
+"s3.write.namespace-tag-enabled" |
+|
+
+public static final boolean |
+WRITE_NAMESPACE_TAG_ENABLED_DEFAULT |
+false |
|||
| + |
public static final java.lang.String |
-S3FILEIO_SSE_TYPE_NONE |
-"none" |
+WRITE_STORAGE_CLASS |
+"s3.write.storage-class" |
| + |
public static final java.lang.String |
-S3FILEIO_SSE_TYPE_S3 |
-"s3" |
+WRITE_TABLE_TAG_ENABLED |
+"s3.write.table-tag-enabled" |
| + |
+
+public static final boolean |
+WRITE_TABLE_TAG_ENABLED_DEFAULT |
+false |
+||
public static final java.lang.String |
-S3FILEIO_STAGING_DIRECTORY |
-"s3.staging-dir" |
+WRITE_TAGS_PREFIX |
+"s3.write.tags." |
| Modifier and Type | Constant Field | @@ -3075,11 +3406,18 @@||||
|---|---|---|---|---|---|
| + |
public static final java.lang.String |
-LF_AUTHORIZED_CALLER |
-"LakeFormationAuthorizedCaller" |
+S3_SIGNER_ENDPOINT |
+"s3.signer.endpoint" |
+
+
+public static final java.lang.String |
+S3_SIGNER_URI |
+"s3.signer.uri" |
| Modifier and Type | Constant Field | @@ -3096,18 +3434,32 @@||||
|---|---|---|---|---|---|
| + |
public static final java.lang.String |
-S3_SIGNER_ENDPOINT |
-"s3.signer.endpoint" |
+ADLS_CONNECTION_STRING_PREFIX |
+"adls.connection-string." |
| + |
public static final java.lang.String |
-S3_SIGNER_URI |
-"s3.signer.uri" |
+ADLS_READ_BLOCK_SIZE |
+"adls.read.block-size-bytes" |
+
+
+public static final java.lang.String |
+ADLS_SAS_TOKEN_PREFIX |
+"adls.sas-token." |
+|||
+
+public static final java.lang.String |
+ADLS_WRITE_BLOCK_SIZE |
+"adls.write.block-size-bytes" |
public static final java.lang.StringICEBERG_METADATA_LOCATION"iceberg_metadata_location"| Modifier and Type | +Constant Field | +Value | +||
|---|---|---|---|---|
+
+public static final int |
+CIPHER_BLOCK_SIZE |
+1048604 |
+||
public static final java.lang.String |
-ICEBERG_METADATA_LOCATION |
-"iceberg_metadata_location" |
+GCM_STREAM_MAGIC_STRING |
+"AGS1" |
+
+
+public static final int |
+GCM_TAG_LENGTH |
+16 |
+||
+
+public static final int |
+NONCE_LENGTH |
+12 |
+||
+
+public static final int |
+PLAIN_BLOCK_SIZE |
+1048576 |
"limit"public static final java.lang.StringMAX_ALLOWED_PLANNING_FAILURES"max-allowed-planning-failures"public static final java.lang.StringMAX_PLANNING_SNAPSHOT_COUNT"max-planning-snapshot-count"public static final java.lang.StringMONITOR_INTERVAL"monitor-interval"public static final java.lang.StringSPLIT_FILE_OPEN_COST"split-file-open-cost"public static final java.lang.StringSPLIT_LOOKBACK"split-lookback"public static final java.lang.StringSPLIT_SIZE"split-size"public static final java.lang.StringSTARTING_STRATEGY"starting-strategy"public static final java.lang.String"gcs.decryption-key"public static final java.lang.StringGCS_DELETE_BATCH_SIZE"gcs.delete.batch-size"public static final intGCS_DELETE_BATCH_SIZE_DEFAULT50public static final java.lang.String"gcs.encryption-key"public static final java.lang.StringGCS_OAUTH2_TOKEN"gcs.oauth2.token"public static final java.lang.StringGCS_OAUTH2_TOKEN_EXPIRES_AT"gcs.oauth2.token-expires-at"public static final java.lang.StringKEEP_HIVE_STATS"iceberg.hive.keep.stats"public static final java.lang.StringLOCK_HIVE_ENABLED"iceberg.engine.hive.lock-enabled"NESSIE_COMMIT_ID_PROPERTY"nessie.commit.id"public static final java.lang.StringNESSIE_GC_NO_WARNING_PROPERTY"nessie.gc.no-warning"public static final java.lang.StringCLIENT_API_VERSION"nessie.client-api-version"public static final java.lang.String"file-open-cost"public static final java.lang.StringFILE_SCAN_TASK_SET_ID"file-scan-task-set-id"public static final java.lang.StringHANDLE_TIMESTAMP_WITHOUT_TIMEZONE"handle-timestamp-without-timezone"public static final java.lang.String"stream-from-timestamp"public static final java.lang.StringSTREAMING_MAX_FILES_PER_MICRO_BATCH"streaming-max-files-per-micro-batch"public static final java.lang.StringSTREAMING_MAX_ROWS_PER_MICRO_BATCH"streaming-max-rows-per-micro-batch"public static final java.lang.Stringpublic static final java.lang.StringADVISORY_PARTITION_SIZE"spark.sql.iceberg.advisory-partition-size"public static final java.lang.StringAGGREGATE_PUSH_DOWN_ENABLED"spark.sql.iceberg.aggregate-push-down.enabled"public static final booleanAGGREGATE_PUSH_DOWN_ENABLED_DEFAULTtruepublic static final java.lang.StringCHECK_NULLABILITY"spark.sql.iceberg.check-nullability"public static final booleanCHECK_NULLABILITY_DEFAULTtruepublic static final java.lang.StringCHECK_ORDERING"spark.sql.iceberg.check-ordering"public static final booleanCHECK_ORDERING_DEFAULTtruepublic static final java.lang.StringCOMPRESSION_CODEC"spark.sql.iceberg.compression-codec"public static final java.lang.StringDISTRIBUTION_MODE"spark.sql.iceberg.distribution-mode"COMPRESSION_LEVEL"spark.sql.iceberg.compression-level"public static final java.lang.StringHANDLE_TIMESTAMP_WITHOUT_TIMEZONE"spark.sql.iceberg.handle-timestamp-without-timezone"COMPRESSION_STRATEGY"spark.sql.iceberg.compression-strategy"public static final booleanHANDLE_TIMESTAMP_WITHOUT_TIMEZONE_DEFAULTfalsepublic static final java.lang.String
+DATA_PLANNING_MODE"spark.sql.iceberg.data-planning-mode"public static final java.lang.StringPRESERVE_DATA_GROUPING"spark.sql.iceberg.planning.preserve-data-grouping"DELETE_PLANNING_MODE"spark.sql.iceberg.delete-planning-mode"public static final booleanPRESERVE_DATA_GROUPING_DEFAULTfalsepublic static final java.lang.String
+DISTRIBUTION_MODE"spark.sql.iceberg.distribution-mode"public static final java.lang.StringUSE_TIMESTAMP_WITHOUT_TIME_ZONE_IN_NEW_TABLES"spark.sql.iceberg.use-timestamp-without-timezone-in-new-tables"PRESERVE_DATA_GROUPING"spark.sql.iceberg.planning.preserve-data-grouping"public static final booleanUSE_TIMESTAMP_WITHOUT_TIME_ZONE_IN_NEW_TABLES_DEFAULTPRESERVE_DATA_GROUPING_DEFAULTfalsepublic static final java.lang.StringADVISORY_PARTITION_SIZE"advisory-partition-size"public static final java.lang.StringCHECK_NULLABILITY"check-nullability"public static final java.lang.StringCHECK_ORDERING"check-ordering"public static final java.lang.StringCOMPRESSION_CODEC"compression-codec"public static final java.lang.StringCOMPRESSION_LEVEL"compression-level"public static final java.lang.StringCOMPRESSION_STRATEGY"compression-strategy""fanout-enabled"public static final java.lang.StringHANDLE_TIMESTAMP_WITHOUT_TIMEZONE"handle-timestamp-without-timezone"public static final java.lang.StringISOLATION_LEVEL"isolation-level"public static final java.lang.StringMERGE_SCHEMA"merge-schema"public static final booleanMERGE_SCHEMA_DEFAULTfalsepublic static final java.lang.StringOUTPUT_SPEC_ID"output-spec-id"| Modifier and Type | +Constant Field | +Value | +
|---|---|---|
+
+protected static final java.lang.String |
+FILE_PATH |
+"file_path" |
+
+
+protected static final java.lang.String |
+LAST_MODIFIED |
+"last_modified" |
+
+
+protected static final java.lang.String |
+MANIFEST |
+"Manifest" |
+
+
+protected static final java.lang.String |
+MANIFEST_LIST |
+"Manifest List" |
+
+
+protected static final java.lang.String |
+OTHERS |
+"Others" |
+
+
+protected static final java.lang.String |
+STATISTICS_FILES |
+"Statistics Files" |
+
| Modifier and Type | @@ -5465,25 +6018,6 @@
|---|
| Modifier and Type | -Constant Field | -Value | -
|---|---|---|
-
-public static final java.lang.String |
-COMPRESSION_FACTOR |
-"compression-factor" |
-
| Modifier and Type | Constant Field | @@ -5564,18 +6098,46 @@||||
|---|---|---|---|---|---|
| + |
-public static final java.lang.String |
-WORKER_THREAD_POOL_SIZE_PROP |
-"iceberg.worker.num-threads" |
+PRIMITIVE_BUFFER_SIZE |
+8 |
+
| Modifier and Type | +Constant Field | +Value | +
|---|---|---|
+
+public static final int |
+DEFAULT_VIEW_FORMAT_VERSION |
+1 |
+
+
+public static final int |
+SUPPORTED_VIEW_FORMAT_VERSION |
+1 |
| Modifier and Type | Constant Field | @@ -5583,17 +6145,22 @@|||
|---|---|---|---|---|
| + |
+
+public static final java.lang.String |
+VERSION_HISTORY_SIZE |
+"version.history.num-entries" |
+|
public static final int |
-PRIMITIVE_BUFFER_SIZE |
-8 |
+VERSION_HISTORY_SIZE_DEFAULT |
+10 |
| org.apache.iceberg.actions.RewriteStrategy
+ since 1.3.0, will be removed in 1.4.0; use
+FileRewriter instead. Note: This
+ can only be removed once Spark 3.2 isn't using this API anymore. |
+
| org.apache.iceberg.rest.requests.UpdateTableRequest.UpdateRequirement
+ will be removed in 1.5.0, use
+UpdateRequirement instead. |
+
SizeBasedFileRewriter instead.
+ Note: This can only be removed once Spark 3.2 isn't using this API anymore.ScanTaskSetManager insteadParquetWriterParquetWriterUpdateReqDeserializer instead.UpdateReqSerializer instead.SizeBasedFileRewriter instead.
+ Note: This can only be removed once Spark 3.2 isn't using this API anymore.SystemConfigs instead; will be removed in 2.0.0UpdateRequirementParser
+ instead.UpdateRequirements
+ instead.OAuth2Properties.TOKEN_REFRESH_ENABLED to control token
- refresh behavior.OAuth2Properties.TOKEN_REFRESH_ENABLED_DEFAULT to control
- default token refresh behavior.TableProperties.WRITE_DATA_LOCATION instead.WORKER_THREAD_POOL_SIZE instead;
+ will be removed in 2.0.0TableProperties.WRITE_DATA_LOCATION instead.Transforms.bucket(int) instead; will be removed in 2.0.0AwsClientProperties.applyClientCredentialConfigurations(AwsClientBuilder) insteadHTTPClient.builder(Map)Transforms.bucket(int) instead; will be removed in 2.0.0VectorizedSparkParquetReaders.buildReader(Schema, MessageType, Map,
- DeleteFilter) instead.UpdateRequirements.forUpdateTable(TableMetadata, List) instead.VectorizedSparkParquetReaders.buildReader(Schema, MessageType, Map,
- DeleteFilter) instead.UpdateRequirements.forCreateTable(List) instead.VectorizedSparkParquetReaders.buildReader(Schema, MessageType, Map,
- DeleteFilter) instead.UpdateRequirements.forReplaceTable(TableMetadata, List) instead.AwsClientProperties.clientRegion() insteadAwsProperties.applyS3EndpointConfigurations(S3ClientBuilder), AwsProperties.applyGlueEndpointConfigurations(GlueClientBuilder), or AwsProperties.applyDynamoDbEndpointConfigurations(DynamoDbClientBuilder) accordingly. It
+S3FileIOProperties.applyEndpointConfigurations(S3ClientBuilder), AwsProperties.applyGlueEndpointConfigurations(GlueClientBuilder), or AwsProperties.applyDynamoDbEndpointConfigurations(DynamoDbClientBuilder) accordingly. It
will be removed in 2.0.0AwsProperties.applyHttpClientConfigurations(AwsSyncClientBuilder). It will be removed in
- 2.0.0HttpClientProperties.applyHttpClientConfigurations(AwsSyncClientBuilder). It will be
+ removed in 2.0.0SparkReadConf.scanTaskSetId() insteadPartitioning.groupingKeyType(Schema, Collection) instead; will be removed in 1.3.0Transforms.hour() instead; will be removed in 2.0.0Transforms.hour() instead; will be removed in 2.0.0HttpClientProperties insteadTransforms.month() instead; will be removed in 2.0.0SparkTableUtil.loadMetadataTable(SparkSession, Table, MetadataTableType).Transforms.month() instead; will be removed in 2.0.0JdbcCatalog.setConf(Object)RESTSessionCatalog.setConf(Object)RESTCatalog.setConf(Object)AwsClientProperties.setClientRegion(String)
+ insteadAuthSession(Map, String, String,
- String, String) instead.BaseFileGroupRewriteResult#BaseFileGroupRewriteResult(FileGroupInfo, int, int, long)
- instead.S3FileIO, prefix used for bucket access point configuration.DataFile to the table.DataFile to the table.ApacheHttpClient.Builder.ApacheHttpClient.Builder.ApacheHttpClient.Builder.ApacheHttpClient.Builder.ApacheHttpClient.Builder.ApacheHttpClient.Builder.ApacheHttpClient.Builder.ApacheHttpClient.Builder.ApacheHttpClient.Builder.AwsClientProperties.applyClientCredentialConfigurations(AwsClientBuilder) insteadUpdateProperties operation.UpdateProperties operation.UpdateSchema operation.UpdateSchema operation.UpdateSchema operation.OAuth2Properties.TOKEN_REFRESH_ENABLED to control token
- refresh behavior.OAuth2Properties.TOKEN_REFRESH_ENABLED_DEFAULT to control
- default token refresh behavior.AuthSession(Map, String, String,
- String, String) instead.GenericRecord.BaseFileGroupRewriteResult#BaseFileGroupRewriteResult(FileGroupInfo, int, int, long)
- instead.Table implementation.SizeBasedFileRewriter instead.
+ Note: This can only be removed once Spark 3.2 isn't using this API anymore.bucket partition
@@ -2083,7 +2258,9 @@ HTTPClient.builder(Map)FlinkSink.Builder to export the data from generic input data stream into iceberg
@@ -2264,7 +2439,11 @@ partition spec builder for the given Schema.UpdateRequirements.forUpdateTable(TableMetadata, List) instead.sort order builder for the given Schema.UpdateRequirements.forCreateTable(List) instead.UpdateRequirements.forReplaceTable(TableMetadata, List) instead.VectorizedSparkParquetReaders.buildReader(Schema, MessageType, Map,
- DeleteFilter) instead.VectorizedSparkParquetReaders.buildReader(Schema, MessageType, Map,
- DeleteFilter) instead.VectorizedSparkParquetReaders.buildReader(Schema, MessageType, Map,
- DeleteFilter) instead.AssumeRoleAwsClientFactory.AwsClientFactories.defaultFactory() and other AWS client factory
+ classes to pass provider-specific properties.AwsClientFactory to customize AWS client configurations.DellClientFactory to customize Dell client configurations.CatalogProperties.URI, to compose the key of
+ the client pool cache.AwsClientFactories.DefaultAwsClientFactory and also
other client factory classes.SdkHttpClient implementation used by AwsClientFactory If set, all AWS clients will use this specified HTTP client.HttpClientProperties.CLIENT_TYPE, ApacheHttpClient will be used as the HTTP Client in AwsClientFactoryHttpClientProperties.CLIENT_TYPE, UrlConnectionHttpClient will be used as the HTTP
+ Client in AwsClientFactoryAwsClientProperties.clientRegion() insteadRowDataSerializer.copyRowData(RowData, RowData) method.Comparator for values produced by this term.RemoveCarryoverIterator and ComputeUpdateIterator
+ to remove carry-over rows and compute update rowsAwsProperties.applyS3EndpointConfigurations(S3ClientBuilder), AwsProperties.applyGlueEndpointConfigurations(GlueClientBuilder), or AwsProperties.applyDynamoDbEndpointConfigurations(DynamoDbClientBuilder) accordingly. It
+S3FileIOProperties.applyEndpointConfigurations(S3ClientBuilder), AwsProperties.applyGlueEndpointConfigurations(GlueClientBuilder), or AwsProperties.applyDynamoDbEndpointConfigurations(DynamoDbClientBuilder) accordingly. It
will be removed in 2.0.0AwsProperties.applyHttpClientConfigurations(AwsSyncClientBuilder). It will be removed in
- 2.0.0HttpClientProperties.applyHttpClientConfigurations(AwsSyncClientBuilder). It will be
+ removed in 2.0.0Type to a Flink type.Flink type to a Type.struct to a Schema with new field ids.struct to a Schema with new field ids.struct to a Type with new field ids.struct to a Schema based on the given schema.TableCommit instance to be applied for a single table with UpdateRequirements to be validated and MetadataUpdates that have been applied.PositionOutputStream to it.StructLike rows.PositionOutputStream to it.Table implementation that exposes a table's data files as rows.DataStatisticsCoordinatorManifestFile.EncryptedInputFile.encryptedInputFile() representing the raw encrypted bytes
@@ -5015,6 +5441,8 @@ Timer implementation that uses a Stopwatch instance internally to
@@ -5082,6 +5525,8 @@ S3FileIO deletes the object when io.delete() is called, default to true.S3FileIO to tag objects when deleting.Runtime.availableProcessors()DataFile from the table.DistributionMode that the flink sink will use.OAuth2Util.AuthSession with empty headers.OutputFile that writes raw bytes to the underlying file system,
@@ -6345,6 +6879,12 @@ addPartitionField
@@ -7035,6 +7581,8 @@ expression that is always false.FileScanTask into a CloseableIteratorSparkReadConf.scanTaskSetId() insteadScanTaskSetManager insteadDataStream.FlinkSink.Builder to export the data from input data stream with Rows into
@@ -9022,6 +9611,8 @@ ChildVectorT from arrow vector value.CodecFactory and modified to include the level in the cache key.field that partitions the given source fieldFieldVector.Partitioning.groupingKeyType(Schema, Collection) instead; will be removed in 1.3.0ApacheHttpClient.Builder.ApacheHttpClient.Builder.ApacheHttpClient.Builder.ApacheHttpClient.Builder.ApacheHttpClient.Builder.ApacheHttpClient.Builder.ApacheHttpClient.Builder.ApacheHttpClient.Builder.ApacheHttpClient.Builder.SdkHttpClient implementation used by AwsClientFactory If set, all AWS clients will use this specified HTTP client.AwsProperties.HTTP_CLIENT_TYPE, ApacheHttpClient will be used as the HTTP Client in AwsClientFactoryAwsProperties.HTTP_CLIENT_TYPE, UrlConnectionHttpClient will be used as the HTTP
- Client in AwsClientFactoryUrlConnectionHttpClient.Builder.UrlConnectionHttpClient.Builder.HttpClientProperties insteadS3FileIOProperties.CLIENT_FACTORY.FileIO that is allowed to have
+ associated ContentCache in memory at a time.FileIO that is allowed to have
associated ContentCache in memory at a time.FileIO supplier, which can potentially be used to dynamically refresh
+ the file IO instance when a table is refreshed.SparkTableUtil.loadMetadataTable(SparkSession, Table, MetadataTableType).ClassLoader used to lookup classes by name.FileIO implementation.MetricsReporter implementation that logs the MetricsReport to the log
@@ -13387,10 +14179,6 @@ FieldMetrics that this MetricsAwareDatumWriter keeps track of.MetricsReporter instances.RewriteDataFiles.TARGET_FILE_SIZE_BYTES.Runtime.availableProcessors()InputFile instance to read bytes from the file at the given path.OutputFile instance to write bytes to the file at the given path.SeekableInputStream for the underlying data fileRewriteDataFilesCommitManager.CommitService.rewritesPerCommit number of file groups are present in the queue.SerializableComparator.StructLike.PartitionSpec.true, requests to S3FileIO will use Path-Style, otherwise, Virtual Hosted-Style
+ will be used.MessageDecoder that constructs datum instances described by the readSchema.ReplaceViewVersion to replace the view's current version.CleanableFailure exception.UpdateReqDeserializer instead.UpdateReqSerializer instead.RewritePositionDeleteFiles.RewritePositionDeleteFiles from different platforms to handle
+ commits.RewritePositionDeleteFiles and the new files which have been written by the action.FileRewriter instead. Note: This
+ can only be removed once Spark 3.2 isn't using this API anymore.ManifestWriter, a rolling writer could produce multiple manifest files.Schema is not actually serializable, even though it
@@ -19084,14 +20324,8 @@ S3FileIO, prefix used for bucket access point configuration.S3FileIO deletes the object when io.delete() is called, default to true.S3FileIO to tag objects when deleting.AwsProperties.S3_WRITE_TAGS_PREFIX when AwsProperties.S3_WRITE_NAMESPACE_TAG_ENABLED is enabledAwsProperties.S3_WRITE_TAGS_PREFIX when AwsProperties.S3_WRITE_TABLE_TAG_ENABLED is enabledGlueCatalog to tag objects when writing.S3FileIOProperties.WRITE_TAGS_PREFIX when S3FileIOProperties.WRITE_NAMESPACE_TAG_ENABLED is enabledGlueCatalog to tag objects when writing.S3FileIOProperties.WRITE_TAGS_PREFIX when S3FileIOProperties.WRITE_TABLE_TAG_ENABLED
+ is enabledS3FileIO to tag objects when writing.Runtime.availableProcessors()Runtime.availableProcessors()true, requests to S3FileIO will use Path-Style, otherwise, Virtual Hosted-Style
- will be used.AwsProperties.S3FILEIO_SSE_TYPE_NONE.Expression with the same structure, but with data values replaced
by descriptions.Expression with the same structure, but with data values replaced
+ by descriptions.ScanMetrics that carries its results.schema for this view.schema for this view.Configuration object in a Serializable layer.AwsClientProperties.setClientRegion(String)
+ insteadJdbcCatalog.setConf(Object)RESTCatalog.setConf(Object)RESTSessionCatalog.setConf(Object)SortOrder.sort order for this table.SizeBasedFileRewriter instead.
+ Note: This can only be removed once Spark 3.2 isn't using this API anymore.Write class for rewriting position delete files from Spark.SerializableComparator
+ which could be used for ordering splits.S3FileIOProperties.SSE_TYPE_NONE.SystemConfigs instead; will be removed in 2.0.0UpdateRequirements to be
+ validated and MetadataUpdates that have been applied.InputFile for the location of this output file.CommitReport without writing a start/end object and is mainly used
@@ -23130,8 +24471,14 @@ PartitionSpec.RowType to a TableSchema.TimeUnit of the timer.UpdateViewProperties to update view properties.MetadataUpdateUpdateViewProperties to update view properties.UpdateRequirementParser
+ instead.UpdateRequirements
+ instead.UpdateRequirement instead.UrlConnectionHttpClient.Builder.UrlConnectionHttpClient.Builder.RowDelta.validateDataFilesExist(Iterable) have not been removed by a delete operation.WORKER_THREAD_POOL_SIZE instead;
+ will be removed in 2.0.0GlueCatalog to tag objects when writing.S3FileIO to tag objects' storage class when writing.GlueCatalog to tag objects when writing.S3FileIO to tag objects when writing.protected static java.util.List<java.lang.String>DELETE_SCAN_WITH_STATS_COLUMNS protected static booleanPLAN_SCANS_WITH_WORKER_POOL protected static java.util.List<java.lang.String>SCAN_COLUMNS protected static java.util.List<java.lang.String>SCAN_WITH_STATS_COLUMNS | Modifier and Type | Method and Description | @@ -227,36 +239,35 @@||
|---|---|---|---|
protected FileIO |
+io() |
+||
boolean |
isCaseSensitive()
Returns whether this scan is case-sensitive with respect to column names.
|
||
protected CloseableIterable<ManifestFile> |
manifests()
Returns an iterable of manifest files to explore for this all files metadata table scan
|
||
protected ThisT |
-newRefinedScan(TableOperations ignored,
- Table newTable,
- Schema newSchema,
- org.apache.iceberg.TableScanContext newContext)
-Deprecated.
-
+will be removed in 1.3.0; use newRefinedScan(Table, Schema, TableScanContext)
- instead.
- | ||
ThisT |
+metricsReporter(MetricsReporter reporter)
+Create a new scan that will report scan metrics to the provided reporter in addition to
+ reporters maintained by the scan.
|
||
protected TableScan |
newRefinedScan(Table table,
Schema schema,
org.apache.iceberg.TableScanContext context) |
||
ThisT |
option(java.lang.String property,
java.lang.String value)
@@ -264,115 +275,115 @@ Method Summarybehavior based on the incoming pair. |
||
protected java.util.Map<java.lang.String,java.lang.String> |
options() |
||
protected java.util.concurrent.ExecutorService |
planExecutor() |
||
CloseableIterable<FileScanTask> |
planFiles()
Plan tasks for this scan where each task reads a single file.
|
||
CloseableIterable<CombinedScanTask> |
planTasks()
Plan balanced task groups for this scan by splitting large and combining small tasks.
|
||
ThisT |
planWith(java.util.concurrent.ExecutorService executorService)
Create a new scan to use a particular executor to plan.
|
||
ThisT |
project(Schema projectedSchema)
Create a new scan from this with the schema as its projection.
|
||
protected CloseableIterable<ManifestFile> |
reachableManifests(org.apache.iceberg.relocated.com.google.common.base.Function<Snapshot,java.lang.Iterable<ManifestFile>> toManifests) |
||
protected Expression |
+residualFilter() |
+||
protected java.util.List<java.lang.String> |
scanColumns() |
||
Schema |
schema()
Returns this scan's projection
Schema. |
||
ThisT |
select(java.util.Collection<java.lang.String> columns)
Create a new scan from this that will read the given data columns.
|
||
protected boolean |
shouldIgnoreResiduals() |
||
protected boolean |
shouldPlanWithExecutor() |
||
protected boolean |
+shouldReturnColumnStats() |
+||
int |
splitLookback()
Returns the split lookback for this scan.
|
||
long |
splitOpenFileCost()
Returns the split open file cost for this scan.
|
||
Table |
table() |
||
protected TableOperations |
-tableOps()
-Deprecated.
-
-will be removed in 1.3.0; avoid using TableOperations for scans or use BaseTable
- |
-||
protected Schema |
tableSchema() |
||
protected MetadataTableType |
tableType()
Type of scan being performed, such as
MetadataTableType.ALL_DATA_FILES when scanning a
table's AllDataFilesTable. |
||
long |
targetSplitSize()
Returns the target split size for this scan.
|
||
TableScan |
useRef(java.lang.String ref)
Create a new
TableScan from this scan's configuration that will use the given
reference. |
||
TableScan |
useSnapshot(long scanSnapshotId)
Create a new TableScan from this scan's configuration that will use the given snapshot
@@ -385,7 +396,7 @@ Method SummaryMethods inherited from class org.apache.iceberg.SnapshotScan-scanMetrics, snapshot, snapshotId, toString
+scanMetrics, snapshot, snapshotId, toString, useSnapshotSchema
caseSensitive, filter, filter, ignoreResiduals, includeColumnStats, isCaseSensitive, metricsReporter, option, planWith, project, schema, select, select, splitLookback, splitOpenFileCost
Methods inherited from interface org.apache.iceberg.
+
+
+SCAN_COLUMNS+protected static final java.util.List<java.lang.String> SCAN_COLUMNS+ SCAN_WITH_STATS_COLUMNS+protected static final java.util.List<java.lang.String> SCAN_WITH_STATS_COLUMNS+
tableOps-@Deprecated -protected TableOperations tableOps()- Deprecated. will be removed in 1.3.0; avoid using TableOperations for scans or use BaseTable
+table+public Table table() shouldReturnColumnStats+protected boolean shouldReturnColumnStats()+ shouldPlanWithExecutor-protected boolean shouldPlanWithExecutor()+ residualFilter+protected Expression residualFilter() planExecutor-protected java.util.concurrent.ExecutorService planExecutor()+ shouldPlanWithExecutor+protected boolean shouldPlanWithExecutor() newRefinedScan-@Deprecated -protected ThisT newRefinedScan(TableOperations ignored, - Table newTable, - Schema newSchema, - org.apache.iceberg.TableScanContext newContext)- Deprecated. will be removed in 1.3.0; use newRefinedScan(Table, Schema, TableScanContext)
- instead.
+planExecutor+protected java.util.concurrent.ExecutorService planExecutor()
| ||
protected static boolean |
+PLAN_SCANS_WITH_WORKER_POOL |
+||
protected static java.util.List<java.lang.String> |
+SCAN_COLUMNS |
+||
protected static java.util.List<java.lang.String> |
+SCAN_WITH_STATS_COLUMNS |
+
| Modifier and Type | Method and Description | @@ -227,36 +239,35 @@||
|---|---|---|---|
protected FileIO |
+io() |
+||
boolean |
isCaseSensitive()
Returns whether this scan is case-sensitive with respect to column names.
|
||
protected CloseableIterable<ManifestFile> |
manifests()
Returns an iterable of manifest files to explore for this all files metadata table scan
|
||
protected ThisT |
-newRefinedScan(TableOperations ignored,
- Table newTable,
- Schema newSchema,
- org.apache.iceberg.TableScanContext newContext)
-Deprecated.
-
+will be removed in 1.3.0; use newRefinedScan(Table, Schema, TableScanContext)
- instead.
- | ||
ThisT |
+metricsReporter(MetricsReporter reporter)
+Create a new scan that will report scan metrics to the provided reporter in addition to
+ reporters maintained by the scan.
|
||
protected TableScan |
newRefinedScan(Table table,
Schema schema,
org.apache.iceberg.TableScanContext context) |
||
ThisT |
option(java.lang.String property,
java.lang.String value)
@@ -264,115 +275,115 @@ Method Summarybehavior based on the incoming pair. |
||
protected java.util.Map<java.lang.String,java.lang.String> |
options() |
||
protected java.util.concurrent.ExecutorService |
planExecutor() |
||
CloseableIterable<FileScanTask> |
planFiles()
Plan tasks for this scan where each task reads a single file.
|
||
CloseableIterable<CombinedScanTask> |
planTasks()
Plan balanced task groups for this scan by splitting large and combining small tasks.
|
||
ThisT |
planWith(java.util.concurrent.ExecutorService executorService)
Create a new scan to use a particular executor to plan.
|
||
ThisT |
project(Schema projectedSchema)
Create a new scan from this with the schema as its projection.
|
||
protected CloseableIterable<ManifestFile> |
reachableManifests(org.apache.iceberg.relocated.com.google.common.base.Function<Snapshot,java.lang.Iterable<ManifestFile>> toManifests) |
||
protected Expression |
+residualFilter() |
+||
protected java.util.List<java.lang.String> |
scanColumns() |
||
Schema |
schema()
Returns this scan's projection
Schema. |
||
ThisT |
select(java.util.Collection<java.lang.String> columns)
Create a new scan from this that will read the given data columns.
|
||
protected boolean |
shouldIgnoreResiduals() |
||
protected boolean |
shouldPlanWithExecutor() |
||
protected boolean |
+shouldReturnColumnStats() |
+||
int |
splitLookback()
Returns the split lookback for this scan.
|
||
long |
splitOpenFileCost()
Returns the split open file cost for this scan.
|
||
Table |
table() |
||
protected TableOperations |
-tableOps()
-Deprecated.
-
-will be removed in 1.3.0; avoid using TableOperations for scans or use BaseTable
- |
-||
protected Schema |
tableSchema() |
||
protected MetadataTableType |
tableType()
Type of scan being performed, such as
MetadataTableType.ALL_DATA_FILES when scanning a
table's AllDataFilesTable. |
||
long |
targetSplitSize()
Returns the target split size for this scan.
|
||
TableScan |
useRef(java.lang.String ref)
Create a new
TableScan from this scan's configuration that will use the given
reference. |
||
TableScan |
useSnapshot(long scanSnapshotId)
Create a new TableScan from this scan's configuration that will use the given snapshot
@@ -385,7 +396,7 @@ Method SummaryMethods inherited from class org.apache.iceberg.SnapshotScan-scanMetrics, snapshot, snapshotId, toString
+scanMetrics, snapshot, snapshotId, toString, useSnapshotSchema
caseSensitive, filter, filter, ignoreResiduals, includeColumnStats, isCaseSensitive, metricsReporter, option, planWith, project, schema, select, select, splitLookback, splitOpenFileCost
Methods inherited from interface org.apache.iceberg.
+
+
+SCAN_COLUMNS+protected static final java.util.List<java.lang.String> SCAN_COLUMNS+ SCAN_WITH_STATS_COLUMNS+protected static final java.util.List<java.lang.String> SCAN_WITH_STATS_COLUMNS+
tableOps-@Deprecated -protected TableOperations tableOps()- Deprecated. will be removed in 1.3.0; avoid using TableOperations for scans or use BaseTable
+table+public Table table() shouldReturnColumnStats+protected boolean shouldReturnColumnStats()+ shouldPlanWithExecutor-protected boolean shouldPlanWithExecutor()+ residualFilter+protected Expression residualFilter() planExecutor-protected java.util.concurrent.ExecutorService planExecutor()+ shouldPlanWithExecutor+protected boolean shouldPlanWithExecutor() newRefinedScan-@Deprecated -protected ThisT newRefinedScan(TableOperations ignored, - Table newTable, - Schema newSchema, - org.apache.iceberg.TableScanContext newContext)- Deprecated. will be removed in 1.3.0; use newRefinedScan(Table, Schema, TableScanContext)
- instead.
+planExecutor+protected java.util.concurrent.ExecutorService planExecutor()
| ||
protected static boolean |
+PLAN_SCANS_WITH_WORKER_POOL |
+||
protected static java.util.List<java.lang.String> |
+SCAN_COLUMNS |
+||
protected static java.util.List<java.lang.String> |
+SCAN_WITH_STATS_COLUMNS |
+
| Modifier and Type | Method and Description | @@ -227,36 +239,35 @@||
|---|---|---|---|
protected FileIO |
+io() |
+||
boolean |
isCaseSensitive()
Returns whether this scan is case-sensitive with respect to column names.
|
||
protected CloseableIterable<ManifestFile> |
manifests()
Returns an iterable of manifest files to explore for this all files metadata table scan
|
||
protected ThisT |
-newRefinedScan(TableOperations ignored,
- Table newTable,
- Schema newSchema,
- org.apache.iceberg.TableScanContext newContext)
-Deprecated.
-
+will be removed in 1.3.0; use newRefinedScan(Table, Schema, TableScanContext)
- instead.
- | ||
ThisT |
+metricsReporter(MetricsReporter reporter)
+Create a new scan that will report scan metrics to the provided reporter in addition to
+ reporters maintained by the scan.
|
||
protected TableScan |
newRefinedScan(Table table,
Schema schema,
org.apache.iceberg.TableScanContext context) |
||
ThisT |
option(java.lang.String property,
java.lang.String value)
@@ -264,115 +275,115 @@ Method Summarybehavior based on the incoming pair. |
||
protected java.util.Map<java.lang.String,java.lang.String> |
options() |
||
protected java.util.concurrent.ExecutorService |
planExecutor() |
||
CloseableIterable<FileScanTask> |
planFiles()
Plan tasks for this scan where each task reads a single file.
|
||
CloseableIterable<CombinedScanTask> |
planTasks()
Plan balanced task groups for this scan by splitting large and combining small tasks.
|
||
ThisT |
planWith(java.util.concurrent.ExecutorService executorService)
Create a new scan to use a particular executor to plan.
|
||
ThisT |
project(Schema projectedSchema)
Create a new scan from this with the schema as its projection.
|
||
protected CloseableIterable<ManifestFile> |
reachableManifests(org.apache.iceberg.relocated.com.google.common.base.Function<Snapshot,java.lang.Iterable<ManifestFile>> toManifests) |
||
protected Expression |
+residualFilter() |
+||
protected java.util.List<java.lang.String> |
scanColumns() |
||
Schema |
schema()
Returns this scan's projection
Schema. |
||
ThisT |
select(java.util.Collection<java.lang.String> columns)
Create a new scan from this that will read the given data columns.
|
||
protected boolean |
shouldIgnoreResiduals() |
||
protected boolean |
shouldPlanWithExecutor() |
||
protected boolean |
+shouldReturnColumnStats() |
+||
int |
splitLookback()
Returns the split lookback for this scan.
|
||
long |
splitOpenFileCost()
Returns the split open file cost for this scan.
|
||
Table |
table() |
||
protected TableOperations |
-tableOps()
-Deprecated.
-
-will be removed in 1.3.0; avoid using TableOperations for scans or use BaseTable
- |
-||
protected Schema |
tableSchema() |
||
protected MetadataTableType |
tableType()
Type of scan being performed, such as
MetadataTableType.ALL_DATA_FILES when scanning a
table's AllDataFilesTable. |
||
long |
targetSplitSize()
Returns the target split size for this scan.
|
||
TableScan |
useRef(java.lang.String ref)
Create a new
TableScan from this scan's configuration that will use the given
reference. |
||
TableScan |
useSnapshot(long scanSnapshotId)
Create a new TableScan from this scan's configuration that will use the given snapshot
@@ -385,7 +396,7 @@ Method SummaryMethods inherited from class org.apache.iceberg.SnapshotScan-scanMetrics, snapshot, snapshotId, toString
+scanMetrics, snapshot, snapshotId, toString, useSnapshotSchema
caseSensitive, filter, filter, ignoreResiduals, includeColumnStats, isCaseSensitive, metricsReporter, option, planWith, project, schema, select, select, splitLookback, splitOpenFileCost
Methods inherited from interface org.apache.iceberg.
+
+
+SCAN_COLUMNS+protected static final java.util.List<java.lang.String> SCAN_COLUMNS+ SCAN_WITH_STATS_COLUMNS+protected static final java.util.List<java.lang.String> SCAN_WITH_STATS_COLUMNS+
tableOps-@Deprecated -protected TableOperations tableOps()- Deprecated. will be removed in 1.3.0; avoid using TableOperations for scans or use BaseTable
+table+public Table table() shouldReturnColumnStats+protected boolean shouldReturnColumnStats()+ shouldPlanWithExecutor-protected boolean shouldPlanWithExecutor()+ residualFilter+protected Expression residualFilter() planExecutor-protected java.util.concurrent.ExecutorService planExecutor()+ shouldPlanWithExecutor+protected boolean shouldPlanWithExecutor() newRefinedScan-@Deprecated -protected ThisT newRefinedScan(TableOperations ignored, - Table newTable, - Schema newSchema, - org.apache.iceberg.TableScanContext newContext)- Deprecated. will be removed in 1.3.0; use newRefinedScan(Table, Schema, TableScanContext)
- instead.
+planExecutor+protected java.util.concurrent.ExecutorService planExecutor()
| ||
protected static boolean |
+PLAN_SCANS_WITH_WORKER_POOL |
+||
protected static java.util.List<java.lang.String> |
+SCAN_COLUMNS |
+||
protected static java.util.List<java.lang.String> |
+SCAN_WITH_STATS_COLUMNS |
+
| Modifier and Type | Method and Description | @@ -227,30 +239,29 @@||
|---|---|---|---|
protected FileIO |
+io() |
+||
boolean |
isCaseSensitive()
Returns whether this scan is case-sensitive with respect to column names.
|
||
protected ThisT |
-newRefinedScan(TableOperations ignored,
- Table newTable,
- Schema newSchema,
- org.apache.iceberg.TableScanContext newContext)
-Deprecated.
-
+will be removed in 1.3.0; use newRefinedScan(Table, Schema, TableScanContext)
- instead.
- | ||
ThisT |
+metricsReporter(MetricsReporter reporter)
+Create a new scan that will report scan metrics to the provided reporter in addition to
+ reporters maintained by the scan.
|
||
protected TableScan |
newRefinedScan(Table table,
Schema schema,
org.apache.iceberg.TableScanContext context) |
||
ThisT |
option(java.lang.String property,
java.lang.String value)
@@ -258,115 +269,115 @@ Method Summarybehavior based on the incoming pair. |
||
protected java.util.Map<java.lang.String,java.lang.String> |
options() |
||
protected java.util.concurrent.ExecutorService |
planExecutor() |
||
CloseableIterable<FileScanTask> |
planFiles()
Plan tasks for this scan where each task reads a single file.
|
||
CloseableIterable<CombinedScanTask> |
planTasks()
Plan balanced task groups for this scan by splitting large and combining small tasks.
|
||
ThisT |
planWith(java.util.concurrent.ExecutorService executorService)
Create a new scan to use a particular executor to plan.
|
||
ThisT |
project(Schema projectedSchema)
Create a new scan from this with the schema as its projection.
|
||
protected CloseableIterable<ManifestFile> |
reachableManifests(org.apache.iceberg.relocated.com.google.common.base.Function<Snapshot,java.lang.Iterable<ManifestFile>> toManifests) |
||
protected Expression |
+residualFilter() |
+||
protected java.util.List<java.lang.String> |
scanColumns() |
||
Schema |
schema()
Returns this scan's projection
Schema. |
||
ThisT |
select(java.util.Collection<java.lang.String> columns)
Create a new scan from this that will read the given data columns.
|
||
protected boolean |
shouldIgnoreResiduals() |
||
protected boolean |
shouldPlanWithExecutor() |
||
protected boolean |
+shouldReturnColumnStats() |
+||
int |
splitLookback()
Returns the split lookback for this scan.
|
||
long |
splitOpenFileCost()
Returns the split open file cost for this scan.
|
||
Table |
table() |
||
protected TableOperations |
-tableOps()
-Deprecated.
-
-will be removed in 1.3.0; avoid using TableOperations for scans or use BaseTable
- |
-||
protected Schema |
tableSchema() |
||
protected MetadataTableType |
tableType()
Type of scan being performed, such as
MetadataTableType.ALL_DATA_FILES when scanning a
table's AllDataFilesTable. |
||
long |
targetSplitSize()
Returns the target split size for this scan.
|
||
TableScan |
useRef(java.lang.String ref)
Create a new
TableScan from this scan's configuration that will use the given
reference. |
||
TableScan |
useSnapshot(long scanSnapshotId)
Create a new TableScan from this scan's configuration that will use the given snapshot
@@ -379,7 +390,7 @@ Method SummaryMethods inherited from class org.apache.iceberg.SnapshotScan-scanMetrics, snapshot, snapshotId, toString
+scanMetrics, snapshot, snapshotId, toString, useSnapshotSchema
caseSensitive, filter, filter, ignoreResiduals, includeColumnStats, isCaseSensitive, metricsReporter, option, planWith, project, schema, select, select, splitLookback, splitOpenFileCost
Methods inherited from interface org.apache.iceberg.
+
+
+SCAN_COLUMNS+protected static final java.util.List<java.lang.String> SCAN_COLUMNS+ SCAN_WITH_STATS_COLUMNS+protected static final java.util.List<java.lang.String> SCAN_WITH_STATS_COLUMNS+
tableOps-@Deprecated -protected TableOperations tableOps()- Deprecated. will be removed in 1.3.0; avoid using TableOperations for scans or use BaseTable
+table+public Table table() shouldReturnColumnStats+protected boolean shouldReturnColumnStats()+ shouldPlanWithExecutor-protected boolean shouldPlanWithExecutor()+ residualFilter+protected Expression residualFilter() planExecutor-protected java.util.concurrent.ExecutorService planExecutor()+ shouldPlanWithExecutor+protected boolean shouldPlanWithExecutor() newRefinedScan-@Deprecated -protected ThisT newRefinedScan(TableOperations ignored, - Table newTable, - Schema newSchema, - org.apache.iceberg.TableScanContext newContext)- Deprecated. will be removed in 1.3.0; use newRefinedScan(Table, Schema, TableScanContext)
- instead.
+planExecutor+protected java.util.concurrent.ExecutorService planExecutor()
| ||
long |
+estimatedRowsCount()
+The estimated number of rows produced by this scan task.
+ |
+||
java.util.Collection<FileScanTask> |
files()
Return the
tasks in this combined task. |
||
int |
+filesCount()
+The number of files that will be opened by this scan task.
+ |
+||
long |
+sizeBytes()
+The number of bytes that should be read by this scan task.
+ |
+||
java.lang.String |
toString() |
||
long |
+estimatedRowsCount()
+The estimated number of rows produced by this scan task.
+ |
+||
F |
file()
The
file to scan. |
||
int |
+filesCount()
+The number of files that will be opened by this scan task.
+ |
+||
long |
length()
The number of bytes to scan from the
ContentScanTask.start() position in the file. |
||
protected FileScanTask |
newSplitTask(FileScanTask parentTask,
long offset,
long length) |
||
Expression |
residual()
Returns the residual expression that should be applied to rows in this file scan.
|
||
Schema |
+schema()
+Return the schema for this file scan task.
+ |
+||
protected FileScanTask |
self() |
||
long |
+sizeBytes()
+The number of bytes that should be read by this scan task.
+ |
+||
PartitionSpec |
spec()
Returns the spec of the partition for this scan task
|
||
java.lang.Iterable<ThisT> |
split(long targetSplitSize)
Attempts to split this scan task into several smaller scan tasks, each close to
splitSize size. |
||
long |
start()
The starting position of this scan range in the file.
|
||
java.lang.String |
toString() |
||
protected |
-BaseMetadataTable(TableOperations ignored,
- Table table,
- java.lang.String name)
-Deprecated.
-
-will be removed in 1.3.0; use BaseMetadataTable(Table, String) instead.
- |
-||
protected |
BaseMetadataTable(Table table,
java.lang.String name) |
TableOperations |
operations()
Deprecated.
-
will be removed in 2.0.0; do not use metadata table TableOperations
+will be removed in 1.4.0; do not use metadata table TableOperations
|
@@ -432,19 +422,6 @@
| Modifier and Type | Method and Description | @@ -166,23 +166,22 @@||
|---|---|---|---|
protected void |
-add(ManifestFile manifest)
-Add all files in a manifest to the new snapshot.
+ | add(DeleteFile file,
+ long dataSequenceNumber)
+Add a delete file to the new snapshot.
|
|
protected org.apache.iceberg.DeleteFileIndex |
-addedDeleteFiles(TableMetadata base,
- java.lang.Long startingSnapshotId,
- Expression dataFilter,
- PartitionSet partitionSet)
-Deprecated.
-
+will be removed in 1.3.0; use
-MergingSnapshotProducer.addedDeleteFiles(TableMetadata, Long, Expression, PartitionSet,
- Snapshot) instead | protected void |
+add(ManifestFile manifest)
+Add all files in a manifest to the new snapshot.
|
protected java.util.List<DataFile> |
+addedDataFiles() |
+||
protected org.apache.iceberg.DeleteFileIndex |
addedDeleteFiles(TableMetadata base,
java.lang.Long startingSnapshotId,
@@ -192,10 +191,6 @@ |
||
protected java.util.List<DataFile> |
-addedFiles() |
-||
OverwriteFiles |
addFile(DataFile file)
@@ -203,239 +198,252 @@ Method Summary |
||
protected boolean |
+addsDataFiles() |
+||
protected boolean |
+addsDeleteFiles() |
+||
Snapshot |
apply()
Apply the pending changes and return the uncommitted changes for validation.
|
||
java.util.List<ManifestFile> |
apply(TableMetadata base,
Snapshot snapshot)
Apply the update's changes to the given metadata and snapshot.
|
||
ThisT |
caseSensitive(boolean isCaseSensitive) |
||
protected void |
cleanAll() |
||
protected void |
cleanUncommitted(java.util.Set<ManifestFile> committed)
Clean up any uncommitted manifests that were created.
|
||
void |
commit()
Apply the pending changes and commit.
|
||
protected CommitMetrics |
commitMetrics() |
||
OverwriteFiles |
conflictDetectionFilter(Expression newConflictDetectionFilter)
Sets a conflict detection filter used to validate concurrently added data and delete files.
|
||
protected TableMetadata |
current() |
||
protected PartitionSpec |
dataSpec() |
||
protected void |
delete(java.lang.CharSequence path)
Add a specific data path to be deleted in the new snapshot.
|
||
protected void |
delete(DataFile file)
Add a specific data file to be deleted in the new snapshot.
|
||
protected void |
delete(DeleteFile file)
Add a specific delete file to be deleted in the new snapshot.
|
||
protected void |
deleteByRowFilter(Expression expr)
Add a filter to match files to delete.
|
||
OverwriteFiles |
deleteFile(DataFile file)
Delete a
DataFile from the table. |
||
protected void |
deleteFile(java.lang.String path) |
||
protected boolean |
+deletesDataFiles() |
+||
protected boolean |
+deletesDeleteFiles() |
+||
ThisT |
deleteWith(java.util.function.Consumer<java.lang.String> deleteCallback)
Set a callback to delete files instead of the table's default.
|
||
protected void |
dropPartition(int specId,
StructLike partition)
Add a partition tuple to drop from the table during the delete phase.
|
||
protected void |
failAnyDelete() |
||
protected void |
failMissingDeletePaths() |
||
protected boolean |
isCaseSensitive() |
||
protected OutputFile |
manifestListPath() |
||
protected ManifestReader<DeleteFile> |
newDeleteManifestReader(ManifestFile manifest) |
||
protected ManifestWriter<DeleteFile> |
newDeleteManifestWriter(PartitionSpec spec) |
||
protected OutputFile |
newManifestOutput() |
||
protected ManifestReader<DataFile> |
newManifestReader(ManifestFile manifest) |
||
protected ManifestWriter<DataFile> |
newManifestWriter(PartitionSpec spec) |
||
protected RollingManifestWriter<DeleteFile> |
+newRollingDeleteManifestWriter(PartitionSpec spec) |
+||
protected RollingManifestWriter<DataFile> |
+newRollingManifestWriter(PartitionSpec spec) |
+||
protected java.lang.String |
operation()
A string that describes the action that produced the new snapshot.
|
||
OverwriteFiles |
overwriteByRowFilter(Expression expr)
Delete files that match an
Expression on data rows from the table. |
||
protected TableMetadata |
refresh() |
||
protected ThisT |
reportWith(MetricsReporter newReporter) |
||
protected Expression |
rowFilter() |
||
ThisT |
scanManifestsWith(java.util.concurrent.ExecutorService executorService)
Use a particular executor to scan manifests.
|
||
protected OverwriteFiles |
self() |
||
ThisT |
set(java.lang.String property,
java.lang.String value)
Set a summary property in the snapshot produced by this update.
|
||
protected void |
-setNewFilesSequenceNumber(long sequenceNumber) |
+setNewDataFilesDataSequenceNumber(long sequenceNumber) |
|
protected long |
snapshotId() |
||
ThisT |
stageOnly()
Called to stage a snapshot in table metadata, but not update the current snapshot id.
|
||
protected java.util.Map<java.lang.String,java.lang.String> |
summary() |
||
protected java.lang.String |
targetBranch() |
||
protected void |
targetBranch(java.lang.String branch)
-* A setter for the target branch on which snapshot producer operation should be performed
+A setter for the target branch on which snapshot producer operation should be performed
|
||
BaseOverwriteFiles |
toBranch(java.lang.String branch)
Perform operations on a particular branch
|
||
java.lang.Object |
updateEvent()
Generates update event to notify about metadata changes
|
||
protected void |
validate(TableMetadata base,
Snapshot parent)
Validate the current metadata.
|
||
protected void |
-validateAddedDataFiles(TableMetadata base,
- java.lang.Long startingSnapshotId,
- Expression conflictDetectionFilter)
-Deprecated.
-
-will be removed in 1.3.0; use
-MergingSnapshotProducer.validateAddedDataFiles(TableMetadata, Long, Expression, Snapshot)
- instead |
-||
protected void |
validateAddedDataFiles(TableMetadata base,
java.lang.Long startingSnapshotId,
@@ -445,18 +453,7 @@ |
||
protected void |
-validateAddedDataFiles(TableMetadata base,
- java.lang.Long startingSnapshotId,
- PartitionSet partitionSet)
-Deprecated.
-
-will be removed in 1.3.0; use
-MergingSnapshotProducer.validateAddedDataFiles(TableMetadata, Long, PartitionSet,
- Snapshot) instead |
-||
protected void |
validateAddedDataFiles(TableMetadata base,
java.lang.Long startingSnapshotId,
@@ -466,23 +463,13 @@ |
||
OverwriteFiles |
validateAddedFilesMatchOverwriteFilter()
Signal that each file added to the table must match the overwrite expression.
|
||
protected void |
-validateDataFilesExist(TableMetadata base,
- java.lang.Long startingSnapshotId,
- CharSequenceSet requiredDataFiles,
- boolean skipDeletes,
- Expression conflictDetectionFilter)
-Deprecated.
- |
-||
protected void |
validateDataFilesExist(TableMetadata base,
java.lang.Long startingSnapshotId,
@@ -491,18 +478,7 @@ |
||
protected void |
-validateDeletedDataFiles(TableMetadata base,
- java.lang.Long startingSnapshotId,
- Expression dataFilter)
-Deprecated.
-
-will be removed in 1.3.0; use
-MergingSnapshotProducer.validateDeletedDataFiles(TableMetadata, Long, Expression,
- Snapshot) instead |
-||
protected void |
validateDeletedDataFiles(TableMetadata base,
java.lang.Long startingSnapshotId,
@@ -512,18 +488,7 @@ |
||
protected void |
-validateDeletedDataFiles(TableMetadata base,
- java.lang.Long startingSnapshotId,
- PartitionSet partitionSet)
-Deprecated.
-
-will be removed in 1.3.0; use
-MergingSnapshotProducer.validateNoNewDeleteFiles(TableMetadata, Long, PartitionSet,
- Snapshot) instead |
-||
protected void |
validateDeletedDataFiles(TableMetadata base,
java.lang.Long startingSnapshotId,
@@ -533,35 +498,25 @@ |
||
OverwriteFiles |
validateFromSnapshot(long snapshotId)
Set the snapshot ID used in any reads for this operation.
|
||
OverwriteFiles |
validateNoConflictingData()
Enables validation that data added concurrently does not conflict with this commit's operation.
|
||
OverwriteFiles |
validateNoConflictingDeletes()
Enables validation that deletes that happened concurrently do not conflict with this commit's
operation.
|
||
protected void |
-validateNoNewDeleteFiles(TableMetadata base,
- java.lang.Long startingSnapshotId,
- Expression dataFilter)
-Deprecated.
-
-will be removed in 1.3.0; use
-MergingSnapshotProducer.validateNoNewDeleteFiles(org.apache.iceberg.TableMetadata, java.lang.Long, org.apache.iceberg.expressions.Expression) instead |
-||
protected void |
validateNoNewDeleteFiles(TableMetadata base,
@@ -574,17 +529,6 @@ | ||
protected void |
-validateNoNewDeleteFiles(TableMetadata base,
- java.lang.Long startingSnapshotId,
- PartitionSet partitionSet)
-Deprecated.
-
-will be removed in 1.3.0; use
-MergingSnapshotProducer.validateNoNewDeleteFiles(TableMetadata, Long, PartitionSet,
- Snapshot) instead |
-||
protected void |
validateNoNewDeleteFiles(TableMetadata base,
java.lang.Long startingSnapshotId,
PartitionSet partitionSet,
@@ -593,18 +537,7 @@ |
||
protected void |
-validateNoNewDeletesForDataFiles(TableMetadata base,
- java.lang.Long startingSnapshotId,
- Expression dataFilter,
- java.lang.Iterable<DataFile> dataFiles)
-Deprecated.
-
-will be removed in 1.3.0; use
-MergingSnapshotProducer.validateNoNewDeletesForDataFiles(org.apache.iceberg.TableMetadata, java.lang.Long, java.lang.Iterable<org.apache.iceberg.DataFile>, org.apache.iceberg.Snapshot) instead |
-||
protected void |
validateNoNewDeletesForDataFiles(TableMetadata base,
java.lang.Long startingSnapshotId,
@@ -615,17 +548,7 @@ |
||
protected void |
-validateNoNewDeletesForDataFiles(TableMetadata base,
- java.lang.Long startingSnapshotId,
- java.lang.Iterable<DataFile> dataFiles)
-Deprecated.
-
-will be removed in 1.3.0; use
-MergingSnapshotProducer.validateNoNewDeletesForDataFiles(org.apache.iceberg.TableMetadata, java.lang.Long, java.lang.Iterable<org.apache.iceberg.DataFile>, org.apache.iceberg.Snapshot) instead |
-||
protected void |
validateNoNewDeletesForDataFiles(TableMetadata base,
java.lang.Long startingSnapshotId,
@@ -635,7 +558,7 @@ |
||
protected java.util.concurrent.ExecutorService |
workerPool() |
| Modifier and Type | Method and Description | @@ -146,23 +146,22 @@||
|---|---|---|---|
protected void |
-add(ManifestFile manifest)
-Add all files in a manifest to the new snapshot.
+ | add(DeleteFile file,
+ long dataSequenceNumber)
+Add a delete file to the new snapshot.
|
|
protected org.apache.iceberg.DeleteFileIndex |
-addedDeleteFiles(TableMetadata base,
- java.lang.Long startingSnapshotId,
- Expression dataFilter,
- PartitionSet partitionSet)
-Deprecated.
-
+will be removed in 1.3.0; use
-MergingSnapshotProducer.addedDeleteFiles(TableMetadata, Long, Expression, PartitionSet,
- Snapshot) instead | protected void |
+add(ManifestFile manifest)
+Add all files in a manifest to the new snapshot.
|
protected java.util.List<DataFile> |
+addedDataFiles() |
+||
protected org.apache.iceberg.DeleteFileIndex |
addedDeleteFiles(TableMetadata base,
java.lang.Long startingSnapshotId,
@@ -172,10 +171,6 @@ |
||
protected java.util.List<DataFile> |
-addedFiles() |
-||
ReplacePartitions |
addFile(DataFile file)
@@ -183,221 +178,234 @@ Method Summary |
||
protected boolean |
+addsDataFiles() |
+||
protected boolean |
+addsDeleteFiles() |
+||
Snapshot |
apply()
Apply the pending changes and return the uncommitted changes for validation.
|
||
java.util.List<ManifestFile> |
apply(TableMetadata base,
Snapshot snapshot)
Apply the update's changes to the given metadata and snapshot.
|
||
ThisT |
caseSensitive(boolean isCaseSensitive) |
||
protected void |
cleanAll() |
||
protected void |
cleanUncommitted(java.util.Set<ManifestFile> committed)
Clean up any uncommitted manifests that were created.
|
||
void |
commit()
Apply the pending changes and commit.
|
||
protected CommitMetrics |
commitMetrics() |
||
protected TableMetadata |
current() |
||
protected PartitionSpec |
dataSpec() |
||
protected void |
delete(java.lang.CharSequence path)
Add a specific data path to be deleted in the new snapshot.
|
||
protected void |
delete(DataFile file)
Add a specific data file to be deleted in the new snapshot.
|
||
protected void |
delete(DeleteFile file)
Add a specific delete file to be deleted in the new snapshot.
|
||
protected void |
deleteByRowFilter(Expression expr)
Add a filter to match files to delete.
|
||
protected void |
deleteFile(java.lang.String path) |
||
protected boolean |
+deletesDataFiles() |
+||
protected boolean |
+deletesDeleteFiles() |
+||
ThisT |
deleteWith(java.util.function.Consumer<java.lang.String> deleteCallback)
Set a callback to delete files instead of the table's default.
|
||
protected void |
dropPartition(int specId,
StructLike partition)
Add a partition tuple to drop from the table during the delete phase.
|
||
protected void |
failAnyDelete() |
||
protected void |
failMissingDeletePaths() |
||
protected boolean |
isCaseSensitive() |
||
protected OutputFile |
manifestListPath() |
||
protected ManifestReader<DeleteFile> |
newDeleteManifestReader(ManifestFile manifest) |
||
protected ManifestWriter<DeleteFile> |
newDeleteManifestWriter(PartitionSpec spec) |
||
protected OutputFile |
newManifestOutput() |
||
protected ManifestReader<DataFile> |
newManifestReader(ManifestFile manifest) |
||
protected ManifestWriter<DataFile> |
newManifestWriter(PartitionSpec spec) |
||
protected RollingManifestWriter<DeleteFile> |
+newRollingDeleteManifestWriter(PartitionSpec spec) |
+||
protected RollingManifestWriter<DataFile> |
+newRollingManifestWriter(PartitionSpec spec) |
+||
protected java.lang.String |
operation()
A string that describes the action that produced the new snapshot.
|
||
protected TableMetadata |
refresh() |
||
protected ThisT |
reportWith(MetricsReporter newReporter) |
||
protected Expression |
rowFilter() |
||
ThisT |
scanManifestsWith(java.util.concurrent.ExecutorService executorService)
Use a particular executor to scan manifests.
|
||
protected ReplacePartitions |
self() |
||
ThisT |
set(java.lang.String property,
java.lang.String value)
Set a summary property in the snapshot produced by this update.
|
||
protected void |
-setNewFilesSequenceNumber(long sequenceNumber) |
+setNewDataFilesDataSequenceNumber(long sequenceNumber) |
|
protected long |
snapshotId() |
||
ThisT |
stageOnly()
Called to stage a snapshot in table metadata, but not update the current snapshot id.
|
||
protected java.util.Map<java.lang.String,java.lang.String> |
summary() |
||
protected java.lang.String |
targetBranch() |
||
protected void |
targetBranch(java.lang.String branch)
-* A setter for the target branch on which snapshot producer operation should be performed
+A setter for the target branch on which snapshot producer operation should be performed
|
||
BaseReplacePartitions |
toBranch(java.lang.String branch)
Perform operations on a particular branch
|
||
java.lang.Object |
updateEvent()
Generates update event to notify about metadata changes
|
||
void |
validate(TableMetadata currentMetadata,
Snapshot parent)
Validate the current metadata.
|
||
protected void |
-validateAddedDataFiles(TableMetadata base,
- java.lang.Long startingSnapshotId,
- Expression conflictDetectionFilter)
-Deprecated.
-
-will be removed in 1.3.0; use
-MergingSnapshotProducer.validateAddedDataFiles(TableMetadata, Long, Expression, Snapshot)
- instead |
-||
protected void |
validateAddedDataFiles(TableMetadata base,
java.lang.Long startingSnapshotId,
@@ -407,18 +415,7 @@ |
||
protected void |
-validateAddedDataFiles(TableMetadata base,
- java.lang.Long startingSnapshotId,
- PartitionSet partitionSet)
-Deprecated.
-
-will be removed in 1.3.0; use
-MergingSnapshotProducer.validateAddedDataFiles(TableMetadata, Long, PartitionSet,
- Snapshot) instead |
-||
protected void |
validateAddedDataFiles(TableMetadata base,
java.lang.Long startingSnapshotId,
@@ -428,23 +425,13 @@ |
||
ReplacePartitions |
validateAppendOnly()
Validate that no partitions will be replaced and the operation is append-only.
|
||
protected void |
-validateDataFilesExist(TableMetadata base,
- java.lang.Long startingSnapshotId,
- CharSequenceSet requiredDataFiles,
- boolean skipDeletes,
- Expression conflictDetectionFilter)
-Deprecated.
- |
-||
protected void |
validateDataFilesExist(TableMetadata base,
java.lang.Long startingSnapshotId,
@@ -453,18 +440,7 @@ |
||
protected void |
-validateDeletedDataFiles(TableMetadata base,
- java.lang.Long startingSnapshotId,
- Expression dataFilter)
-Deprecated.
-
-will be removed in 1.3.0; use
-MergingSnapshotProducer.validateDeletedDataFiles(TableMetadata, Long, Expression,
- Snapshot) instead |
-||
protected void |
validateDeletedDataFiles(TableMetadata base,
java.lang.Long startingSnapshotId,
@@ -474,18 +450,7 @@ |
||
protected void |
-validateDeletedDataFiles(TableMetadata base,
- java.lang.Long startingSnapshotId,
- PartitionSet partitionSet)
-Deprecated.
-
-will be removed in 1.3.0; use
-MergingSnapshotProducer.validateNoNewDeleteFiles(TableMetadata, Long, PartitionSet,
- Snapshot) instead |
-||
protected void |
validateDeletedDataFiles(TableMetadata base,
java.lang.Long startingSnapshotId,
@@ -495,35 +460,25 @@ |
||
ReplacePartitions |
validateFromSnapshot(long newStartingSnapshotId)
Set the snapshot ID used in validations for this operation.
|
||
ReplacePartitions |
validateNoConflictingData()
Enables validation that data added concurrently does not conflict with this commit's operation.
|
||
ReplacePartitions |
validateNoConflictingDeletes()
Enables validation that deletes that happened concurrently do not conflict with this commit's
operation.
|
||
protected void |
-validateNoNewDeleteFiles(TableMetadata base,
- java.lang.Long startingSnapshotId,
- Expression dataFilter)
-Deprecated.
-
-will be removed in 1.3.0; use
-MergingSnapshotProducer.validateNoNewDeleteFiles(org.apache.iceberg.TableMetadata, java.lang.Long, org.apache.iceberg.expressions.Expression) instead |
-||
protected void |
validateNoNewDeleteFiles(TableMetadata base,
@@ -536,17 +491,6 @@ | ||
protected void |
-validateNoNewDeleteFiles(TableMetadata base,
- java.lang.Long startingSnapshotId,
- PartitionSet partitionSet)
-Deprecated.
-
-will be removed in 1.3.0; use
-MergingSnapshotProducer.validateNoNewDeleteFiles(TableMetadata, Long, PartitionSet,
- Snapshot) instead |
-||
protected void |
validateNoNewDeleteFiles(TableMetadata base,
java.lang.Long startingSnapshotId,
PartitionSet partitionSet,
@@ -555,18 +499,7 @@ |
||
protected void |
-validateNoNewDeletesForDataFiles(TableMetadata base,
- java.lang.Long startingSnapshotId,
- Expression dataFilter,
- java.lang.Iterable<DataFile> dataFiles)
-Deprecated.
-
-will be removed in 1.3.0; use
-MergingSnapshotProducer.validateNoNewDeletesForDataFiles(org.apache.iceberg.TableMetadata, java.lang.Long, java.lang.Iterable<org.apache.iceberg.DataFile>, org.apache.iceberg.Snapshot) instead |
-||
protected void |
validateNoNewDeletesForDataFiles(TableMetadata base,
java.lang.Long startingSnapshotId,
@@ -577,17 +510,7 @@ |
||
protected void |
-validateNoNewDeletesForDataFiles(TableMetadata base,
- java.lang.Long startingSnapshotId,
- java.lang.Iterable<DataFile> dataFiles)
-Deprecated.
-
-will be removed in 1.3.0; use
-MergingSnapshotProducer.validateNoNewDeletesForDataFiles(org.apache.iceberg.TableMetadata, java.lang.Long, java.lang.Iterable<org.apache.iceberg.DataFile>, org.apache.iceberg.Snapshot) instead |
-||
protected void |
validateNoNewDeletesForDataFiles(TableMetadata base,
java.lang.Long startingSnapshotId,
@@ -597,7 +520,7 @@ |
||
protected java.util.concurrent.ExecutorService |
workerPool() |
||
ReplaceSortOrder |
+caseSensitive(boolean caseSensitive)
+Set case sensitivity of sort column name resolution.
+ |
+||
void |
commit()
Apply the pending changes and commit.
|
||
ReplaceSortOrder |
desc(Term term,
NullOrder nullOrder)
@@ -256,7 +262,7 @@ asc-
Method Summary | newManifestWriter(PartitionSpec spec) |
|
protected RollingManifestWriter<DeleteFile> |
+newRollingDeleteManifestWriter(PartitionSpec spec) |
+||
protected RollingManifestWriter<DataFile> |
+newRollingManifestWriter(PartitionSpec spec) |
+||
protected java.lang.String |
operation()
A string that describes the action that produced the new snapshot.
|
||
protected TableMetadata |
refresh() |
||
protected ThisT |
reportWith(MetricsReporter newReporter) |
||
RewriteManifests |
rewriteIf(java.util.function.Predicate<ManifestFile> pred)
Determines which existing
ManifestFile for the table should be rewritten. |
||
ThisT |
scanManifestsWith(java.util.concurrent.ExecutorService executorService)
Use a particular executor to scan manifests.
|
||
protected RewriteManifests |
self() |
||
RewriteManifests |
set(java.lang.String property,
java.lang.String value)
Set a summary property in the snapshot produced by this update.
|
||
protected long |
snapshotId() |
||
ThisT |
stageOnly()
Called to stage a snapshot in table metadata, but not update the current snapshot id.
|
||
protected java.util.Map<java.lang.String,java.lang.String> |
summary() |
||
protected java.lang.String |
targetBranch() |
||
protected void |
targetBranch(java.lang.String branch)
-* A setter for the target branch on which snapshot producer operation should be performed
+A setter for the target branch on which snapshot producer operation should be performed
|
||
protected void |
validate(TableMetadata currentMetadata,
Snapshot snapshot)
Validate the current metadata.
|
||
protected java.util.concurrent.ExecutorService |
workerPool() |
Method and Description | |
long |
+estimatedRowsCount()
+The estimated number of rows produced by this scan task.
+ |
+||
int |
+filesCount()
+The number of files that will be opened by this scan task.
+ |
+||
StructLike |
groupingKey()
Returns a grouping key for this task group.
|
||
long |
+sizeBytes()
+The number of bytes that should be read by this scan task.
+ |
+||
java.util.Collection<T> |
tasks()
Returns scan tasks in this group.
|
||
java.lang.String |
toString() |
||
TableMetadata |
+currentMetadata() |
+||
ExpireSnapshots |
expireSnapshots()
Create a new
expire API to manage snapshots in this table. |
||
ManageSnapshots |
manageSnapshots()
Create a new
manage snapshot API to manage snapshots in this table. |
||
AppendFiles |
newAppend()
Create a new
append API to add files to this table. |
||
DeleteFiles |
newDelete()
Create a new
delete API to replace files in this table. |
||
AppendFiles |
newFastAppend()
Create a new
append API to add files to this table. |
||
OverwriteFiles |
newOverwrite()
Create a new
overwrite API to overwrite files by a filter expression. |
||
ReplacePartitions |
newReplacePartitions()
Not recommended: Create a new
replace partitions API to dynamically
overwrite partitions in the table with new data. |
||
RewriteFiles |
newRewrite()
Create a new
rewrite API to replace files in this table. |
||
RowDelta |
newRowDelta()
Create a new
row-level delta API to remove or replace rows in existing data
files. |
||
ReplaceSortOrder |
replaceSortOrder()
Create a new
ReplaceSortOrder to set a table sort order and commit the change. |
||
RewriteManifests |
rewriteManifests()
Create a new
rewrite manifests API to replace manifests for this
table. |
||
TableMetadata |
startMetadata() |
||
Table |
table()
Return the
Table that this transaction will update. |
||
java.lang.String |
tableName() |
||
TableOperations |
underlyingOps() |
||
UpdateLocation |
updateLocation()
Create a new
UpdateLocation to update table location. |
||
UpdateProperties |
updateProperties()
Create a new
UpdateProperties to update table properties. |
||
UpdateSchema |
updateSchema()
Create a new
UpdateSchema to alter the columns of this table. |
||
UpdatePartitionSpec |
updateSpec()
Create a new
UpdatePartitionSpec to alter the partition spec of this table. |
||
UpdateStatistics |
updateStatistics()
Create a new update table statistics API to add or remove statistics
@@ -336,6 +340,15 @@ startMetadatapublic TableMetadata startMetadata()+ + + +
Interface BatchScan
@@ -168,7 +168,7 @@ Method SummaryMethods inherited from interface org.apache.iceberg.Scan-caseSensitive, filter, filter, ignoreResiduals, includeColumnStats, isCaseSensitive, option, planFiles, planTasks, planWith, project, schema, select, select, splitLookback, splitOpenFileCost, targetSplitSize
+caseSensitive, filter, filter, ignoreResiduals, includeColumnStats, isCaseSensitive, metricsReporter, option, planFiles, planTasks, planWith, project, schema, select, select, splitLookback, splitOpenFileCost, targetSplitSize
diff --git a/javadoc/org/apache/iceberg/CatalogProperties.html b/javadoc/org/apache/iceberg/CatalogProperties.html
index 907c09de8..5a60c52bf 100644
--- a/javadoc/org/apache/iceberg/CatalogProperties.html
+++ b/javadoc/org/apache/iceberg/CatalogProperties.html
@@ -127,29 +127,21 @@ Field Summary | ||
static java.lang.String |
-AUTH_DEFAULT_REFRESH_ENABLED
-Deprecated.
-
-Will be removed in 1.3.0; Use
-OAuth2Properties.TOKEN_REFRESH_ENABLED to control token
- refresh behavior. |
+AUTH_SESSION_TIMEOUT_MS |
|
static boolean |
-AUTH_DEFAULT_REFRESH_ENABLED_DEFAULT
-Deprecated.
-
-Will be removed in 1.3.0; Use
-OAuth2Properties.TOKEN_REFRESH_ENABLED_DEFAULT to control
- default token refresh behavior. |
+static long |
+AUTH_SESSION_TIMEOUT_MS_DEFAULT |
static java.lang.String |
-AUTH_SESSION_TIMEOUT_MS |
+CACHE_CASE_SENSITIVE
+Controls whether the caching catalog will cache table entries using case sensitive keys.
+ |
|
static long |
-AUTH_SESSION_TIMEOUT_MS_DEFAULT |
+static boolean |
+CACHE_CASE_SENSITIVE_DEFAULT |
static java.lang.String |
@@ -189,125 +181,132 @@ |||
static java.lang.String |
-CLIENT_POOL_SIZE |
+CLIENT_POOL_CACHE_KEYS
+A comma separated list of elements used, in addition to the
+URI, to compose the key of
+ the client pool cache. |
|
static java.lang.String |
+CLIENT_POOL_SIZE |
+||
static int |
CLIENT_POOL_SIZE_DEFAULT |
||
static java.lang.String |
FILE_IO_IMPL |
||
static java.lang.String |
IO_MANIFEST_CACHE_ENABLED
Controls whether to use caching during manifest reads or not.
|
||
static boolean |
IO_MANIFEST_CACHE_ENABLED_DEFAULT |
||
static java.lang.String |
IO_MANIFEST_CACHE_EXPIRATION_INTERVAL_MS
Controls the maximum duration for which an entry stays in the manifest cache.
|
||
static long |
IO_MANIFEST_CACHE_EXPIRATION_INTERVAL_MS_DEFAULT |
||
static java.lang.String |
IO_MANIFEST_CACHE_MAX_CONTENT_LENGTH
Controls the maximum length of file to be considered for caching.
|
||
static long |
IO_MANIFEST_CACHE_MAX_CONTENT_LENGTH_DEFAULT |
||
static java.lang.String |
IO_MANIFEST_CACHE_MAX_TOTAL_BYTES
Controls the maximum total amount of bytes to cache in manifest cache.
|
||
static long |
IO_MANIFEST_CACHE_MAX_TOTAL_BYTES_DEFAULT |
||
static java.lang.String |
LOCK_ACQUIRE_INTERVAL_MS |
||
static long |
LOCK_ACQUIRE_INTERVAL_MS_DEFAULT |
||
static java.lang.String |
LOCK_ACQUIRE_TIMEOUT_MS |
||
static long |
LOCK_ACQUIRE_TIMEOUT_MS_DEFAULT |
||
static java.lang.String |
LOCK_HEARTBEAT_INTERVAL_MS |
||
static long |
LOCK_HEARTBEAT_INTERVAL_MS_DEFAULT |
||
static java.lang.String |
LOCK_HEARTBEAT_THREADS |
||
static int |
LOCK_HEARTBEAT_THREADS_DEFAULT |
||
static java.lang.String |
LOCK_HEARTBEAT_TIMEOUT_MS |
||
static long |
LOCK_HEARTBEAT_TIMEOUT_MS_DEFAULT |
||
static java.lang.String |
LOCK_IMPL |
||
static java.lang.String |
LOCK_TABLE |
||
static java.lang.String |
METRICS_REPORTER_IMPL |
||
static java.lang.String |
TABLE_DEFAULT_PREFIX |
||
static java.lang.String |
TABLE_OVERRIDE_PREFIX |
||
static java.lang.String |
URI |
||
static java.lang.String |
USER |
||
static java.lang.String |
WAREHOUSE_LOCATION |
||
static void |
+deleteFiles(FileIO io,
+ java.lang.Iterable<java.lang.String> files,
+ java.lang.String type,
+ boolean concurrent)
+Helper to delete files.
+ |
+||
static void |
dropTableData(FileIO io,
TableMetadata metadata)
Drops all data and metadata files referenced by TableMetadata.
|
||
static Catalog |
loadCatalog(java.lang.String impl,
java.lang.String catalogName,
@@ -205,7 +214,7 @@ |
||
static FileIO |
loadFileIO(java.lang.String impl,
java.util.Map<java.lang.String,java.lang.String> properties,
@@ -213,7 +222,7 @@ |
||
static MetricsReporter |
loadMetricsReporter(java.util.Map<java.lang.String,java.lang.String> properties)
Load a custom
@@ -367,6 +376,26 @@ MetricsReporter implementation.dropTableData+ + + +
Method Summary | close(C client) |
|
boolean |
+isClosed() |
+||
protected boolean |
isConnectionException(java.lang.Exception exc) |
||
protected abstract C |
newClient() |
||
int |
poolSize() |
||
protected abstract C |
reconnect(C client) |
||
<R> R |
run(ClientPool.Action<R,C,E> action) |
||
<R> R |
run(ClientPool.Action<R,C,E> action,
boolean retry) |
@@ -339,12 +343,21 @@ ||
default java.lang.Long |
+dataSequenceNumber()
+Returns the data sequence number of the file.
+ |
+||
java.util.List<java.lang.Integer> |
equalityFieldIds()
Returns the set of field IDs used for equality comparison, in equality delete files.
|
||
default java.lang.Long |
+fileSequenceNumber()
+Returns the file sequence number.
+ |
+||
long |
fileSizeInBytes()
Returns the file size in bytes.
|
||
FileFormat |
format()
Returns format of the file.
|
||
java.nio.ByteBuffer |
keyMetadata()
Returns metadata about how this file is encrypted, or null if the file is stored in plain text.
|
||
java.util.Map<java.lang.Integer,java.nio.ByteBuffer> |
lowerBounds()
Returns if collected, map from column ID to value lower bounds, null otherwise.
|
||
java.util.Map<java.lang.Integer,java.lang.Long> |
nanValueCounts()
Returns if collected, map from column ID to its NaN value count, null otherwise.
|
||
java.util.Map<java.lang.Integer,java.lang.Long> |
nullValueCounts()
Returns if collected, map from column ID to its null value count, null otherwise.
|
||
StructLike |
partition()
Returns partition for this file as a
StructLike. |
||
java.lang.CharSequence |
path()
Returns fully qualified path to the file, suitable for constructing a Hadoop Path.
|
||
java.lang.Long |
pos()
Returns the ordinal position of the file in a manifest, or null if it was not read from a
manifest.
|
||
long |
recordCount()
Returns the number of top-level records in the file.
|
||
default java.lang.Integer |
sortOrderId()
Returns the sort order id of this file, which describes how the file is ordered.
|
||
int |
specId()
Returns id of the partition spec used for partition metadata.
|
||
java.util.List<java.lang.Long> |
splitOffsets()
Returns list of recommended split locations, if applicable, null otherwise.
|
||
java.util.Map<java.lang.Integer,java.nio.ByteBuffer> |
upperBounds()
Returns if collected, map from column ID to value upper bounds, null otherwise.
|
||
java.util.Map<java.lang.Integer,java.lang.Long> |
valueCounts()
-Returns if collected, map from column ID to the count of its non-null values, null otherwise.
+Returns if collected, map from column ID to the count of its values (including null and NaN
+ values), null otherwise.
|
java.util.Map<java.lang.Integer,java.lang.Long> valueCounts()-
default java.lang.Long dataSequenceNumber()+
This method represents the sequence number to which the file should apply. Note the data + sequence number may differ from the sequence number of the snapshot in which the underlying + file was added (a.k.a the file sequence number). New snapshots can add files that belong to + older sequence numbers (e.g. compaction). The data sequence number also does not change when + the file is marked as deleted. + +
This method can return null if the data sequence number is unknown. This may happen while + reading a v2 manifest that did not persist the data sequence number for manifest entries with + status DELETED (older Iceberg versions).
default java.lang.Long fileSequenceNumber()+
The file sequence number represents the sequence number of the snapshot in which the + underlying file was added. The file sequence number is always assigned at commit and cannot be + provided explicitly, unlike the data sequence number. The file sequence number does not change + upon assigning. In case of rewrite (like compaction), file sequence number can be higher than + the data sequence number. + +
This method can return null if the file sequence number is unknown. This may happen while + reading a v2 manifest that did not persist the file sequence number for manifest entries with + status EXISTING or DELETED (older Iceberg versions).
columnSizes, copy, copy, copyWithoutStats, fileSizeInBytes, format, keyMetadata, lowerBounds, nanValueCounts, nullValueCounts, partition, path, pos, recordCount, sortOrderId, specId, splitOffsets, upperBounds, valueCounts
+columnSizes, copy, copy, copyWithoutStats, dataSequenceNumber, fileSequenceNumber, fileSizeInBytes, format, keyMetadata, lowerBounds, nanValueCounts, nullValueCounts, partition, path, pos, recordCount, sortOrderId, specId, splitOffsets, upperBounds, valueCounts
diff --git a/javadoc/org/apache/iceberg/DataFiles.Builder.html b/javadoc/org/apache/iceberg/DataFiles.Builder.html
index b08f84038..1bd4fecf0 100644
--- a/javadoc/org/apache/iceberg/DataFiles.Builder.html
+++ b/javadoc/org/apache/iceberg/DataFiles.Builder.html
@@ -17,7 +17,7 @@
catch(err) {
}
//-->
-var methods = {"i0":10,"i1":10,"i2":10,"i3":10,"i4":10,"i5":10,"i6":10,"i7":10,"i8":10,"i9":10,"i10":10,"i11":10,"i12":10,"i13":10,"i14":10,"i15":10,"i16":10,"i17":10};
+var methods = {"i0":10,"i1":10,"i2":10,"i3":10,"i4":10,"i5":10,"i6":10,"i7":10,"i8":10,"i9":10,"i10":10,"i11":10,"i12":10,"i13":10,"i14":10,"i15":10,"i16":10,"i17":10,"i18":10,"i19":10};
var tabs = {65535:["t0","All Methods"],2:["t2","Instance Methods"],8:["t4","Concrete Methods"]};
var altColor = "altColor";
var rowColor = "rowColor";
@@ -174,50 +174,58 @@ DataFiles.BuilderwithFileSizeInBytes(long newFileSizeInBytes) withEqualityFieldIds(java.util.List<java.lang.Integer> equalityIds) DataFiles.BuilderwithFormat(FileFormat newFormat) withFileSizeInBytes(long newFileSizeInBytes) DataFiles.BuilderwithFormat(java.lang.String newFormat) withFormat(FileFormat newFormat) DataFiles.BuilderwithInputFile(InputFile file) withFormat(java.lang.String newFormat) DataFiles.BuilderwithMetrics(Metrics metrics) withInputFile(InputFile file) DataFiles.BuilderwithPartition(StructLike newPartition) withMetrics(Metrics metrics) DataFiles.BuilderwithPartitionPath(java.lang.String newPartitionPath) withPartition(StructLike newPartition) DataFiles.BuilderwithPath(java.lang.String newFilePath) withPartitionPath(java.lang.String newPartitionPath) DataFiles.BuilderwithRecordCount(long newRecordCount) withPartitionValues(java.util.List<java.lang.String> partitionValues) DataFiles.BuilderwithSortOrder(SortOrder newSortOrder) withPath(java.lang.String newFilePath) DataFiles.BuilderwithSplitOffsets(java.util.List<java.lang.Long> offsets) withRecordCount(long newRecordCount) DataFiles.BuilderwithSortOrder(SortOrder newSortOrder) DataFiles.BuilderwithSplitOffsets(java.util.List<java.lang.Long> offsets) DataFiles.BuilderwithStatus(org.apache.hadoop.fs.FileStatus stat) public DataFiles.Builder withPartitionPath(java.lang.String newPartitionPath)+ + + +
public DataFiles.Builder withPartitionValues(java.util.List<java.lang.String> partitionValues)+
public DataFiles.Builder withSplitOffsets(java.util.List<java.lang.Long> offsets)+ + + +
public DataFiles.Builder withEqualityFieldIds(java.util.List<java.lang.Integer> equalityIds)+
protected static java.util.List<java.lang.String>DELETE_SCAN_WITH_STATS_COLUMNS protected static booleanPLAN_SCANS_WITH_WORKER_POOL protected static java.util.List<java.lang.String>SCAN_COLUMNS protected static java.util.List<java.lang.String>SCAN_WITH_STATS_COLUMNS | Modifier and Type | Method and Description | @@ -219,36 +231,35 @@||
|---|---|---|---|
protected FileIO |
+io() |
+||
boolean |
isCaseSensitive()
Returns whether this scan is case-sensitive with respect to column names.
|
||
protected CloseableIterable<ManifestFile> |
manifests()
Returns an iterable of manifest files to explore for this files metadata table scan
|
||
protected ThisT |
-newRefinedScan(TableOperations ignored,
- Table newTable,
- Schema newSchema,
- org.apache.iceberg.TableScanContext newContext)
-Deprecated.
-
+will be removed in 1.3.0; use newRefinedScan(Table, Schema, TableScanContext)
- instead.
- | ||
ThisT |
+metricsReporter(MetricsReporter reporter)
+Create a new scan that will report scan metrics to the provided reporter in addition to
+ reporters maintained by the scan.
|
||
protected TableScan |
newRefinedScan(Table table,
Schema schema,
org.apache.iceberg.TableScanContext context) |
||
ThisT |
option(java.lang.String property,
java.lang.String value)
@@ -256,92 +267,92 @@ Method Summarybehavior based on the incoming pair. |
||
protected java.util.Map<java.lang.String,java.lang.String> |
options() |
||
protected java.util.concurrent.ExecutorService |
planExecutor() |
||
CloseableIterable<CombinedScanTask> |
planTasks()
Plan balanced task groups for this scan by splitting large and combining small tasks.
|
||
ThisT |
planWith(java.util.concurrent.ExecutorService executorService)
Create a new scan to use a particular executor to plan.
|
||
ThisT |
project(Schema projectedSchema)
Create a new scan from this with the schema as its projection.
|
||
protected Expression |
+residualFilter() |
+||
protected java.util.List<java.lang.String> |
scanColumns() |
||
Schema |
schema()
Returns this scan's projection
Schema. |
||
ThisT |
select(java.util.Collection<java.lang.String> columns)
Create a new scan from this that will read the given data columns.
|
||
protected boolean |
shouldIgnoreResiduals() |
||
protected boolean |
shouldPlanWithExecutor() |
||
protected boolean |
+shouldReturnColumnStats() |
+||
int |
splitLookback()
Returns the split lookback for this scan.
|
||
long |
splitOpenFileCost()
Returns the split open file cost for this scan.
|
||
Table |
table() |
||
protected TableOperations |
-tableOps()
-Deprecated.
-
-will be removed in 1.3.0; avoid using TableOperations for scans or use BaseTable
- |
-||
protected Schema |
tableSchema() |
||
protected MetadataTableType |
tableType()
Type of scan being performed, such as
MetadataTableType.ALL_DATA_FILES when scanning a
table's AllDataFilesTable. |
||
long |
targetSplitSize()
Returns the target split size for this scan.
@@ -353,7 +364,7 @@ Method SummaryMethods inherited from class org.apache.iceberg.SnapshotScan-asOfTime, planFiles, scanMetrics, snapshot, snapshotId, toString, useRef, useSnapshot
+asOfTime, planFiles, scanMetrics, snapshot, snapshotId, toString, useRef, useSnapshot, useSnapshotSchema
caseSensitive, filter, filter, ignoreResiduals, includeColumnStats, isCaseSensitive, metricsReporter, option, planFiles, planWith, project, schema, select, select, splitLookback, splitOpenFileCost
Methods inherited from interface org.apache.iceberg.
+
+
+SCAN_COLUMNS+protected static final java.util.List<java.lang.String> SCAN_COLUMNS+ SCAN_WITH_STATS_COLUMNS+protected static final java.util.List<java.lang.String> SCAN_WITH_STATS_COLUMNS+
tableOps-@Deprecated -protected TableOperations tableOps()- Deprecated. will be removed in 1.3.0; avoid using TableOperations for scans or use BaseTable
+table+public Table table() shouldReturnColumnStats+protected boolean shouldReturnColumnStats()+ shouldPlanWithExecutor-protected boolean shouldPlanWithExecutor()+ residualFilter+protected Expression residualFilter() planExecutor-protected java.util.concurrent.ExecutorService planExecutor()+ shouldPlanWithExecutor+protected boolean shouldPlanWithExecutor() newRefinedScan-@Deprecated -protected ThisT newRefinedScan(TableOperations ignored, - Table newTable, - Schema newSchema, - org.apache.iceberg.TableScanContext newContext)- Deprecated. will be removed in 1.3.0; use newRefinedScan(Table, Schema, TableScanContext)
- instead.
+planExecutor+protected java.util.concurrent.ExecutorService planExecutor()
| ||
protected static boolean |
+PLAN_SCANS_WITH_WORKER_POOL |
+||
protected static java.util.List<java.lang.String> |
+SCAN_COLUMNS |
+||
protected static java.util.List<java.lang.String> |
+SCAN_WITH_STATS_COLUMNS |
+
DataTableScan(TableOperations ignored,
- Table table)
-protected DataTableScan(TableOperations ignored,
- Table table,
- Schema schema,
- org.apache.iceberg.TableScanContext context)
-protected DataTableScan(Table table,
Schema schema,
@@ -197,7 +187,7 @@ Constructor Summary
Method Summary
-All Methods Instance Methods Concrete Methods Deprecated Methods
+All Methods Instance Methods Concrete Methods
Modifier and Type
Method and Description
@@ -258,30 +248,29 @@ Method Summary
+protected FileIOio()
+
booleanisCaseSensitive()
Returns whether this scan is case-sensitive with respect to column names.
-
-protected ThisTnewRefinedScan(TableOperations ignored,
- Table newTable,
- Schema newSchema,
- org.apache.iceberg.TableScanContext newContext)
-Deprecated.
-will be removed in 1.3.0; use newRefinedScan(Table, Schema, TableScanContext)
- instead.
-
+
+ThisTmetricsReporter(MetricsReporter reporter)
+Create a new scan that will report scan metrics to the provided reporter in addition to
+ reporters maintained by the scan.
-
+
protected TableScannewRefinedScan(Table table,
Schema schema,
org.apache.iceberg.TableScanContext context)
-
+
ThisToption(java.lang.String property,
java.lang.String value)
@@ -289,96 +278,93 @@ Method Summary
behavior based on the incoming pair.
-
+
protected java.util.Map<java.lang.String,java.lang.String>options()
-
+
protected java.util.concurrent.ExecutorServiceplanExecutor()
-
+
CloseableIterable<CombinedScanTask>planTasks()
Plan balanced task groups for this scan by splitting large and combining small tasks.
-
+
ThisTplanWith(java.util.concurrent.ExecutorService executorService)
Create a new scan to use a particular executor to plan.
-
+
ThisTproject(Schema projectedSchema)
Create a new scan from this with the schema as its projection.
-
+
+protected ExpressionresidualFilter()
+
protected java.util.List<java.lang.String>scanColumns()
-
+
Schemaschema()
Returns this scan's projection Schema.
-
+
ThisTselect(java.util.Collection<java.lang.String> columns)
Create a new scan from this that will read the given data columns.
-
+
protected booleanshouldIgnoreResiduals()
-
+
protected booleanshouldPlanWithExecutor()
-
+
+protected booleanshouldReturnColumnStats()
+
intsplitLookback()
Returns the split lookback for this scan.
-
+
longsplitOpenFileCost()
Returns the split open file cost for this scan.
-
+
Tabletable()
-
-protected TableOperationstableOps()
-Deprecated.
-will be removed in 1.3.0; avoid using TableOperations for scans or use BaseTable
-
-
-
+
protected SchematableSchema()
-
+
longtargetSplitSize()
Returns the target split size for this scan.
-
-TableScanuseSnapshot(long scanSnapshotId)
-Create a new TableScan from this scan's configuration that will use the given snapshot
- by ID.
-
+protected booleanuseSnapshotSchema()
@@ -386,7 +372,7 @@ Method Summary
Methods inherited from class org.apache.iceberg.SnapshotScan
-asOfTime, planFiles, scanMetrics, snapshot, snapshotId, toString, useRef
+asOfTime, planFiles, scanMetrics, snapshot, snapshotId, toString, useRef, useSnapshot
-
@@ -400,14 +386,14 @@
Methods inherited from class java.lang.Object
Methods inherited from interface org.apache.iceberg.TableScan
-asOfTime, snapshot, table, useRef
+asOfTime, snapshot, table, useRef, useSnapshot
-
Methods inherited from interface org.apache.iceberg.Scan
-caseSensitive, filter, filter, ignoreResiduals, includeColumnStats, isCaseSensitive, option, planFiles, planWith, project, schema, select, select, splitLookback, splitOpenFileCost, targetSplitSize
+caseSensitive, filter, filter, ignoreResiduals, includeColumnStats, isCaseSensitive, metricsReporter, option, planFiles, planWith, project, schema, select, select, splitLookback, splitOpenFileCost, targetSplitSize
@@ -423,6 +409,24 @@ Methods inherited from interface org.apache.iceberg.
+
+
+
+-
+
SCAN_COLUMNS
+protected static final java.util.List<java.lang.String> SCAN_COLUMNS
+
+
+
+
+
+
+-
+
SCAN_WITH_STATS_COLUMNS
+protected static final java.util.List<java.lang.String> SCAN_WITH_STATS_COLUMNS
+
+
@@ -435,12 +439,21 @@ DELETE_SCAN_COLUMNS
-
+
-
DELETE_SCAN_WITH_STATS_COLUMNS
protected static final java.util.List<java.lang.String> DELETE_SCAN_WITH_STATS_COLUMNS
+
+
+
+
+-
+
PLAN_SCANS_WITH_WORKER_POOL
+protected static final boolean PLAN_SCANS_WITH_WORKER_POOL
+
+
@@ -449,34 +462,6 @@ DELETE_SCAN_WITH_STATS_COLUMNS
Constructor Detail
-
-
-
-
--
-
DataTableScan
-@Deprecated
-public DataTableScan(TableOperations ignored,
- Table table)
-Deprecated. will be removed in 1.3.0; use DataTableScan(Table, Schema, TableScanContext)
- instead.
-
-
-
-
-
-
--
-
DataTableScan
-@Deprecated
-protected DataTableScan(TableOperations ignored,
- Table table,
- Schema schema,
- org.apache.iceberg.TableScanContext context)
-Deprecated. will be removed in 1.3.0; use DataTableScan(Table, Schema, TableScanContext)
- instead.
-
-
@@ -540,25 +525,16 @@ appendsAfter
-
+
-
-
useSnapshot
-public TableScan useSnapshot(long scanSnapshotId)
-Description copied from interface: TableScan
-Create a new TableScan from this scan's configuration that will use the given snapshot
- by ID.
+useSnapshotSchema
+protected boolean useSnapshotSchema()
-- Specified by:
-useSnapshot in interface TableScan- Overrides:
-useSnapshot in class SnapshotScan<TableScan,FileScanTask,CombinedScanTask>- Parameters:
-scanSnapshotId - a snapshot ID- Returns:
-- a new scan based on this with the given snapshot ID
+useSnapshotSchema in class SnapshotScan<TableScan,FileScanTask,CombinedScanTask>
@@ -606,24 +582,22 @@ planTasks
-
+
-
-
tableOps
-@Deprecated
-protected TableOperations tableOps()
-Deprecated. will be removed in 1.3.0; avoid using TableOperations for scans or use BaseTable
+table
+public Table table()
-
+
@@ -662,6 +636,15 @@ scanColumns
protected java.util.List<java.lang.String> scanColumns()
+
+
+
+
+-
+
shouldReturnColumnStats
+protected boolean shouldReturnColumnStats()
+
+
@@ -671,37 +654,31 @@ shouldIgnoreResiduals
protected boolean shouldIgnoreResiduals()
-
+
-
-
shouldPlanWithExecutor
-protected boolean shouldPlanWithExecutor()
+residualFilter
+protected Expression residualFilter()
-
+
-
-
planExecutor
-protected java.util.concurrent.ExecutorService planExecutor()
+shouldPlanWithExecutor
+protected boolean shouldPlanWithExecutor()
-
+
-
-
newRefinedScan
-@Deprecated
-protected ThisT newRefinedScan(TableOperations ignored,
- Table newTable,
- Schema newSchema,
- org.apache.iceberg.TableScanContext newContext)
-Deprecated. will be removed in 1.3.0; use newRefinedScan(Table, Schema, TableScanContext)
- instead.
+planExecutor
+protected java.util.concurrent.ExecutorService planExecutor()
@@ -951,7 +928,7 @@ splitLookback
-
+
-
splitOpenFileCost
public long splitOpenFileCost()
@@ -963,6 +940,22 @@ splitOpenFileCost
+
+
+
+
+-
+
metricsReporter
+public ThisT metricsReporter(MetricsReporter reporter)
+Description copied from interface: Scan
+Create a new scan that will report scan metrics to the provided reporter in addition to
+ reporters maintained by the scan.
+
+- Specified by:
+metricsReporter in interface Scan<ThisT,T extends ScanTask,G extends ScanTaskGroup<T>>
+
+
diff --git a/javadoc/org/apache/iceberg/DataTask.html b/javadoc/org/apache/iceberg/DataTask.html
index c1eb65c59..16eabf331 100644
--- a/javadoc/org/apache/iceberg/DataTask.html
+++ b/javadoc/org/apache/iceberg/DataTask.html
@@ -148,7 +148,7 @@ Method Summary
Methods inherited from interface org.apache.iceberg.FileScanTask
-asFileScanTask, deletes, filesCount, isFileScanTask, sizeBytes
+asFileScanTask, deletes, filesCount, isFileScanTask, schema, sizeBytes
-
diff --git a/javadoc/org/apache/iceberg/DeleteFile.html b/javadoc/org/apache/iceberg/DeleteFile.html
index d848e82e3..d6cc9a07c 100644
--- a/javadoc/org/apache/iceberg/DeleteFile.html
+++ b/javadoc/org/apache/iceberg/DeleteFile.html
@@ -136,7 +136,7 @@
Method Summary
Methods inherited from interface org.apache.iceberg.ContentFile
-columnSizes, content, copy, copy, copyWithoutStats, equalityFieldIds, fileSizeInBytes, format, keyMetadata, lowerBounds, nanValueCounts, nullValueCounts, partition, path, pos, recordCount, sortOrderId, specId, upperBounds, valueCounts
+columnSizes, content, copy, copy, copyWithoutStats, dataSequenceNumber, equalityFieldIds, fileSequenceNumber, fileSizeInBytes, format, keyMetadata, lowerBounds, nanValueCounts, nullValueCounts, partition, path, pos, recordCount, sortOrderId, specId, upperBounds, valueCounts
diff --git a/javadoc/org/apache/iceberg/DeleteFiles.html b/javadoc/org/apache/iceberg/DeleteFiles.html
index f809ae0f8..19e9400c2 100644
--- a/javadoc/org/apache/iceberg/DeleteFiles.html
+++ b/javadoc/org/apache/iceberg/DeleteFiles.html
@@ -17,7 +17,7 @@
catch(err) {
}
//-->
-var methods = {"i0":6,"i1":6,"i2":18,"i3":6};
+var methods = {"i0":6,"i1":6,"i2":18,"i3":6,"i4":18};
var tabs = {65535:["t0","All Methods"],2:["t2","Instance Methods"],4:["t3","Abstract Methods"],16:["t5","Default Methods"]};
var altColor = "altColor";
var rowColor = "rowColor";
@@ -158,6 +158,13 @@ Method Summary
Delete files that match an Expression on data rows from the table.
default DeleteFilesvalidateFilesExist()
+DeleteFiles caseSensitive(boolean caseSensitive)@@ -266,6 +273,21 @@
default DeleteFiles validateFilesExist()+
protected static java.util.List<java.lang.String>DELETE_SCAN_WITH_STATS_COLUMNS protected static booleanPLAN_SCANS_WITH_WORKER_POOL protected static java.util.List<java.lang.String>SCAN_COLUMNS protected static java.util.List<java.lang.String>SCAN_WITH_STATS_COLUMNS | Modifier and Type | Method and Description | @@ -219,36 +231,35 @@
|---|---|
protected FileIO |
+io() |
+
boolean |
isCaseSensitive()
Returns whether this scan is case-sensitive with respect to column names.
|
protected CloseableIterable<ManifestFile> |
manifests()
Returns an iterable of manifest files to explore for this files metadata table scan
|
protected ThisT |
-newRefinedScan(TableOperations ignored,
- Table newTable,
- Schema newSchema,
- org.apache.iceberg.TableScanContext newContext)
-Deprecated.
-
+will be removed in 1.3.0; use newRefinedScan(Table, Schema, TableScanContext)
- instead.
- |
ThisT |
+metricsReporter(MetricsReporter reporter)
+Create a new scan that will report scan metrics to the provided reporter in addition to
+ reporters maintained by the scan.
|
protected TableScan |
newRefinedScan(Table table,
Schema schema,
org.apache.iceberg.TableScanContext context) |
ThisT |
option(java.lang.String property,
java.lang.String value)
@@ -256,92 +267,92 @@ Method Summarybehavior based on the incoming pair. |
protected java.util.Map<java.lang.String,java.lang.String> |
options() |
protected java.util.concurrent.ExecutorService |
planExecutor() |
CloseableIterable<CombinedScanTask> |
planTasks()
Plan balanced task groups for this scan by splitting large and combining small tasks.
|
ThisT |
planWith(java.util.concurrent.ExecutorService executorService)
Create a new scan to use a particular executor to plan.
|
ThisT |
project(Schema projectedSchema)
Create a new scan from this with the schema as its projection.
|
protected Expression |
+residualFilter() |
+
protected java.util.List<java.lang.String> |
scanColumns() |
Schema |
schema()
Returns this scan's projection
Schema. |
ThisT |
select(java.util.Collection<java.lang.String> columns)
Create a new scan from this that will read the given data columns.
|
protected boolean |
shouldIgnoreResiduals() |
protected boolean |
shouldPlanWithExecutor() |
protected boolean |
+shouldReturnColumnStats() |
+
int |
splitLookback()
Returns the split lookback for this scan.
|
long |
splitOpenFileCost()
Returns the split open file cost for this scan.
|
Table |
table() |
protected TableOperations |
-tableOps()
-Deprecated.
-
-will be removed in 1.3.0; avoid using TableOperations for scans or use BaseTable
- |
-
protected Schema |
tableSchema() |
protected MetadataTableType |
tableType()
Type of scan being performed, such as
MetadataTableType.ALL_DATA_FILES when scanning a
table's AllDataFilesTable. |
long |
targetSplitSize()
Returns the target split size for this scan.
@@ -353,7 +364,7 @@ Method SummaryMethods inherited from class org.apache.iceberg.SnapshotScan-asOfTime, planFiles, scanMetrics, snapshot, snapshotId, toString, useRef, useSnapshot
+asOfTime, planFiles, scanMetrics, snapshot, snapshotId, toString, useRef, useSnapshot, useSnapshotSchema
caseSensitive, filter, filter, ignoreResiduals, includeColumnStats, isCaseSensitive, metricsReporter, option, planFiles, planWith, project, schema, select, select, splitLookback, splitOpenFileCost
Methods inherited from interface org.apache.iceberg.
+
+
+SCAN_COLUMNS+protected static final java.util.List<java.lang.String> SCAN_COLUMNS+ SCAN_WITH_STATS_COLUMNS+protected static final java.util.List<java.lang.String> SCAN_WITH_STATS_COLUMNS+
tableOps-@Deprecated -protected TableOperations tableOps()- Deprecated. will be removed in 1.3.0; avoid using TableOperations for scans or use BaseTable
+table+public Table table() shouldReturnColumnStats+protected boolean shouldReturnColumnStats()+ shouldPlanWithExecutor-protected boolean shouldPlanWithExecutor()+ residualFilter+protected Expression residualFilter() planExecutor-protected java.util.concurrent.ExecutorService planExecutor()+ shouldPlanWithExecutor+protected boolean shouldPlanWithExecutor() newRefinedScan-@Deprecated -protected ThisT newRefinedScan(TableOperations ignored, - Table newTable, - Schema newSchema, - org.apache.iceberg.TableScanContext newContext)- Deprecated. will be removed in 1.3.0; use newRefinedScan(Table, Schema, TableScanContext)
- instead.
+planExecutor+protected java.util.concurrent.ExecutorService planExecutor()
|
FileMetadata.Builder |
+withSplitOffsets(java.util.List<java.lang.Long> offsets) |
+
FileMetadata.Builder |
withStatus(org.apache.hadoop.fs.FileStatus stat) |
public FileMetadata.Builder withMetrics(Metrics metrics)+ + + +
public FileMetadata.Builder withSplitOffsets(java.util.List<java.lang.Long> offsets)+
| Modifier and Type | Method and Description | @@ -219,36 +231,35 @@||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
protected FileIO |
+io() |
+||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
boolean |
isCaseSensitive()
Returns whether this scan is case-sensitive with respect to column names.
|
||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
protected CloseableIterable<ManifestFile> |
manifests()
Returns an iterable of manifest files to explore for this files metadata table scan
|
||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
protected ThisT |
-newRefinedScan(TableOperations ignored,
- Table newTable,
- Schema newSchema,
- org.apache.iceberg.TableScanContext newContext)
-Deprecated.
-
+will be removed in 1.3.0; use newRefinedScan(Table, Schema, TableScanContext)
- instead.
- | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
ThisT |
+metricsReporter(MetricsReporter reporter)
+Create a new scan that will report scan metrics to the provided reporter in addition to
+ reporters maintained by the scan.
|
||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
protected TableScan |
newRefinedScan(Table table,
Schema schema,
org.apache.iceberg.TableScanContext context) |
||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
ThisT |
option(java.lang.String property,
java.lang.String value)
@@ -256,92 +267,92 @@ Method Summarybehavior based on the incoming pair. |
||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
protected java.util.Map<java.lang.String,java.lang.String> |
options() |
||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
protected java.util.concurrent.ExecutorService |
planExecutor() |
||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
CloseableIterable<CombinedScanTask> |
planTasks()
Plan balanced task groups for this scan by splitting large and combining small tasks.
|
||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
ThisT |
planWith(java.util.concurrent.ExecutorService executorService)
Create a new scan to use a particular executor to plan.
|
||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
ThisT |
project(Schema projectedSchema)
Create a new scan from this with the schema as its projection.
|
||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
protected Expression |
+residualFilter() |
+||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
protected java.util.List<java.lang.String> |
scanColumns() |
||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
Schema |
schema()
Returns this scan's projection
Schema. |
||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
ThisT |
select(java.util.Collection<java.lang.String> columns)
Create a new scan from this that will read the given data columns.
|
||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
protected boolean |
shouldIgnoreResiduals() |
||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
protected boolean |
shouldPlanWithExecutor() |
||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
protected boolean |
+shouldReturnColumnStats() |
+||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
int |
splitLookback()
Returns the split lookback for this scan.
|
||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
long |
splitOpenFileCost()
Returns the split open file cost for this scan.
|
||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
Table |
table() |
||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
protected TableOperations |
-tableOps()
-Deprecated.
-
-will be removed in 1.3.0; avoid using TableOperations for scans or use BaseTable
- |
-||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
protected Schema |
tableSchema() |
||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
protected MetadataTableType |
tableType()
Type of scan being performed, such as
MetadataTableType.ALL_DATA_FILES when scanning a
table's AllDataFilesTable. |
||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
long |
targetSplitSize()
Returns the target split size for this scan.
@@ -353,7 +364,7 @@ Method SummaryMethods inherited from class org.apache.iceberg.SnapshotScan-asOfTime, planFiles, scanMetrics, snapshot, snapshotId, toString, useRef, useSnapshot
+asOfTime, planFiles, scanMetrics, snapshot, snapshotId, toString, useRef, useSnapshot, useSnapshotSchema
caseSensitive, filter, filter, ignoreResiduals, includeColumnStats, isCaseSensitive, metricsReporter, option, planFiles, planWith, project, schema, select, select, splitLookback, splitOpenFileCost
Methods inherited from interface org.apache.iceberg.
+
+
+SCAN_COLUMNS+protected static final java.util.List<java.lang.String> SCAN_COLUMNS+ SCAN_WITH_STATS_COLUMNS+protected static final java.util.List<java.lang.String> SCAN_WITH_STATS_COLUMNS+
tableOps-@Deprecated -protected TableOperations tableOps()- Deprecated. will be removed in 1.3.0; avoid using TableOperations for scans or use BaseTable
+table+public Table table() shouldReturnColumnStats+protected boolean shouldReturnColumnStats()+ shouldPlanWithExecutor-protected boolean shouldPlanWithExecutor()+ residualFilter+protected Expression residualFilter() planExecutor-protected java.util.concurrent.ExecutorService planExecutor()+ shouldPlanWithExecutor+protected boolean shouldPlanWithExecutor() newRefinedScan-@Deprecated -protected ThisT newRefinedScan(TableOperations ignored, - Table newTable, - Schema newSchema, - org.apache.iceberg.TableScanContext newContext)- Deprecated. will be removed in 1.3.0; use newRefinedScan(Table, Schema, TableScanContext)
- instead.
+planExecutor+protected java.util.concurrent.ExecutorService planExecutor()
Method SummaryMethods inherited from class java.lang.Objectclone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait
+
Nested Class SummaryNested classes/interfaces inherited from interface org.apache.iceberg.MetadataUpdate-MetadataUpdate.AddPartitionSpec, MetadataUpdate.AddSchema, MetadataUpdate.AddSnapshot, MetadataUpdate.AddSortOrder, MetadataUpdate.AssignUUID, MetadataUpdate.RemoveProperties, MetadataUpdate.RemoveSnapshot, MetadataUpdate.RemoveSnapshotRef, MetadataUpdate.RemoveStatistics, MetadataUpdate.SetCurrentSchema, MetadataUpdate.SetDefaultPartitionSpec, MetadataUpdate.SetDefaultSortOrder, MetadataUpdate.SetLocation, MetadataUpdate.SetProperties, MetadataUpdate.SetSnapshotRef, MetadataUpdate.SetStatistics, MetadataUpdate.UpgradeFormatVersion
+MetadataUpdate.AddPartitionSpec, MetadataUpdate.AddSchema, MetadataUpdate.AddSnapshot, MetadataUpdate.AddSortOrder, MetadataUpdate.AddViewVersion, MetadataUpdate.AssignUUID, MetadataUpdate.RemoveProperties, MetadataUpdate.RemoveSnapshot, MetadataUpdate.RemoveSnapshotRef, MetadataUpdate.RemoveStatistics, MetadataUpdate.SetCurrentSchema, MetadataUpdate.SetCurrentViewVersion, MetadataUpdate.SetDefaultPartitionSpec, MetadataUpdate.SetDefaultSortOrder, MetadataUpdate.SetLocation, MetadataUpdate.SetProperties, MetadataUpdate.SetSnapshotRef, MetadataUpdate.SetStatistics, MetadataUpdate.UpgradeFormatVersion
@@ -177,6 +177,10 @@ Method SummaryapplyTo(TableMetadata.Builder metadataBuilder) |
| Modifier and Type | Method and Description | @@ -135,24 +135,48 @@
|---|---|
default ThisT |
+fromSnapshotExclusive(java.lang.String ref)
+Instructs this scan to look for changes starting from a particular snapshot (exclusive).
+ |
+
ThisT |
fromSnapshotInclusive(long fromSnapshotId)
Instructs this scan to look for changes starting from a particular snapshot (inclusive).
|
default ThisT |
+fromSnapshotInclusive(java.lang.String ref)
+Instructs this scan to look for changes starting from a particular snapshot (inclusive).
+ |
+
ThisT |
toSnapshot(long toSnapshotId)
Instructs this scan to look for changes up to a particular snapshot (inclusive).
|
default ThisT |
+toSnapshot(java.lang.String ref)
+Instructs this scan to look for changes up to a particular snapshot ref (inclusive).
+ |
+
default ThisT |
+useBranch(java.lang.String branch)
+Use the specified branch
+ |
+
caseSensitive, filter, filter, ignoreResiduals, includeColumnStats, isCaseSensitive, option, planFiles, planTasks, planWith, project, schema, select, select, splitLookback, splitOpenFileCost, targetSplitSizecaseSensitive, filter, filter, ignoreResiduals, includeColumnStats, isCaseSensitive, metricsReporter, option, planFiles, planTasks, planWith, project, schema, select, select, splitLookback, splitOpenFileCost, targetSplitSize
ThisT fromSnapshotInclusive(long fromSnapshotId)
If the start snapshot is not configured, it is defaulted to the oldest ancestor of the end +
If the start snapshot is not configured, it defaults to the oldest ancestor of the end snapshot (inclusive).
default ThisT fromSnapshotInclusive(java.lang.String ref)+
If the start snapshot is not configured, it defaults to the oldest ancestor of the end + snapshot (inclusive).
ref - the start ref name that points to a particular snapshot ID (inclusive)java.lang.IllegalArgumentException - if the start snapshot is not an ancestor of the end snapshotThisT fromSnapshotExclusive(long fromSnapshotId)
If the start snapshot is not configured, it is defaulted to the oldest ancestor of the end +
If the start snapshot is not configured, it defaults to the oldest ancestor of the end snapshot (inclusive).
default ThisT fromSnapshotExclusive(java.lang.String ref)+
If the start snapshot is not configured, it defaults to the oldest ancestor of the end + snapshot (inclusive).
ref - the start ref name that points to a particular snapshot ID (exclusive)java.lang.IllegalArgumentException - if the start snapshot is not an ancestor of the end snapshotThisT toSnapshot(long toSnapshotId)
If the end snapshot is not configured, it is defaulted to the current table snapshot +
If the end snapshot is not configured, it defaults to the current table snapshot (inclusive).
default ThisT toSnapshot(java.lang.String ref)+
If the end snapshot is not configured, it defaults to the current table snapshot + (inclusive).
ref - the end snapshot Ref (inclusive)default ThisT useBranch(java.lang.String branch)+
branch - the branch name| Modifier and Type | Method and Description | @@ -150,20 +150,26 @@|
|---|---|---|
default ManageSnapshots |
+createBranch(java.lang.String name)
+Create a new branch.
+ |
+|
ManageSnapshots |
createBranch(java.lang.String name,
long snapshotId)
Create a new branch pointing to the given snapshot id.
|
|
ManageSnapshots |
createTag(java.lang.String name,
long snapshotId)
Create a new tag pointing to the given snapshot id
|
|
ManageSnapshots |
fastForwardBranch(java.lang.String name,
java.lang.String source)
@@ -171,79 +177,79 @@ Method Summaryancestor of source. |
|
ManageSnapshots |
removeBranch(java.lang.String name)
Remove a branch by name
|
|
ManageSnapshots |
removeTag(java.lang.String name)
Remove the tag with the given name.
|
|
ManageSnapshots |
renameBranch(java.lang.String name,
java.lang.String newName)
Rename a branch
|
|
ManageSnapshots |
replaceBranch(java.lang.String name,
long snapshotId)
Replaces the branch with the given name to point to the specified snapshot
|
|
ManageSnapshots |
replaceBranch(java.lang.String name,
java.lang.String source)
Replaces the branch with the given name to point to the source snapshot.
|
|
ManageSnapshots |
replaceTag(java.lang.String name,
long snapshotId)
Replaces the tag with the given name to point to the specified snapshot.
|
|
ManageSnapshots |
rollbackTo(long snapshotId)
Rollback table's state to a specific
Snapshot identified by id. |
|
ManageSnapshots |
rollbackToTime(long timestampMillis)
Roll this table's data back to the last
Snapshot before the given timestamp. |
|
ManageSnapshots |
setCurrentSnapshot(long snapshotId)
Roll this table's data back to a specific
Snapshot identified by id. |
|
ManageSnapshots |
setMaxRefAgeMs(java.lang.String name,
long maxRefAgeMs)
Updates the retention policy for a reference.
|
|
ManageSnapshots |
setMaxSnapshotAgeMs(java.lang.String branchName,
long maxSnapshotAgeMs)
Updates the max snapshot age for a branch.
|
|
ManageSnapshots |
setMinSnapshotsToKeep(java.lang.String branchName,
int minSnapshotsToKeep)
@@ -347,6 +353,25 @@ cherrypick+ + + +
Method Summary | |
static boolean |
-isMetadataColumn(java.lang.String name) |
+isMetadataColumn(int id) |
static boolean |
+isMetadataColumn(java.lang.String name) |
+|
static Types.NestedField |
metadataColumn(Table table,
java.lang.String name) |
|
static java.util.Set<java.lang.Integer> |
metadataFieldIds() |
|
static boolean |
nonMetadataColumn(java.lang.String name) |
applyTo(TableMetadata.Builder metadataBuilder) |
void |
+applyTo(ViewMetadata.Builder viewMetadataBuilder) |
+|
int |
lastColumnId() |
|
Schema |
schema() |
applyTo(TableMetadata.Builder metadataBuilder) |
void |
+applyTo(ViewMetadata.Builder metadataBuilder) |
+|
java.lang.String |
uuid() |
applyTo(TableMetadata.Builder metadataBuilder) |
void |
+applyTo(ViewMetadata.Builder viewMetadataBuilder) |
+|
java.util.Set<java.lang.String> |
removed() |
Constructor and Description |
SetDefaultPartitionSpec(int schemaId) |
+SetDefaultPartitionSpec(int specId) |
clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait
+applyTopublic SetDefaultPartitionSpec(int schemaId)+
public SetDefaultPartitionSpec(int specId)
voidapplyTo(ViewMetadata.Builder viewMetadataBuilder) java.lang.Stringlocation() public void applyTo(TableMetadata.Builder metadataBuilder)@@ -241,6 +245,19 @@
public void applyTo(ViewMetadata.Builder viewMetadataBuilder)+
applyTo in interface MetadataUpdateMetadataUpdate.AddPartitionSpec, MetadataUpdate.AddSchema, MetadataUpdate.AddSnapshot, MetadataUpdate.AddSortOrder, MetadataUpdate.AssignUUID, MetadataUpdate.RemoveProperties, MetadataUpdate.RemoveSnapshot, MetadataUpdate.RemoveSnapshotRef, MetadataUpdate.RemoveStatistics, MetadataUpdate.SetCurrentSchema, MetadataUpdate.SetDefaultPartitionSpec, MetadataUpdate.SetDefaultSortOrder, MetadataUpdate.SetLocation, MetadataUpdate.SetProperties, MetadataUpdate.SetSnapshotRef, MetadataUpdate.SetStatistics, MetadataUpdate.UpgradeFormatVersion
+MetadataUpdate.AddPartitionSpec, MetadataUpdate.AddSchema, MetadataUpdate.AddSnapshot, MetadataUpdate.AddSortOrder, MetadataUpdate.AddViewVersion, MetadataUpdate.AssignUUID, MetadataUpdate.RemoveProperties, MetadataUpdate.RemoveSnapshot, MetadataUpdate.RemoveSnapshotRef, MetadataUpdate.RemoveStatistics, MetadataUpdate.SetCurrentSchema, MetadataUpdate.SetCurrentViewVersion, MetadataUpdate.SetDefaultPartitionSpec, MetadataUpdate.SetDefaultSortOrder, MetadataUpdate.SetLocation, MetadataUpdate.SetProperties, MetadataUpdate.SetSnapshotRef, MetadataUpdate.SetStatistics, MetadataUpdate.UpgradeFormatVersion
@@ -177,6 +177,10 @@ applyTo(TableMetadata.Builder metadataBuilder) voidapplyTo(ViewMetadata.Builder viewMetadataBuilder) java.util.Map<java.lang.String,java.lang.String>updated() public void applyTo(TableMetadata.Builder metadataBuilder)@@ -241,6 +245,19 @@
public void applyTo(ViewMetadata.Builder viewMetadataBuilder)+
applyTo in interface MetadataUpdateMetadataUpdate.AddPartitionSpec, MetadataUpdate.AddSchema, MetadataUpdate.AddSnapshot, MetadataUpdate.AddSortOrder, MetadataUpdate.AssignUUID, MetadataUpdate.RemoveProperties, MetadataUpdate.RemoveSnapshot, MetadataUpdate.RemoveSnapshotRef, MetadataUpdate.RemoveStatistics, MetadataUpdate.SetCurrentSchema, MetadataUpdate.SetDefaultPartitionSpec, MetadataUpdate.SetDefaultSortOrder, MetadataUpdate.SetLocation, MetadataUpdate.SetProperties, MetadataUpdate.SetSnapshotRef, MetadataUpdate.SetStatistics, MetadataUpdate.UpgradeFormatVersion
+MetadataUpdate.AddPartitionSpec, MetadataUpdate.AddSchema, MetadataUpdate.AddSnapshot, MetadataUpdate.AddSortOrder, MetadataUpdate.AddViewVersion, MetadataUpdate.AssignUUID, MetadataUpdate.RemoveProperties, MetadataUpdate.RemoveSnapshot, MetadataUpdate.RemoveSnapshotRef, MetadataUpdate.RemoveStatistics, MetadataUpdate.SetCurrentSchema, MetadataUpdate.SetCurrentViewVersion, MetadataUpdate.SetDefaultPartitionSpec, MetadataUpdate.SetDefaultSortOrder, MetadataUpdate.SetLocation, MetadataUpdate.SetProperties, MetadataUpdate.SetSnapshotRef, MetadataUpdate.SetStatistics, MetadataUpdate.UpgradeFormatVersion
@@ -213,6 +213,13 @@ clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait
+applyToMetadataUpdate.AddPartitionSpec, MetadataUpdate.AddSchema, MetadataUpdate.AddSnapshot, MetadataUpdate.AddSortOrder, MetadataUpdate.AssignUUID, MetadataUpdate.RemoveProperties, MetadataUpdate.RemoveSnapshot, MetadataUpdate.RemoveSnapshotRef, MetadataUpdate.RemoveStatistics, MetadataUpdate.SetCurrentSchema, MetadataUpdate.SetDefaultPartitionSpec, MetadataUpdate.SetDefaultSortOrder, MetadataUpdate.SetLocation, MetadataUpdate.SetProperties, MetadataUpdate.SetSnapshotRef, MetadataUpdate.SetStatistics, MetadataUpdate.UpgradeFormatVersion
+MetadataUpdate.AddPartitionSpec, MetadataUpdate.AddSchema, MetadataUpdate.AddSnapshot, MetadataUpdate.AddSortOrder, MetadataUpdate.AddViewVersion, MetadataUpdate.AssignUUID, MetadataUpdate.RemoveProperties, MetadataUpdate.RemoveSnapshot, MetadataUpdate.RemoveSnapshotRef, MetadataUpdate.RemoveStatistics, MetadataUpdate.SetCurrentSchema, MetadataUpdate.SetCurrentViewVersion, MetadataUpdate.SetDefaultPartitionSpec, MetadataUpdate.SetDefaultSortOrder, MetadataUpdate.SetLocation, MetadataUpdate.SetProperties, MetadataUpdate.SetSnapshotRef, MetadataUpdate.SetStatistics, MetadataUpdate.UpgradeFormatVersion
@@ -193,6 +193,13 @@ clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait
+applyToMetadataUpdate.AddPartitionSpec, MetadataUpdate.AddSchema, MetadataUpdate.AddSnapshot, MetadataUpdate.AddSortOrder, MetadataUpdate.AssignUUID, MetadataUpdate.RemoveProperties, MetadataUpdate.RemoveSnapshot, MetadataUpdate.RemoveSnapshotRef, MetadataUpdate.RemoveStatistics, MetadataUpdate.SetCurrentSchema, MetadataUpdate.SetDefaultPartitionSpec, MetadataUpdate.SetDefaultSortOrder, MetadataUpdate.SetLocation, MetadataUpdate.SetProperties, MetadataUpdate.SetSnapshotRef, MetadataUpdate.SetStatistics, MetadataUpdate.UpgradeFormatVersion
+MetadataUpdate.AddPartitionSpec, MetadataUpdate.AddSchema, MetadataUpdate.AddSnapshot, MetadataUpdate.AddSortOrder, MetadataUpdate.AddViewVersion, MetadataUpdate.AssignUUID, MetadataUpdate.RemoveProperties, MetadataUpdate.RemoveSnapshot, MetadataUpdate.RemoveSnapshotRef, MetadataUpdate.RemoveStatistics, MetadataUpdate.SetCurrentSchema, MetadataUpdate.SetCurrentViewVersion, MetadataUpdate.SetDefaultPartitionSpec, MetadataUpdate.SetDefaultSortOrder, MetadataUpdate.SetLocation, MetadataUpdate.SetProperties, MetadataUpdate.SetSnapshotRef, MetadataUpdate.SetStatistics, MetadataUpdate.UpgradeFormatVersion
@@ -177,6 +177,10 @@ applyTo(TableMetadata.Builder metadataBuilder) voidapplyTo(ViewMetadata.Builder viewMetadataBuilder) intformatVersion() public void applyTo(TableMetadata.Builder metadataBuilder)@@ -241,6 +245,19 @@
public void applyTo(ViewMetadata.Builder viewMetadataBuilder)+
applyTo in interface MetadataUpdatepublic interface MetadataUpdate
extends java.io.Serializable
-static class MetadataUpdate.AssignUUID MetadataUpdate.AddViewVersion static class MetadataUpdate.RemoveProperties MetadataUpdate.AssignUUID static class MetadataUpdate.RemoveSnapshot MetadataUpdate.RemoveProperties static class MetadataUpdate.RemoveSnapshotRef MetadataUpdate.RemoveSnapshot static class MetadataUpdate.RemoveStatistics MetadataUpdate.RemoveSnapshotRef static class MetadataUpdate.RemoveStatistics static class MetadataUpdate.SetCurrentSchema static class MetadataUpdate.SetCurrentViewVersion static class MetadataUpdate.SetDefaultPartitionSpec | Modifier and Type | Method and Description | |
|---|---|---|
void |
+default void |
applyTo(TableMetadata.Builder metadataBuilder) |
default void |
+applyTo(ViewMetadata.Builder viewMetadataBuilder) |
+
default void applyTo(TableMetadata.Builder metadataBuilder)+
void applyTo(TableMetadata.Builder metadataBuilder)+
default void applyTo(ViewMetadata.Builder viewMetadataBuilder)
static MetricscopyWithoutFieldCounts(Metrics metrics,
+ java.util.Set<java.lang.Integer> excludedFieldIds)
+static MetricscopyWithoutFieldCountsAndBounds(Metrics metrics,
+ java.util.Set<java.lang.Integer> excludedFieldIds)
+static java.util.Map<java.lang.Integer,java.lang.Long>createNanValueCounts(java.util.stream.Stream<FieldMetrics<?>> fieldMetrics,
MetricsConfig metricsConfig,
@@ -193,7 +207,7 @@ Method Summary
metrics config.
static MetricsModes.MetricsModemetricsMode(Schema inputSchema,
MetricsConfig metricsConfig,
@@ -201,14 +215,14 @@ Method Summary
Extract MetricsMode for the given field id from metrics config.
static SchemareadableMetricsSchema(Schema dataTableSchema,
Schema metadataTableSchema)
static MetricsUtil.ReadableMetricsStructreadableMetricsStruct(Schema schema,
ContentFile<?> file,
@@ -268,6 +282,40 @@ READABLE_METRICS
Method Detail
+
+
+
+
+-
+
copyWithoutFieldCounts
+public static Metrics copyWithoutFieldCounts(Metrics metrics,
+ java.util.Set<java.lang.Integer> excludedFieldIds)
+Copies a metrics object without value, NULL and NaN counts for given fields.
+
+- Parameters:
+excludedFieldIds - field IDs for which the counts must be dropped- Returns:
+- a new metrics object without counts for given fields
+
+
+
+
+
+
+
+-
+
copyWithoutFieldCountsAndBounds
+public static Metrics copyWithoutFieldCountsAndBounds(Metrics metrics,
+ java.util.Set<java.lang.Integer> excludedFieldIds)
+Copies a metrics object without counts and bounds for given fields.
+
+- Parameters:
+excludedFieldIds - field IDs for which the counts and bounds must be dropped- Returns:
+- a new metrics object without lower and upper bounds for given fields
+
+
+
diff --git a/javadoc/org/apache/iceberg/MicroBatches.MicroBatchBuilder.html b/javadoc/org/apache/iceberg/MicroBatches.MicroBatchBuilder.html
index 51fbc845c..3d84d57ac 100644
--- a/javadoc/org/apache/iceberg/MicroBatches.MicroBatchBuilder.html
+++ b/javadoc/org/apache/iceberg/MicroBatches.MicroBatchBuilder.html
@@ -17,7 +17,7 @@
catch(err) {
}
//-->
-var methods = {"i0":10,"i1":10,"i2":10};
+var methods = {"i0":10,"i1":10,"i2":10,"i3":10};
var tabs = {65535:["t0","All Methods"],2:["t2","Instance Methods"],8:["t4","Concrete Methods"]};
var altColor = "altColor";
var rowColor = "rowColor";
@@ -142,6 +142,13 @@ Method Summary
boolean scanAllFiles) MicroBatches.MicroBatchgenerate(long startFileIndex,
+ long endFileIndex,
+ long targetSizeInBytes,
+ boolean scanAllFiles) MicroBatches.MicroBatchBuilderspecsById(java.util.Map<java.lang.Integer,PartitionSpec> specs) public MicroBatches.MicroBatch generate(long startFileIndex, + long targetSizeInBytes, + boolean scanAllFiles)+
public MicroBatches.MicroBatch generate(long startFileIndex, + long endFileIndex, long targetSizeInBytes, boolean scanAllFiles)
from(Snapshot snapshot,
FileIO io) static CloseableIterable<FileScanTask>openManifestFile(FileIO io,
+ java.util.Map<java.lang.Integer,PartitionSpec> specsById,
+ boolean caseSensitive,
+ Snapshot snapshot,
+ ManifestFile manifestFile,
+ boolean scanAllFiles) static java.util.List<Pair<ManifestFile,java.lang.Integer>>skippedManifestIndexesFromSnapshot(FileIO io,
+ Snapshot snapshot,
+ long startFileIndex,
+ boolean scanAllFiles) public static java.util.List<Pair<ManifestFile,java.lang.Integer>> skippedManifestIndexesFromSnapshot(FileIO io, + Snapshot snapshot, + long startFileIndex, + boolean scanAllFiles)+
public static CloseableIterable<FileScanTask> openManifestFile(FileIO io, + java.util.Map<java.lang.Integer,PartitionSpec> specsById, + boolean caseSensitive, + Snapshot snapshot, + ManifestFile manifestFile, + boolean scanAllFiles)+
| Modifier and Type | Method and Description |
|---|---|
static Types.StructType |
-groupingKeyType(java.util.Collection<PartitionSpec> specs)
-Deprecated.
-
-use
-groupingKeyType(Schema, Collection) instead; will be removed in 1.3.0 |
-
static Types.StructType |
groupingKeyType(Schema schema,
java.util.Collection<PartitionSpec> specs)
Builds a grouping key type considering the provided schema and specs.
|
static boolean |
hasBucketField(PartitionSpec spec)
Check whether the spec contains a bucketed partition field.
|
static Types.StructType |
partitionType(Table table)
Builds a unified partition type considering all specs in a table.
|
static SortOrder |
sortOrderFor(PartitionSpec spec)
Create a sort order that will group data for a partition spec.
@@ -218,24 +210,6 @@ sortOrderFor- - - -
|
protected static boolean |
+PLAN_SCANS_WITH_WORKER_POOL |
+
protected static java.util.List<java.lang.String> |
+SCAN_COLUMNS |
+
protected static java.util.List<java.lang.String> |
+SCAN_WITH_STATS_COLUMNS |
+
protected PositionDeletesBatchScan(Table table,
Schema schema,
- org.apache.iceberg.TableScanContext context) protected PositionDeletesBatchScan(Table table,
+ Schema schema,
+ org.apache.iceberg.TableScanContext context,
+ Expression baseTableFilter) | Modifier and Type | Method and Description |
|---|---|
BatchScan |
+baseTableFilter(Expression expr)
+Sets a filter that applies on base table of this position deletes table, to use for this
+ scan.
+ |
+
ThisT |
caseSensitive(boolean caseSensitive)
Create a new scan from this that, if data columns where selected via
Scan.select(java.util.Collection), controls whether the match to the schema will be done with case
sensitivity. |
protected org.apache.iceberg.TableScanContext |
context() |
protected CloseableIterable<ScanTask> |
doPlanFiles() |
Expression |
filter()
Returns this scan's filter
Expression. |
ThisT |
filter(Expression expr)
Create a new scan from the results of this filtered by the
Expression. |
ThisT |
ignoreResiduals()
Create a new scan from this that applies data filtering to files but not to rows in those
files.
|
ThisT |
includeColumnStats()
Create a new scan from this that loads the column stats with each data file.
|
protected FileIO |
+io() |
+
boolean |
isCaseSensitive()
Returns whether this scan is case-sensitive with respect to column names.
|
protected ThisT |
-newRefinedScan(TableOperations ignored,
- Table newTable,
- Schema newSchema,
- org.apache.iceberg.TableScanContext newContext)
-Deprecated.
-
+will be removed in 1.3.0; use newRefinedScan(Table, Schema, TableScanContext)
- instead.
- |
ThisT |
+metricsReporter(MetricsReporter reporter)
+Create a new scan that will report scan metrics to the provided reporter in addition to
+ reporters maintained by the scan.
|
protected PositionDeletesTable.PositionDeletesBatchScan |
newRefinedScan(Table newTable,
Schema newSchema,
org.apache.iceberg.TableScanContext newContext) |
ThisT |
option(java.lang.String property,
java.lang.String value)
@@ -262,85 +291,85 @@ Method Summarybehavior based on the incoming pair. |
protected java.util.Map<java.lang.String,java.lang.String> |
options() |
protected java.util.concurrent.ExecutorService |
planExecutor() |
CloseableIterable<ScanTaskGroup<ScanTask>> |
planTasks()
Plan balanced task groups for this scan by splitting large and combining small tasks.
|
ThisT |
planWith(java.util.concurrent.ExecutorService executorService)
Create a new scan to use a particular executor to plan.
|
ThisT |
project(Schema projectedSchema)
Create a new scan from this with the schema as its projection.
|
protected Expression |
+residualFilter() |
+
protected java.util.List<java.lang.String> |
scanColumns() |
Schema |
schema()
Returns this scan's projection
Schema. |
ThisT |
select(java.util.Collection<java.lang.String> columns)
Create a new scan from this that will read the given data columns.
|
protected boolean |
shouldIgnoreResiduals() |
protected boolean |
shouldPlanWithExecutor() |
protected boolean |
+shouldReturnColumnStats() |
+
int |
splitLookback()
Returns the split lookback for this scan.
|
long |
splitOpenFileCost()
Returns the split open file cost for this scan.
|
Table |
table() |
protected TableOperations |
-tableOps()
-Deprecated.
-
-will be removed in 1.3.0; avoid using TableOperations for scans or use BaseTable
- |
-
protected Schema |
tableSchema() |
long |
targetSplitSize()
Returns the target split size for this scan.
@@ -352,7 +381,7 @@ Method SummaryMethods inherited from class org.apache.iceberg.SnapshotScan-asOfTime, planFiles, scanMetrics, snapshot, snapshotId, toString, useRef, useSnapshot
+asOfTime, planFiles, scanMetrics, snapshot, snapshotId, toString, useRef, useSnapshot, useSnapshotSchema
caseSensitive, filter, filter, ignoreResiduals, includeColumnStats, isCaseSensitive, metricsReporter, option, planFiles, planWith, project, schema, select, select, splitLookback, splitOpenFileCost, targetSplitSize
Methods inherited from interface org.apache.iceberg.
+
+
+SCAN_COLUMNS+protected static final java.util.List<java.lang.String> SCAN_COLUMNS+ SCAN_WITH_STATS_COLUMNS+protected static final java.util.List<java.lang.String> SCAN_WITH_STATS_COLUMNS+
PositionDeletesBatchScan+@Deprecated +protected PositionDeletesBatchScan(Table table, + Schema schema, + org.apache.iceberg.TableScanContext context)+ Deprecated. the API will be removed in v1.5.0
+PositionDeletesBatchScanprotected PositionDeletesBatchScan(Table table, Schema schema, - org.apache.iceberg.TableScanContext context)+ org.apache.iceberg.TableScanContext context, + Expression baseTableFilter) baseTableFilter+public BatchScan baseTableFilter(Expression expr)+ Sets a filter that applies on base table of this position deletes table, to use for this
+ scan.
+
+
+Only the partition expressions part of the filter will be applied to the position deletes + table, as the schema of the base table does not otherwise match the schema of position + deletes table. + +
tableOps-@Deprecated -protected TableOperations tableOps()- Deprecated. will be removed in 1.3.0; avoid using TableOperations for scans or use BaseTable
+table+public Table table() shouldReturnColumnStats+protected boolean shouldReturnColumnStats()+ shouldPlanWithExecutor-protected boolean shouldPlanWithExecutor()+ residualFilter+protected Expression residualFilter() planExecutor-protected java.util.concurrent.ExecutorService planExecutor()+ shouldPlanWithExecutor+protected boolean shouldPlanWithExecutor() newRefinedScan-@Deprecated -protected ThisT newRefinedScan(TableOperations ignored, - Table newTable, - Schema newSchema, - org.apache.iceberg.TableScanContext newContext)- Deprecated. will be removed in 1.3.0; use newRefinedScan(Table, Schema, TableScanContext)
- instead.
+planExecutor+protected java.util.concurrent.ExecutorService planExecutor()
|
| Modifier and Type | +Field and Description | +
|---|---|
static java.lang.String |
+DELETE_FILE_PATH |
+
static java.lang.String |
+PARTITION |
+
static java.lang.String |
+SPEC_ID |
+
java.util.Map<java.lang.String,java.lang.String>properties()
+ReplaceSortOrderreplaceSortOrder()
ReplaceSortOrder to set the table sort order and commit the change.RewriteManifestsrewriteManifests()
rewrite manifests API to replace manifests for this table
and commit.Schemaschema()
schema for this table.PartitionSpecspec()
partition spec for this table.java.util.Map<java.lang.Integer,PartitionSpec>specs()
partition specs for this table.UpdateLocationupdateLocation()
UpdateLocation to update table location and commit the changes.UpdatePropertiesupdateProperties()
UpdateProperties to update table properties and commit the changes.UpdateSchemaupdateSchema()
UpdateSchema to alter the columns of this table and commit the change.UpdatePartitionSpecupdateSpec()
UpdatePartitionSpec to alter the partition spec of this table and commit
the change.UpdateStatisticsupdateStatistics()
update table statistics API to add or remove statistics
@@ -299,7 +332,7 @@ currentSnapshot, encryption, history, io, location, locationProvider, name, operations, properties, refresh, refs, schemas, snapshot, snapshots, sortOrder, sortOrders, statisticsFiles, table, toString
+currentSnapshot, encryption, history, io, location, locationProvider, name, operations, refresh, refs, schemas, snapshot, snapshots, sortOrder, sortOrders, statisticsFiles, table, toString
public static final java.lang.String PARTITION+
public static final java.lang.String SPEC_ID+
public static final java.lang.String DELETE_FILE_PATH+
public java.util.Map<java.lang.String,java.lang.String> properties()+
Tableproperties in interface Tableproperties in class BaseMetadataTableasc, asc, asc, asc, desc, desc, desc, descasc, asc, asc, asc, caseSensitive, desc, desc, desc, descWhen committing, these changes will be applied to the latest table snapshot. Commit conflicts
will be resolved by applying the changes to the new latest snapshot and reattempting the commit.
If any of the deleted files are no longer in the latest snapshot when reattempting, the commit
- will throw a ValidationException.
ValidationException.
+
+ Note that the new state of the table after each rewrite must be logically equivalent to the + original table state. @@ -127,36 +130,79 @@
| Modifier and Type | Method and Description | ||
|---|---|---|---|
default RewriteFiles |
+addFile(DataFile dataFile)
+Add a new data file.
+ |
+||
default RewriteFiles |
+addFile(DeleteFile deleteFile)
+Add a new delete file.
+ |
+||
default RewriteFiles |
+addFile(DeleteFile deleteFile,
+ long dataSequenceNumber)
+Add a new delete file with the given data sequence number.
+ |
+||
default RewriteFiles |
+dataSequenceNumber(long sequenceNumber)
+Configure the data sequence number for this rewrite operation.
+ |
+||
default RewriteFiles |
+deleteFile(DataFile dataFile)
+Remove a data file from the current table state.
+ |
+||
default RewriteFiles |
+deleteFile(DeleteFile deleteFile)
+Remove a delete file from the table state.
+ |
+||
default RewriteFiles |
rewriteFiles(java.util.Set<DataFile> filesToDelete,
java.util.Set<DataFile> filesToAdd)
-Add a rewrite that replaces one set of data files with another set that contains the same data.
+Deprecated.
+
since 1.3.0, will be removed in 2.0.0
+ |
||
RewriteFiles |
rewriteFiles(java.util.Set<DataFile> filesToDelete,
java.util.Set<DataFile> filesToAdd,
long sequenceNumber)
-Add a rewrite that replaces one set of data files with another set that contains the same data.
+Deprecated.
+
since 1.3.0, will be removed in 2.0.0
+ |
||
RewriteFiles |
rewriteFiles(java.util.Set<DataFile> dataFilesToReplace,
java.util.Set<DeleteFile> deleteFilesToReplace,
java.util.Set<DataFile> dataFilesToAdd,
java.util.Set<DeleteFile> deleteFilesToAdd)
-Add a rewrite that replaces one set of files with another set that contains the same data.
+Deprecated.
+
since 1.3.0, will be removed in 2.0.0
+ |
||
RewriteFiles |
validateFromSnapshot(long snapshotId)
Set the snapshot ID used in any reads for this operation.
@@ -191,14 +237,140 @@ Methods inherited from interface org.apache.iceberg.
+
+
+deleteFile+default RewriteFiles deleteFile(DataFile dataFile)+ Remove a data file from the current table state.
+
+
+This rewrite operation may change the size or layout of the data files. When applicable, it + is also recommended to discard already deleted records while rewriting data files. However, the + set of live data records must never change.
deleteFile+default RewriteFiles deleteFile(DeleteFile deleteFile)+ Remove a delete file from the table state.
+
+
+This rewrite operation may change the size or layout of the delete files. When applicable, + it is also recommended to discard delete records for files that are no longer part of the table + state. However, the set of applicable delete records must never change.
addFile+default RewriteFiles addFile(DataFile dataFile)+ Add a new data file.
+
+
+This rewrite operation may change the size or layout of the data files. When applicable, it + is also recommended to discard already deleted records while rewriting data files. However, the + set of live data records must never change.
addFile+default RewriteFiles addFile(DeleteFile deleteFile)+ Add a new delete file.
+
+
+This rewrite operation may change the size or layout of the delete files. When applicable, + it is also recommended to discard delete records for files that are no longer part of the table + state. However, the set of applicable delete records must never change.
addFile+default RewriteFiles addFile(DeleteFile deleteFile, + long dataSequenceNumber)+ Add a new delete file with the given data sequence number.
+
+
+This rewrite operation may change the size or layout of the delete files. When applicable, + it is also recommended to discard delete records for files that are no longer part of the table + state. However, the set of applicable delete records must never change. + + To ensure equivalence in the set of applicable delete records, the sequence number of the + delete file must be the max sequence number of the delete files that it is replacing. Rewriting + equality deletes that belong to different sequence numbers is not allowed.
dataSequenceNumber+default RewriteFiles dataSequenceNumber(long sequenceNumber)+ Configure the data sequence number for this rewrite operation. This data sequence number will
+ be used for all new data files that are added in this rewrite. This method is helpful to avoid
+ commit conflicts between data compaction and adding equality deletes.
+
|
||
default ThisT |
+metricsReporter(MetricsReporter reporter)
+Create a new scan that will report scan metrics to the provided reporter in addition to
+ reporters maintained by the scan.
+ |
+||
ThisT |
option(java.lang.String property,
java.lang.String value)
@@ -179,61 +186,61 @@ Method Summarybehavior based on the incoming pair. |
||
CloseableIterable<T> |
planFiles()
Plan tasks for this scan where each task reads a single file.
|
||
CloseableIterable<G> |
planTasks()
Plan balanced task groups for this scan by splitting large and combining small tasks.
|
||
ThisT |
planWith(java.util.concurrent.ExecutorService executorService)
Create a new scan to use a particular executor to plan.
|
||
ThisT |
project(Schema schema)
Create a new scan from this with the schema as its projection.
|
||
Schema |
schema()
Returns this scan's projection
Schema. |
||
ThisT |
select(java.util.Collection<java.lang.String> columns)
Create a new scan from this that will read the given data columns.
|
||
default ThisT |
select(java.lang.String... columns)
Create a new scan from this that will read the given columns.
|
||
int |
splitLookback()
Returns the split lookback for this scan.
|
||
long |
splitOpenFileCost()
Returns the split open file cost for this scan.
|
||
long |
targetSplitSize()
Returns the target split size for this scan.
@@ -509,13 +516,24 @@ splitLookback-
addedDataFilesReturn all data files added to the table in this snapshot.
+ record_count, and file_size_in_bytes. Data and file sequence number are populated. Other
+ columns will be null.
The files returned include the following columns: file_path, file_format, partition, - record_count, and file_size_in_bytes. Other columns will be null.
removedDataFilesReturn all data files removed from the table in this snapshot.
+ record_count, and file_size_in_bytes. Data and file sequence number are populated. Other
+ columns will be null.
The files returned include the following columns: file_path, file_format, partition, - record_count, and file_size_in_bytes. Other columns will be null. Method Summary | ||
ManageSnapshots |
+createBranch(java.lang.String name)
+Create a new branch.
+ |
+||
ManageSnapshots |
createBranch(java.lang.String name,
long snapshotId)
Create a new branch pointing to the given snapshot id.
|
||
ManageSnapshots |
createTag(java.lang.String name,
long snapshotId)
Create a new tag pointing to the given snapshot id
|
||
ManageSnapshots |
fastForwardBranch(java.lang.String name,
java.lang.String source)
@@ -173,79 +179,79 @@ Method Summaryancestor of source. |
||
ManageSnapshots |
removeBranch(java.lang.String name)
Remove a branch by name
|
||
ManageSnapshots |
removeTag(java.lang.String name)
Remove the tag with the given name.
|
||
ManageSnapshots |
renameBranch(java.lang.String name,
java.lang.String newName)
Rename a branch
|
||
ManageSnapshots |
replaceBranch(java.lang.String name,
long snapshotId)
Replaces the branch with the given name to point to the specified snapshot
|
||
ManageSnapshots |
replaceBranch(java.lang.String name,
java.lang.String source)
Replaces the branch with the given name to point to the source snapshot.
|
||
ManageSnapshots |
replaceTag(java.lang.String name,
long snapshotId)
Replaces the tag with the given name to point to the specified snapshot.
|
||
ManageSnapshots |
rollbackTo(long snapshotId)
Rollback table's state to a specific
Snapshot identified by id. |
||
ManageSnapshots |
rollbackToTime(long timestampMillis)
Roll this table's data back to the last
Snapshot before the given timestamp. |
||
ManageSnapshots |
setCurrentSnapshot(long snapshotId)
Roll this table's data back to a specific
Snapshot identified by id. |
||
ManageSnapshots |
setMaxRefAgeMs(java.lang.String name,
long maxRefAgeMs)
Updates the retention policy for a reference.
|
||
ManageSnapshots |
setMaxSnapshotAgeMs(java.lang.String name,
long maxSnapshotAgeMs)
Updates the max snapshot age for a branch.
|
||
ManageSnapshots |
setMinSnapshotsToKeep(java.lang.String name,
int minSnapshotsToKeep)
@@ -357,6 +363,26 @@ rollbackTo+ + + +
Class SnapshotScan<ThisT,T exten
protected static java.util.List<java.lang.String>DELETE_SCAN_WITH_STATS_COLUMNS | ||
protected static boolean |
+PLAN_SCANS_WITH_WORKER_POOL |
+||
protected static java.util.List<java.lang.String> |
+SCAN_COLUMNS |
+||
protected static java.util.List<java.lang.String> |
+SCAN_WITH_STATS_COLUMNS |
+
| Modifier and Type | Method and Description | @@ -232,30 +244,29 @@
|---|---|
protected FileIO |
+io() |
+
boolean |
isCaseSensitive()
Returns whether this scan is case-sensitive with respect to column names.
|
protected ThisT |
-newRefinedScan(TableOperations ignored,
- Table newTable,
- Schema newSchema,
- org.apache.iceberg.TableScanContext newContext)
-Deprecated.
-
+will be removed in 1.3.0; use newRefinedScan(Table, Schema, TableScanContext)
- instead.
- |
ThisT |
+metricsReporter(MetricsReporter reporter)
+Create a new scan that will report scan metrics to the provided reporter in addition to
+ reporters maintained by the scan.
|
protected abstract ThisT |
newRefinedScan(Table newTable,
Schema newSchema,
org.apache.iceberg.TableScanContext newContext) |
ThisT |
option(java.lang.String property,
java.lang.String value)
@@ -263,114 +274,118 @@ Method Summarybehavior based on the incoming pair. |
protected java.util.Map<java.lang.String,java.lang.String> |
options() |
protected java.util.concurrent.ExecutorService |
planExecutor() |
CloseableIterable<T> |
planFiles()
Plan tasks for this scan where each task reads a single file.
|
ThisT |
planWith(java.util.concurrent.ExecutorService executorService)
Create a new scan to use a particular executor to plan.
|
ThisT |
project(Schema projectedSchema)
Create a new scan from this with the schema as its projection.
|
protected Expression |
+residualFilter() |
+
protected java.util.List<java.lang.String> |
scanColumns() |
protected ScanMetrics |
scanMetrics() |
Schema |
schema()
Returns this scan's projection
Schema. |
ThisT |
select(java.util.Collection<java.lang.String> columns)
Create a new scan from this that will read the given data columns.
|
protected boolean |
shouldIgnoreResiduals() |
protected boolean |
shouldPlanWithExecutor() |
protected boolean |
+shouldReturnColumnStats() |
+
Snapshot |
snapshot() |
protected java.lang.Long |
snapshotId() |
int |
splitLookback()
Returns the split lookback for this scan.
|
long |
splitOpenFileCost()
Returns the split open file cost for this scan.
|
Table |
table() |
protected TableOperations |
-tableOps()
-Deprecated.
-
-will be removed in 1.3.0; avoid using TableOperations for scans or use BaseTable
- |
-
protected Schema |
tableSchema() |
long |
targetSplitSize()
Returns the target split size for this scan.
|
java.lang.String |
toString() |
ThisT |
useRef(java.lang.String name) |
ThisT |
useSnapshot(long scanSnapshotId) |
protected boolean |
+useSnapshotSchema() |
+
protected static final java.util.List<java.lang.String> SCAN_COLUMNS+
protected static final java.util.List<java.lang.String> SCAN_WITH_STATS_COLUMNS+
protected static final java.util.List<java.lang.String> DELETE_SCAN_WITH_STATS_COLUMNS
protected static final boolean PLAN_SCANS_WITH_WORKER_POOL+
protected abstract CloseableIterable<T> doPlanFiles()+ + + +
protected boolean useSnapshotSchema()+
@Deprecated -protected TableOperations tableOps()-
public Table table()
protected java.util.List<java.lang.String> scanColumns()+ + + +
protected boolean shouldReturnColumnStats()+
protected boolean shouldIgnoreResiduals()- +
protected boolean shouldPlanWithExecutor()+
protected Expression residualFilter()
protected java.util.concurrent.ExecutorService planExecutor()+
protected boolean shouldPlanWithExecutor()
@Deprecated -protected ThisT newRefinedScan(TableOperations ignored, - Table newTable, - Schema newSchema, - org.apache.iceberg.TableScanContext newContext)-
protected java.util.concurrent.ExecutorService planExecutor()
public long splitOpenFileCost()@@ -907,6 +959,22 @@
public ThisT metricsReporter(MetricsReporter reporter)+
ScanmetricsReporter in interface Scan<ThisT,T extends ScanTask,G extends ScanTaskGroup<T>>SortOrder.BuildercaseSensitive(boolean sortCaseSensitive) caseSensitive(boolean sortCaseSensitive)
+SortOrder.Builderpublic SortOrder.Builder caseSensitive(boolean sortCaseSensitive)+
SortOrderBuildercaseSensitive in interface SortOrderBuilder<SortOrder.Builder>sortCaseSensitive - when true, column name resolution is case-sensitivedefault RcaseSensitive(boolean caseSensitive)
+default Rdesc(java.lang.String name)
default Rdesc(java.lang.String name,
NullOrder nullOrder)
default Rdesc(Term term)
Rdesc(Term term,
NullOrder nullOrder)
@@ -315,7 +321,7 @@ default R caseSensitive(boolean caseSensitive)+
caseSensitive - when true, column name resolution is case-sensitivestatic TableMetadatafromJson(com.fasterxml.jackson.databind.JsonNode node) fromJson(InputFile file,
+ com.fasterxml.jackson.databind.JsonNode node) static TableMetadatafromJson(com.fasterxml.jackson.databind.JsonNode node) static TableMetadatafromJson(java.lang.String json)
static TableMetadatafromJson(java.lang.String metadataLocation,
+ com.fasterxml.jackson.databind.JsonNode node) static TableMetadatafromJson(java.lang.String metadataLocation,
java.lang.String json)
static java.lang.StringgetFileExtension(java.lang.String codecName) static java.lang.StringgetFileExtension(TableMetadataParser.Codec codec) static java.lang.StringgetOldFileExtension(TableMetadataParser.Codec codec) static voidinternalWrite(TableMetadata metadata,
OutputFile outputFile,
boolean overwrite) static voidoverwrite(TableMetadata metadata,
OutputFile outputFile) static TableMetadataread(FileIO io,
InputFile file) static TableMetadataread(FileIO io,
java.lang.String path) static java.lang.StringtoJson(TableMetadata metadata) static voidtoJson(TableMetadata metadata,
com.fasterxml.jackson.core.JsonGenerator generator) static voidwrite(TableMetadata metadata,
OutputFile outputFile) public static TableMetadata fromJson(InputFile file, + com.fasterxml.jackson.databind.JsonNode node)+
public static TableMetadata fromJson(com.fasterxml.jackson.databind.JsonNode node)
public static TableMetadata fromJson(java.lang.String metadataLocation, + com.fasterxml.jackson.databind.JsonNode node)+
default booleanrequireStrictCleanup()
+CleanableFailure exception.default TableOperationstemp(TableMetadata uncommittedMetadata)
TableOperations instance that uses configuration from uncommitted
@@ -329,7 +335,7 @@ default long newSnapshotId()@@ -340,6 +346,18 @@
default boolean requireStrictCleanup()+
CleanableFailure exception.
+
+ This defaults to true: cleanup will only occur for exceptions marked as CleanableFailure
static java.lang.StringADAPTIVE_SPLIT_SIZE_ENABLED static booleanADAPTIVE_SPLIT_SIZE_ENABLED_DEFAULT static java.lang.StringAVRO_COMPRESSION static java.lang.StringDEFAULT_FILE_FORMAT DATA_PLANNING_MODE static java.lang.StringDEFAULT_FILE_FORMAT_DEFAULT DEFAULT_FILE_FORMAT static java.lang.StringDEFAULT_NAME_MAPPING DEFAULT_FILE_FORMAT_DEFAULT static java.lang.StringDEFAULT_NAME_MAPPING static java.lang.StringDEFAULT_PARTITION_SPEC
static java.lang.StringDEFAULT_SORT_ORDER
static java.lang.StringDEFAULT_WRITE_METRICS_MODE static java.lang.StringDEFAULT_WRITE_METRICS_MODE_DEFAULT static java.lang.StringDELETE_AVRO_COMPRESSION static java.lang.StringDELETE_AVRO_COMPRESSION_LEVEL static java.lang.StringDELETE_DEFAULT_FILE_FORMAT static java.lang.StringDELETE_DISTRIBUTION_MODE static java.lang.StringDELETE_ISOLATION_LEVEL static java.lang.StringDELETE_ISOLATION_LEVEL_DEFAULT static java.lang.StringDELETE_MODE static java.lang.StringDELETE_MODE_DEFAULT static java.lang.StringDELETE_ORC_BLOCK_SIZE_BYTES static java.lang.StringDELETE_ORC_COMPRESSION static java.lang.StringDELETE_ORC_COMPRESSION_STRATEGY static java.lang.StringDELETE_ORC_STRIPE_SIZE_BYTES static java.lang.StringDELETE_ORC_WRITE_BATCH_SIZE static java.lang.StringDELETE_PARQUET_COMPRESSION static java.lang.StringDELETE_PARQUET_COMPRESSION_LEVEL static java.lang.StringDELETE_PARQUET_DICT_SIZE_BYTES static java.lang.StringDELETE_PARQUET_PAGE_ROW_LIMIT static java.lang.StringDELETE_PARQUET_PAGE_SIZE_BYTES static java.lang.StringDELETE_PARQUET_ROW_GROUP_CHECK_MAX_RECORD_COUNT static java.lang.StringDELETE_PARQUET_ROW_GROUP_CHECK_MIN_RECORD_COUNT static java.lang.StringDELETE_PARQUET_ROW_GROUP_SIZE_BYTES static java.lang.StringDELETE_PLANNING_MODE static java.lang.StringDELETE_TARGET_FILE_SIZE_BYTES static java.lang.StringHMS_TABLE_OWNER
-HIVE_LOCK_ENABLED static booleanHIVE_LOCK_ENABLED_DEFAULT static java.lang.StringMANIFEST_LISTS_ENABLED
static booleanMANIFEST_LISTS_ENABLED_DEFAULT
static java.lang.StringMANIFEST_MERGE_ENABLED static booleanMANIFEST_MERGE_ENABLED_DEFAULT static java.lang.StringMANIFEST_MIN_MERGE_COUNT static intMANIFEST_MIN_MERGE_COUNT_DEFAULT static java.lang.StringMANIFEST_TARGET_SIZE_BYTES static longMANIFEST_TARGET_SIZE_BYTES_DEFAULT static java.lang.StringMAX_REF_AGE_MS static longMAX_REF_AGE_MS_DEFAULT static java.lang.StringMAX_SNAPSHOT_AGE_MS static longMAX_SNAPSHOT_AGE_MS_DEFAULT static java.lang.StringMERGE_CARDINALITY_CHECK_ENABLED
-static booleanMERGE_CARDINALITY_CHECK_ENABLED_DEFAULT
-static java.lang.StringMERGE_DISTRIBUTION_MODE static java.lang.StringMERGE_ISOLATION_LEVEL static java.lang.StringMERGE_ISOLATION_LEVEL_DEFAULT static java.lang.StringMERGE_MODE static java.lang.StringMERGE_MODE_DEFAULT static java.lang.StringMETADATA_COMPRESSION static java.lang.StringMETADATA_COMPRESSION_DEFAULT static java.lang.StringMETADATA_DELETE_AFTER_COMMIT_ENABLED static booleanMETADATA_DELETE_AFTER_COMMIT_ENABLED_DEFAULT static java.lang.StringMETADATA_PREVIOUS_VERSIONS_MAX static intMETADATA_PREVIOUS_VERSIONS_MAX_DEFAULT static java.lang.StringMETADATA_SPLIT_SIZE static longMETADATA_SPLIT_SIZE_DEFAULT static java.lang.StringMETRICS_MAX_INFERRED_COLUMN_DEFAULTS static intMETRICS_MAX_INFERRED_COLUMN_DEFAULTS_DEFAULT static java.lang.StringMETRICS_MODE_COLUMN_CONF_PREFIX static java.lang.StringMIN_SNAPSHOTS_TO_KEEP static intMIN_SNAPSHOTS_TO_KEEP_DEFAULT static java.lang.StringOBJECT_STORE_ENABLED static booleanOBJECT_STORE_ENABLED_DEFAULT static java.lang.StringOBJECT_STORE_PATH
static java.lang.StringORC_BATCH_SIZE static intORC_BATCH_SIZE_DEFAULT static java.lang.StringORC_BLOCK_SIZE_BYTES static longORC_BLOCK_SIZE_BYTES_DEFAULT static java.lang.StringORC_BLOOM_FILTER_COLUMNS static java.lang.StringORC_BLOOM_FILTER_COLUMNS_DEFAULT static java.lang.StringORC_BLOOM_FILTER_FPP static doubleORC_BLOOM_FILTER_FPP_DEFAULT static java.lang.StringORC_COMPRESSION static java.lang.StringORC_COMPRESSION_DEFAULT static java.lang.StringORC_COMPRESSION_STRATEGY static java.lang.StringORC_COMPRESSION_STRATEGY_DEFAULT static java.lang.StringORC_STRIPE_SIZE_BYTES static longORC_STRIPE_SIZE_BYTES_DEFAULT static java.lang.StringORC_VECTORIZATION_ENABLED static booleanORC_VECTORIZATION_ENABLED_DEFAULT static java.lang.StringORC_WRITE_BATCH_SIZE static intORC_WRITE_BATCH_SIZE_DEFAULT static java.lang.StringPARQUET_BATCH_SIZE static intPARQUET_BATCH_SIZE_DEFAULT static java.lang.StringPARQUET_BLOOM_FILTER_COLUMN_ENABLED_PREFIX static java.lang.StringPARQUET_BLOOM_FILTER_MAX_BYTES static intPARQUET_BLOOM_FILTER_MAX_BYTES_DEFAULT static java.lang.StringPARQUET_COMPRESSION static java.lang.StringPARQUET_COMPRESSION_DEFAULT static java.lang.StringPARQUET_COMPRESSION_DEFAULT_SINCE_1_4_0 static java.lang.StringPARQUET_COMPRESSION_LEVEL PARQUET_VECTORIZATION_ENABLED_DEFAULT static java.lang.StringPLANNING_MODE_DEFAULT static java.util.Set<java.lang.String>RESERVED_PROPERTIES
static java.lang.StringSNAPSHOT_COUNT
static java.lang.StringSNAPSHOT_ID_INHERITANCE_ENABLED static booleanSNAPSHOT_ID_INHERITANCE_ENABLED_DEFAULT static java.lang.StringSPARK_WRITE_ACCEPT_ANY_SCHEMA static booleanSPARK_WRITE_ACCEPT_ANY_SCHEMA_DEFAULT static java.lang.StringSPARK_WRITE_ADVISORY_PARTITION_SIZE_BYTES static java.lang.StringSPARK_WRITE_PARTITIONED_FANOUT_ENABLED public static final java.lang.String PARQUET_COMPRESSION_DEFAULT_SINCE_1_4_0+
public static final java.lang.String ADAPTIVE_SPLIT_SIZE_ENABLED+
public static final boolean ADAPTIVE_SPLIT_SIZE_ENABLED_DEFAULT+
public static final java.lang.String DATA_PLANNING_MODE+
public static final java.lang.String DELETE_PLANNING_MODE+
public static final java.lang.String PLANNING_MODE_DEFAULT+
public static final java.lang.String SPARK_WRITE_ADVISORY_PARTITION_SIZE_BYTES+
public static final java.lang.String HIVE_LOCK_ENABLED+
public static final boolean HIVE_LOCK_ENABLED_DEFAULT+
public static final java.lang.String MERGE_MODE_DEFAULT- - - -
@Deprecated -public static final java.lang.String MERGE_CARDINALITY_CHECK_ENABLED-
@Deprecated -public static final boolean MERGE_CARDINALITY_CHECK_ENABLED_DEFAULT-
public static final boolean UPSERT_ENABLED_DEFAULT@@ -3125,21 +3216,6 @@
@Deprecated -public static final java.lang.String HMS_TABLE_OWNER-
caseSensitive, filter, filter, ignoreResiduals, includeColumnStats, isCaseSensitive, option, planFiles, planTasks, planWith, project, schema, select, select, splitLookback, splitOpenFileCost, targetSplitSize
+caseSensitive, filter, filter, ignoreResiduals, includeColumnStats, isCaseSensitive, metricsReporter, option, planFiles, planTasks, planWith, project, schema, select, select, splitLookback, splitOpenFileCost, targetSplitSize
diff --git a/javadoc/org/apache/iceberg/UpdateProperties.html b/javadoc/org/apache/iceberg/UpdateProperties.html
index 34d2c4e43..df12d3b79 100644
--- a/javadoc/org/apache/iceberg/UpdateProperties.html
+++ b/javadoc/org/apache/iceberg/UpdateProperties.html
@@ -48,7 +48,7 @@
default RewritePositionDeleteFilesrewritePositionDeletes(Table table)
+default SnapshotTablesnapshotTable(java.lang.String sourceTableIdent)
default DeleteReachableFiles deleteReachableFiles(java.lang.String metadataLocation)
default RewritePositionDeleteFiles rewritePositionDeletes(Table table)+
BaseRewriteDataFilesAction<ThisT>useStartingSequenceNumber(boolean useStarting)
+public BaseRewriteDataFilesAction<ThisT> useStartingSequenceNumber(boolean useStarting)+
This avoids commit conflicts with updates that add newer equality deletes at a higher + sequence number.
useStarting - use starting sequence number if set to truepublic abstract class BinPackStrategy +@Deprecated +public abstract class BinPackStrategy extends java.lang.Object implements RewriteStrategyA rewrite strategy for data files which determines which files to rewrite based on their size. If @@ -149,54 +154,69 @@Field Summary
static java.lang.StringDELETE_FILE_THRESHOLD+Deprecated.The minimum number of deletes that needs to be associated with a data file for it to be considered for rewriting.- static int+ DELETE_FILE_THRESHOLD_DEFAULTDELETE_FILE_THRESHOLD_DEFAULT+Deprecated.+static java.lang.StringMAX_FILE_SIZE_BYTES+Deprecated.Adjusts files which will be considered for rewriting.- static double+ MAX_FILE_SIZE_DEFAULT_RATIOMAX_FILE_SIZE_DEFAULT_RATIO+Deprecated.+static java.lang.StringMIN_FILE_SIZE_BYTES+Deprecated.Adjusts files which will be considered for rewriting.- static double+ MIN_FILE_SIZE_DEFAULT_RATIOMIN_FILE_SIZE_DEFAULT_RATIO+Deprecated.+static java.lang.StringMIN_INPUT_FILES+Deprecated.The minimum number of files that need to be in a file group for it to be considered for compaction if the total size of that group is less than theRewriteDataFiles.TARGET_FILE_SIZE_BYTES.- static int+ MIN_INPUT_FILES_DEFAULTMIN_INPUT_FILES_DEFAULT+Deprecated.+static java.lang.StringREWRITE_ALL+Deprecated.Rewrites all files, regardless of their size.@@ -213,7 +233,9 @@ - static boolean+ REWRITE_ALL_DEFAULTREWRITE_ALL_DEFAULT+Deprecated.+Constructor Summary
Constructor and Description - @@ -225,65 +247,77 @@+ BinPackStrategy()BinPackStrategy()+Deprecated.+Constructor Summary
Method Summary
-
+ +All Methods Instance Methods Concrete Methods +All Methods Instance Methods Concrete Methods Deprecated Methods Modifier and Type Method and Description - protected long+ inputFileSize(java.util.List<FileScanTask> fileToRewrite)inputFileSize(java.util.List<FileScanTask> fileToRewrite)+Deprecated.+java.lang.Stringname()+Deprecated.Returns the name of this rewrite strategyprotected longnumOutputFiles(long totalSizeInBytes)+Deprecated.Determine how many output files to create when rewriting.RewriteStrategyoptions(java.util.Map<java.lang.String,java.lang.String> options)+Deprecated.Sets options to be used with this strategyjava.lang.Iterable<java.util.List<FileScanTask>>planFileGroups(java.lang.Iterable<FileScanTask> dataFiles)+Deprecated.Groups file scans into lists which will be processed in a single executable unit.java.lang.Iterable<FileScanTask>selectFilesToRewrite(java.lang.Iterable<FileScanTask> dataFiles)+Deprecated.Selects files which this strategy believes are valid targets to be rewritten.protected longsplitSize(long totalSizeInBytes)+Deprecated.Returns the smallest of our max write file threshold, and our estimated split size based on the number of output files we want to generate.- protected long+ targetFileSize()targetFileSize()+Deprecated.+java.util.Set<java.lang.String>validOptions()+Deprecated.Returns a set of options which this rewrite strategy can use.protected longwriteMaxFileSize()+Deprecated.Estimates a larger max target file size than our target size used in task creation to avoid tasks which are predicted to have a certain size, but exceed that target size when serde is complete creating tiny remainder files.@@ -325,6 +359,7 @@Field Detail
MIN_INPUT_FILES
public static final java.lang.String MIN_INPUT_FILES+Deprecated.The minimum number of files that need to be in a file group for it to be considered for compaction if the total size of that group is less than the@@ -341,6 +376,7 @@RewriteDataFiles.TARGET_FILE_SIZE_BYTES. This can also be thought of as the maximum number of non-target-size files that could remain in a file group (partition) after rewriting.MIN_INPUT_FILES
MIN_INPUT_FILES_DEFAULT
public static final int MIN_INPUT_FILES_DEFAULT+Deprecated.
- See Also:
- Constant Field Values
@@ -354,6 +390,7 @@MIN_INPUT_FILES_DEFAULT
MIN_FILE_SIZE_BYTES
public static final java.lang.String MIN_FILE_SIZE_BYTES+Deprecated.Adjusts files which will be considered for rewriting. Files smaller than@@ -370,6 +407,7 @@MIN_FILE_SIZE_BYTESwill be considered for rewriting. This functions independently ofMAX_FILE_SIZE_BYTES.Defaults to 75% of the target file size
MIN_FILE_SIZE_BYTES
MIN_FILE_SIZE_DEFAULT_RATIO
public static final double MIN_FILE_SIZE_DEFAULT_RATIO+Deprecated.
- See Also:
- Constant Field Values
@@ -383,6 +421,7 @@MIN_FILE_SIZE_DEFAULT_RATIO
MAX_FILE_SIZE_BYTES
public static final java.lang.String MAX_FILE_SIZE_BYTES+Deprecated.Adjusts files which will be considered for rewriting. Files larger than@@ -399,6 +438,7 @@MAX_FILE_SIZE_BYTESwill be considered for rewriting. This functions independently ofMIN_FILE_SIZE_BYTES.Defaults to 180% of the target file size
MAX_FILE_SIZE_BYTES
MAX_FILE_SIZE_DEFAULT_RATIO
public static final double MAX_FILE_SIZE_DEFAULT_RATIO+Deprecated.
- See Also:
- Constant Field Values
@@ -412,6 +452,7 @@MAX_FILE_SIZE_DEFAULT_RATIO
DELETE_FILE_THRESHOLD
public static final java.lang.String DELETE_FILE_THRESHOLD+Deprecated.The minimum number of deletes that needs to be associated with a data file for it to be considered for rewriting. If a data file has this number of deletes or more, it will be rewritten regardless of its file size determined byMIN_FILE_SIZE_BYTESandMAX_FILE_SIZE_BYTES. If a file group contains a file that satisfies this condition, the file @@ -432,6 +473,7 @@DELETE_FILE_THRESHOLD
DELETE_FILE_THRESHOLD_DEFAULT
public static final int DELETE_FILE_THRESHOLD_DEFAULT+Deprecated.
- See Also:
- Constant Field Values
@@ -445,6 +487,7 @@DELETE_FILE_THRESHOLD_DEFAULT
REWRITE_ALL
public static final java.lang.String REWRITE_ALL+Deprecated.Rewrites all files, regardless of their size. Defaults to false, rewriting only mis-sized files;@@ -460,6 +503,7 @@
REWRITE_ALL
REWRITE_ALL_DEFAULT
public static final boolean REWRITE_ALL_DEFAULT+Deprecated.
- See Also:
- Constant Field Values
@@ -481,6 +525,7 @@Constructor Detail
- @@ -498,6 +543,7 @@
BinPackStrategy
public BinPackStrategy()+Deprecated.Method Detail
name
public java.lang.String name()+Deprecated.Description copied from interface:RewriteStrategyReturns the name of this rewrite strategy@@ -513,6 +559,7 @@
name
validOptions
public java.util.Set<java.lang.String> validOptions()+Deprecated.Description copied from interface:RewriteStrategyReturns a set of options which this rewrite strategy can use. This is an allowed-list and any options not specified here will be rejected at runtime.@@ -529,6 +576,7 @@validOptions
options
public RewriteStrategy options(java.util.Map<java.lang.String,java.lang.String> options)+Deprecated.Description copied from interface:RewriteStrategySets options to be used with this strategy@@ -544,6 +592,7 @@
options
selectFilesToRewrite
public java.lang.Iterable<FileScanTask> selectFilesToRewrite(java.lang.Iterable<FileScanTask> dataFiles)+Deprecated.Description copied from interface:RewriteStrategySelects files which this strategy believes are valid targets to be rewritten.@@ -563,6 +612,7 @@
selectFilesToRewrite
planFileGroups
public java.lang.Iterable<java.util.List<FileScanTask>> planFileGroups(java.lang.Iterable<FileScanTask> dataFiles)+Deprecated.Description copied from interface:RewriteStrategyGroups file scans into lists which will be processed in a single executable unit. Each group will end up being committed as an independent set of changes. This creates the jobs which will @@ -584,6 +634,7 @@planFileGroups
- @@ -593,6 +644,7 @@
targetFileSize
protected long targetFileSize()+Deprecated.targetFileSize
numOutputFiles
protected long numOutputFiles(long totalSizeInBytes)+Deprecated.Determine how many output files to create when rewriting. We use this to determine the split-size we want to use when actually writing files to avoid the following situation. @@ -617,6 +669,7 @@numOutputFiles
splitSize
protected long splitSize(long totalSizeInBytes)+Deprecated.Returns the smallest of our max write file threshold, and our estimated split size based on the number of output files we want to generate. Add a overhead onto the estimated splitSize to try to avoid small errors in size creating brand-new files.@@ -629,6 +682,7 @@splitSize
- @@ -638,6 +692,7 @@
inputFileSize
protected long inputFileSize(java.util.List<FileScanTask> fileToRewrite)+Deprecated.inputFileSize
writeMaxFileSize
protected long writeMaxFileSize()+Deprecated.Estimates a larger max target file size than our target size used in task creation to avoid tasks which are predicted to have a certain size, but exceed that target size when serde is complete creating tiny remainder files. @@ -684,7 +739,7 @@writeMaxFileSize
+ + default MigrateTable+ backupTableName(java.lang.String tableName)+Sets a table name for the backup of the original table.++ -default MigrateTabledropBackup()Drops the backup of the original table after a successful migration+ - MigrateTabletableProperties(java.util.Map<java.lang.String,java.lang.String> properties)Sets table properties in the newly created Iceberg table.+ MigrateTabletableProperty(java.lang.String name, java.lang.String value)@@ -229,7 +235,7 @@tableProperty
-+
@@ -267,7 +289,7 @@+ + + +
dropBackup
default MigrateTable dropBackup()@@ -240,6 +246,22 @@dropBackup
+
- +
+backupTableName
+default MigrateTable backupTableName(java.lang.String tableName)+Sets a table name for the backup of the original table.++
+- Parameters:
+- +
tableName- the table name for backup- Returns:
+- this for method chaining
+dropBackup
+ + static interface+ RewritePositionDeleteFiles.FileGroupInfo+A description of a position delete file group, when it was processed, and within which + partition.++ ++ static interface+ RewritePositionDeleteFiles.FileGroupRewriteResult+For a particular position delete file group, the number of position delete files which are + newly created and the number of files which were formerly part of the table but have been + rewritten.++ static interface@@ -135,6 +154,63 @@ RewritePositionDeleteFiles.ResultThe action result that contains a summary of the execution.Nested Class Summary
+
- + + +
+Field Summary
++
+Fields ++ +Modifier and Type +Field and Description ++ ++ static java.lang.String+ MAX_CONCURRENT_FILE_GROUP_REWRITES+The max number of file groups to be simultaneously rewritten by the rewrite strategy.++ ++ static int+ MAX_CONCURRENT_FILE_GROUP_REWRITES_DEFAULT+ ++ static java.lang.String+ PARTIAL_PROGRESS_ENABLED+Enable committing groups of files (see max-file-group-size-bytes) prior to the entire rewrite + completing.++ ++ static boolean+ PARTIAL_PROGRESS_ENABLED_DEFAULT+ ++ static java.lang.String+ PARTIAL_PROGRESS_MAX_COMMITS+The maximum amount of Iceberg commits that this rewrite is allowed to produce if partial + progress is enabled.++ ++ static int+ PARTIAL_PROGRESS_MAX_COMMITS_DEFAULT+ ++ static java.lang.String+ REWRITE_JOB_ORDER+Forces the rewrite job order based on the value.++ ++ static java.lang.String+ REWRITE_JOB_ORDER_DEFAULT
- @@ -176,6 +252,137 @@
Methods inherited from interface org.apache.iceberg.actions.
- + +
+
- + + +
+Field Detail
+ + + ++
+ + + +- +
+PARTIAL_PROGRESS_ENABLED
+static final java.lang.String PARTIAL_PROGRESS_ENABLED+Enable committing groups of files (see max-file-group-size-bytes) prior to the entire rewrite + completing. This will produce additional commits but allow for progress even if some groups + fail to commit. This setting will not change the correctness of the rewrite operation as file + groups can be compacted independently. + ++The default is false, which produces a single commit when the entire job has completed.
+
+- See Also:
+- Constant Field Values
++
+ + + +- +
+PARTIAL_PROGRESS_ENABLED_DEFAULT
+static final boolean PARTIAL_PROGRESS_ENABLED_DEFAULT++
+- See Also:
+- Constant Field Values
++
+ + + +- +
+PARTIAL_PROGRESS_MAX_COMMITS
+static final java.lang.String PARTIAL_PROGRESS_MAX_COMMITS+The maximum amount of Iceberg commits that this rewrite is allowed to produce if partial + progress is enabled. This setting has no effect if partial progress is disabled.++
+- See Also:
+- Constant Field Values
++
+ + + +- +
+PARTIAL_PROGRESS_MAX_COMMITS_DEFAULT
+static final int PARTIAL_PROGRESS_MAX_COMMITS_DEFAULT++
+- See Also:
+- Constant Field Values
++
+ + + +- +
+MAX_CONCURRENT_FILE_GROUP_REWRITES
+static final java.lang.String MAX_CONCURRENT_FILE_GROUP_REWRITES+The max number of file groups to be simultaneously rewritten by the rewrite strategy. The + structure and contents of the group is determined by the rewrite strategy. Each file group will + be rewritten independently and asynchronously.++
+- See Also:
+- Constant Field Values
++
+ + + +- +
+MAX_CONCURRENT_FILE_GROUP_REWRITES_DEFAULT
+static final int MAX_CONCURRENT_FILE_GROUP_REWRITES_DEFAULT++
+- See Also:
+- Constant Field Values
++
+ + + +- +
+REWRITE_JOB_ORDER
+static final java.lang.String REWRITE_JOB_ORDER+Forces the rewrite job order based on the value. + +++
+ +- If rewrite-job-order=bytes-asc, then rewrite the smallest job groups first. +
- If rewrite-job-order=bytes-desc, then rewrite the largest job groups first. +
- If rewrite-job-order=files-asc, then rewrite the job groups with the least files first. +
- If rewrite-job-order=files-desc, then rewrite the job groups with the most files first. +
- If rewrite-job-order=none, then rewrite job groups in the order they were planned (no + specific ordering). +
Defaults to none.
+
+- See Also:
+- Constant Field Values
++
+- +
+REWRITE_JOB_ORDER_DEFAULT
+static final java.lang.String REWRITE_JOB_ORDER_DEFAULT+
- @@ -230,7 +437,7 @@
@@ -181,7 +184,7 @@filter
@@ -122,7 +128,7 @@Interface RewriteStrategy
Method Summary
-
All Methods Instance Methods Abstract Methods +All Methods Instance Methods Abstract Methods Deprecated Methods Modifier and Type Method and Description @@ -130,42 +136,49 @@Method Summary
java.lang.Stringname()+Deprecated.Returns the name of this rewrite strategyRewriteStrategyoptions(java.util.Map<java.lang.String,java.lang.String> options)+Deprecated.Sets options to be used with this strategyjava.lang.Iterable<java.util.List<FileScanTask>>planFileGroups(java.lang.Iterable<FileScanTask> dataFiles)+Deprecated.Groups file scans into lists which will be processed in a single executable unit.java.util.Set<DataFile>rewriteFiles(java.util.List<FileScanTask> filesToRewrite)+Deprecated.Method which will rewrite files based on this particular RewriteStrategy's algorithm.java.lang.Iterable<FileScanTask>selectFilesToRewrite(java.lang.Iterable<FileScanTask> dataFiles)+Deprecated.Selects files which this strategy believes are valid targets to be rewritten.Tabletable()+Deprecated.Returns the table being modified by this rewrite strategy@@ -191,6 +204,7 @@ java.util.Set<java.lang.String>validOptions()+Deprecated.Returns a set of options which this rewrite strategy can use.Method Detail
- @@ -201,6 +215,7 @@
name
java.lang.String name()+Deprecated.Returns the name of this rewrite strategyname
- @@ -211,6 +226,7 @@
table
Table table()+Deprecated.Returns the table being modified by this rewrite strategytable
- @@ -222,6 +238,7 @@
validOptions
java.util.Set<java.lang.String> validOptions()+Deprecated.Returns a set of options which this rewrite strategy can use. This is an allowed-list and any options not specified here will be rejected at runtime.validOptions
- @@ -232,6 +249,7 @@
options
RewriteStrategy options(java.util.Map<java.lang.String,java.lang.String> options)+Deprecated.Sets options to be used with this strategyoptions
selectFilesToRewrite
java.lang.Iterable<FileScanTask> selectFilesToRewrite(java.lang.Iterable<FileScanTask> dataFiles)+Deprecated.Selects files which this strategy believes are valid targets to be rewritten.
- Parameters:
@@ -248,6 +266,7 @@selectFilesToRewrite
planFileGroups
java.lang.Iterable<java.util.List<FileScanTask>> planFileGroups(java.lang.Iterable<FileScanTask> dataFiles)+Deprecated.Groups file scans into lists which will be processed in a single executable unit. Each group will end up being committed as an independent set of changes. This creates the jobs which will eventually be run as by the underlying Action.@@ -266,6 +285,7 @@planFileGroups
rewriteFiles
java.util.Set<DataFile> rewriteFiles(java.util.List<FileScanTask> filesToRewrite)+Deprecated.Method which will rewrite files based on this particular RewriteStrategy's algorithm. This will most likely be Action framework specific (Spark/Presto/Flink ....).@@ -303,8 +323,8 @@
rewriteFiles
Constructor Summary
Method Summary