diff --git a/.github/actions/run-gradle/action.yml b/.github/actions/run-gradle/action.yml
new file mode 100644
index 0000000000000..18d6bdeb1f558
--- /dev/null
+++ b/.github/actions/run-gradle/action.yml
@@ -0,0 +1,75 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+---
+name: "Run Tests with Gradle"
+description: "Run specified Gradle test tasks with configuration for timeout and test catalog."
+inputs:
+ # Composite actions do not support typed parameters. Everything is treated as a string
+ # See: https://github.com/actions/runner/issues/2238
+ test-task:
+ description: "The test suite to run. Either 'test' or 'quarantinedTest'."
+ required: true
+ timeout-minutes:
+ description: "The timeout for the tests, in minutes."
+ required: true
+ test-catalog-path:
+ description: "The file path of the test catalog file."
+ required: true
+ build-scan-artifact-name:
+ description: "The name to use for archiving the build scan."
+ required: true
+outputs:
+ gradle-exitcode:
+ description: "The result of the Gradle test task."
+ value: ${{ steps.run-tests.outputs.exitcode }}
+runs:
+ using: "composite"
+ steps:
+ - name: Run JUnit Tests (${{ inputs.test-task }})
+ # Gradle flags
+ # --build-cache: Let Gradle restore the build cache
+ # --no-scan: Don't attempt to publish the scan yet. We want to archive it first.
+ # --continue: Keep running even if a test fails
+ # -PcommitId Prevent the Git SHA being written into the jar files (which breaks caching)
+ shell: bash
+ id: run-tests
+ env:
+ TIMEOUT_MINUTES: ${{ inputs.timeout-minutes}}
+ TEST_CATALOG: ${{ inputs.test-catalog-path }}
+ TEST_TASK: ${{ inputs.test-task }}
+ run: |
+ set +e
+ ./.github/scripts/thread-dump.sh &
+ timeout ${TIMEOUT_MINUTES}m ./gradlew --build-cache --continue --no-scan \
+ -PtestLoggingEvents=started,passed,skipped,failed \
+ -PmaxParallelForks=2 \
+ -PmaxTestRetries=1 -PmaxTestRetryFailures=3 \
+ -PmaxQuarantineTestRetries=3 -PmaxQuarantineTestRetryFailures=0 \
+ -Pkafka.test.catalog.file=$TEST_CATALOG \
+ -PcommitId=xxxxxxxxxxxxxxxx \
+ $TEST_TASK
+ exitcode="$?"
+ echo "exitcode=$exitcode" >> $GITHUB_OUTPUT
+ - name: Archive build scan (${{ inputs.test-task }})
+ if: always()
+ uses: actions/upload-artifact@v4
+ with:
+ name: ${{ inputs.build-scan-artifact-name }}
+ path: ~/.gradle/build-scan-data
+ compression-level: 9
+ if-no-files-found: ignore
\ No newline at end of file
diff --git a/.github/configs/labeler.yml b/.github/configs/labeler.yml
index be6944c0ab90e..24a7a643c9042 100644
--- a/.github/configs/labeler.yml
+++ b/.github/configs/labeler.yml
@@ -124,3 +124,15 @@ kraft:
- any-glob-to-any-file:
- 'metadata/**'
- 'raft/**'
+
+dependencies:
+ - changed-files:
+ - any-glob-to-any-file:
+ - "grade/dependencies.gradle"
+ - "LICENSE-binary"
+
+tiered-storage:
+ - changed-files:
+ - any-glob-to-any-file:
+ - '*/src/*/java/org/apache/kafka/server/log/remote/**'
+ - '*/src/*/java/kafka/log/remote/**'
diff --git a/.github/scripts/develocity_reports.py b/.github/scripts/develocity_reports.py
new file mode 100644
index 0000000000000..74df32a3edeaf
--- /dev/null
+++ b/.github/scripts/develocity_reports.py
@@ -0,0 +1,1270 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import requests
+import json
+from dataclasses import dataclass, field, asdict
+from typing import Dict, List, Tuple, Optional
+from datetime import datetime, timedelta
+import pytz # Add this import for timezone handling
+from collections import defaultdict
+import time
+import logging
+import concurrent.futures # Add this import at the top
+import pathlib
+import pickle
+from abc import ABC, abstractmethod
+
+logger = logging.getLogger(__name__)
+
+@dataclass
+class TestOutcome:
+ passed: int
+ failed: int
+ skipped: int
+ flaky: int
+ not_selected: int = field(metadata={'name': 'notSelected'})
+ total: int
+
+@dataclass
+class BuildInfo:
+ id: str
+ timestamp: datetime
+ duration: int
+ has_failed: bool
+
+@dataclass
+class TestTimelineEntry:
+ build_id: str
+ timestamp: datetime
+ outcome: str # "passed", "failed", "flaky", etc.
+
+@dataclass
+class TestResult:
+ name: str
+ outcome_distribution: TestOutcome
+ first_seen: datetime
+ timeline: List[TestTimelineEntry] = field(default_factory=list)
+ recent_failure_rate: float = 0.0 # Added to track recent failure trends
+
+@dataclass
+class TestContainerResult:
+ build_id: str
+ outcome: str
+ timestamp: Optional[datetime] = None
+
+@dataclass
+class TestCaseResult(TestResult):
+ """Extends TestResult to include container-specific information"""
+ container_name: str = ""
+
+@dataclass
+class BuildCache:
+ last_update: datetime
+ builds: Dict[str, 'BuildInfo']
+
+ def to_dict(self):
+ return {
+ 'last_update': self.last_update.isoformat(),
+ 'builds': {k: asdict(v) for k, v in self.builds.items()}
+ }
+
+ @classmethod
+ def from_dict(cls, data: dict) -> 'BuildCache':
+ return cls(
+ last_update=datetime.fromisoformat(data['last_update']),
+ builds={k: BuildInfo(**v) for k, v in data['builds'].items()}
+ )
+
+class CacheProvider(ABC):
+ @abstractmethod
+ def get_cache(self) -> Optional[BuildCache]:
+ pass
+
+ @abstractmethod
+ def save_cache(self, cache: BuildCache):
+ pass
+
+class LocalCacheProvider(CacheProvider):
+ def __init__(self, cache_dir: str = None):
+ if cache_dir is None:
+ cache_dir = os.path.join(os.path.expanduser("~"), ".develocity_cache")
+ self.cache_file = os.path.join(cache_dir, "build_cache.pkl")
+ os.makedirs(cache_dir, exist_ok=True)
+
+ def get_cache(self) -> Optional[BuildCache]:
+ try:
+ if os.path.exists(self.cache_file):
+ with open(self.cache_file, 'rb') as f:
+ return pickle.load(f)
+ except Exception as e:
+ logger.warning(f"Failed to load local cache: {e}")
+ return None
+
+ def save_cache(self, cache: BuildCache):
+ try:
+ with open(self.cache_file, 'wb') as f:
+ pickle.dump(cache, f)
+ except Exception as e:
+ logger.warning(f"Failed to save local cache: {e}")
+
+class GitHubActionsCacheProvider(CacheProvider):
+ def __init__(self):
+ self.cache_key = "develocity-build-cache"
+
+ def get_cache(self) -> Optional[BuildCache]:
+ try:
+ # Check if running in GitHub Actions
+ if not os.environ.get('GITHUB_ACTIONS'):
+ return None
+
+ cache_path = os.environ.get('GITHUB_WORKSPACE', '')
+ cache_file = os.path.join(cache_path, self.cache_key + '.json')
+
+ if os.path.exists(cache_file):
+ with open(cache_file, 'r') as f:
+ data = json.load(f)
+ return BuildCache.from_dict(data)
+ except Exception as e:
+ logger.warning(f"Failed to load GitHub Actions cache: {e}")
+ return None
+
+ def save_cache(self, cache: BuildCache):
+ try:
+ if not os.environ.get('GITHUB_ACTIONS'):
+ return
+
+ cache_path = os.environ.get('GITHUB_WORKSPACE', '')
+ cache_file = os.path.join(cache_path, self.cache_key + '.json')
+
+ with open(cache_file, 'w') as f:
+ json.dump(cache.to_dict(), f)
+ except Exception as e:
+ logger.warning(f"Failed to save GitHub Actions cache: {e}")
+
+class TestAnalyzer:
+ def __init__(self, base_url: str, auth_token: str):
+ self.base_url = base_url
+ self.headers = {
+ 'Authorization': f'Bearer {auth_token}',
+ 'Accept': 'application/json'
+ }
+ self.default_chunk_size = timedelta(days=14)
+ self.api_retry_delay = 2 # seconds
+ self.max_api_retries = 3
+
+ # Initialize cache providers
+ self.cache_providers = [
+ GitHubActionsCacheProvider(),
+ LocalCacheProvider()
+ ]
+ self.build_cache = None
+ self._load_cache()
+
+ def _load_cache(self):
+ """Load cache from the first available provider"""
+ for provider in self.cache_providers:
+ cache = provider.get_cache()
+ if cache is not None:
+ self.build_cache = cache
+ logger.info(f"Loaded cache from {provider.__class__.__name__}")
+ return
+ logger.info("No existing cache found")
+
+ def _save_cache(self):
+ """Save cache to all providers"""
+ if self.build_cache:
+ for provider in self.cache_providers:
+ provider.save_cache(self.build_cache)
+ logger.info(f"Saved cache to {provider.__class__.__name__}")
+
+ def build_query(self, project: str, chunk_start: datetime, chunk_end: datetime, test_type: str) -> str:
+ """
+ Constructs the query string to be used in both build info and test containers API calls.
+
+ Args:
+ project: The project name.
+ chunk_start: The start datetime for the chunk.
+ chunk_end: The end datetime for the chunk.
+ test_type: The type of tests to query.
+
+ Returns:
+ A formatted query string.
+ """
+ return f'project:{project} buildStartTime:[{chunk_start.isoformat()} TO {chunk_end.isoformat()}] gradle.requestedTasks:{test_type} tag:github tag:trunk'
+
+ def process_chunk(
+ self,
+ chunk_start: datetime,
+ chunk_end: datetime,
+ project: str,
+ test_type: str,
+ remaining_build_ids: set | None,
+ max_builds_per_request: int
+ ) -> Dict[str, BuildInfo]:
+ """Helper method to process a single chunk of build information"""
+ chunk_builds = {}
+
+ # Use the helper method to build the query
+ query = self.build_query(project, chunk_start, chunk_end, test_type)
+
+ # Initialize pagination for this chunk
+ from_build = None
+ continue_chunk = True
+
+ while continue_chunk and (remaining_build_ids is None or remaining_build_ids):
+ query_params = {
+ 'query': query,
+ 'models': ['gradle-attributes'],
+ 'allModels': 'false',
+ 'maxBuilds': max_builds_per_request,
+ 'reverse': 'false',
+ 'fromInstant': int(chunk_start.timestamp() * 1000)
+ }
+
+ if from_build:
+ query_params['fromBuild'] = from_build
+
+ for attempt in range(self.max_api_retries):
+ try:
+ response = requests.get(
+ f'{self.base_url}/api/builds',
+ headers=self.headers,
+ params=query_params,
+ timeout=(5, 30)
+ )
+ response.raise_for_status()
+ break
+ except requests.exceptions.Timeout:
+ if attempt == self.max_api_retries - 1:
+ raise
+ time.sleep(self.api_retry_delay * (attempt + 1))
+ except requests.exceptions.RequestException:
+ raise
+
+ response_json = response.json()
+
+ if not response_json:
+ break
+
+ for build in response_json:
+ build_id = build['id']
+
+ if 'models' in build and 'gradleAttributes' in build['models']:
+ gradle_attrs = build['models']['gradleAttributes']
+ if 'model' in gradle_attrs:
+ attrs = gradle_attrs['model']
+ build_timestamp = datetime.fromtimestamp(attrs['buildStartTime'] / 1000, pytz.UTC)
+
+ if build_timestamp >= chunk_end:
+ continue_chunk = False
+ break
+
+ if remaining_build_ids is None or build_id in remaining_build_ids:
+ if 'problem' not in gradle_attrs:
+ chunk_builds[build_id] = BuildInfo(
+ id=build_id,
+ timestamp=build_timestamp,
+ duration=attrs.get('buildDuration'),
+ has_failed=attrs.get('hasFailed', False)
+ )
+ if remaining_build_ids is not None:
+ remaining_build_ids.remove(build_id)
+
+ if continue_chunk and response_json:
+ from_build = response_json[-1]['id']
+ else:
+ continue_chunk = False
+
+ time.sleep(0.5) # Rate limiting between pagination requests
+
+ return chunk_builds
+ def get_build_info(self, build_ids: List[str] = None, project: str = None, test_type: str = None, query_days: int = None, bypass_cache: bool = False, fetch_all: bool = False) -> Dict[str, BuildInfo]:
+ builds = {}
+ max_builds_per_request = 100
+ cutoff_date = datetime.now(pytz.UTC) - timedelta(days=query_days)
+ current_time = datetime.now(pytz.UTC)
+
+ if not fetch_all and not build_ids:
+ raise ValueError("Either build_ids must be provided or fetch_all must be True")
+
+ # Get builds from cache if available and bypass_cache is False
+ if not bypass_cache and self.build_cache:
+ cached_builds = self.build_cache.builds
+ cached_cutoff = self.build_cache.last_update - timedelta(days=query_days)
+
+ if fetch_all:
+ # Use all cached builds within the time period
+ for build_id, build in cached_builds.items():
+ if build.timestamp >= cached_cutoff:
+ builds[build_id] = build
+ else:
+ # Use cached data for specific builds within the cache period
+ for build_id in build_ids:
+ if build_id in cached_builds:
+ build = cached_builds[build_id]
+ if build.timestamp >= cached_cutoff:
+ builds[build_id] = build
+
+ # Update cutoff date to only fetch new data
+ cutoff_date = self.build_cache.last_update
+ logger.info(f"Using cached data up to {cutoff_date.isoformat()}")
+
+ if not fetch_all:
+ # Remove already found builds from the search list
+ build_ids = [bid for bid in build_ids if bid not in builds]
+
+ if not build_ids:
+ logger.info("All builds found in cache")
+ return builds
+
+ # Fetch remaining builds from API
+ remaining_build_ids = set(build_ids) if not fetch_all else None
+ chunk_size = self.default_chunk_size
+
+ # Create time chunks
+ chunks = []
+ chunk_start = cutoff_date
+ while chunk_start < current_time:
+ chunk_end = min(chunk_start + chunk_size, current_time)
+ chunks.append((chunk_start, chunk_end))
+ chunk_start = chunk_end
+
+ total_start_time = time.time()
+
+ # Process chunks in parallel
+ with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:
+ future_to_chunk = {
+ executor.submit(
+ self.process_chunk,
+ chunk[0],
+ chunk[1],
+ project,
+ test_type,
+ remaining_build_ids.copy() if remaining_build_ids else None,
+ max_builds_per_request
+ ): chunk for chunk in chunks
+ }
+
+ for future in concurrent.futures.as_completed(future_to_chunk):
+ try:
+ chunk_builds = future.result()
+ builds.update(chunk_builds)
+ if remaining_build_ids:
+ remaining_build_ids -= set(chunk_builds.keys())
+ except Exception as e:
+ logger.error(f"Chunk processing generated an exception: {str(e)}")
+
+ total_duration = time.time() - total_start_time
+ logger.info(
+ f"\nBuild Info Performance:"
+ f"\n Total Duration: {total_duration:.2f}s"
+ f"\n Builds Retrieved: {len(builds)}"
+ f"\n Builds Not Found: {len(remaining_build_ids) if remaining_build_ids else 0}"
+ )
+
+ # Update cache with new data if not bypassing cache
+ if builds and not bypass_cache:
+ if not self.build_cache:
+ self.build_cache = BuildCache(current_time, {})
+ self.build_cache.builds.update(builds)
+ self.build_cache.last_update = current_time
+ self._save_cache()
+
+ return builds
+ def get_test_results(self, project: str, threshold_days: int, test_type: str = "quarantinedTest",
+ outcomes: List[str] = None) -> List[TestResult]:
+ """Fetch test results with timeline information"""
+ if outcomes is None:
+ outcomes = ["failed", "flaky"]
+
+ logger.debug(f"Fetching test results for project {project}, last {threshold_days} days")
+
+ end_time = datetime.now(pytz.UTC)
+ start_time = end_time - timedelta(days=threshold_days)
+
+ all_results = {}
+ build_ids = set()
+ test_container_results = defaultdict(list)
+
+ chunk_size = self.default_chunk_size
+ chunk_start = start_time
+
+ while chunk_start < end_time:
+ chunk_end = min(chunk_start + chunk_size, end_time)
+ logger.debug(f"Processing chunk: {chunk_start} to {chunk_end}")
+
+ # Use the helper method to build the query
+ query = self.build_query(project, chunk_start, chunk_end, test_type)
+
+ query_params = {
+ 'query': query,
+ 'testOutcomes': outcomes,
+ 'container': '*',
+ 'include': ['buildScanIds'] # Explicitly request build scan IDs
+ }
+
+ response = requests.get(
+ f'{self.base_url}/api/tests/containers',
+ headers=self.headers,
+ params=query_params
+ )
+ response.raise_for_status()
+
+ for test in response.json()['content']:
+ test_name = test['name']
+ logger.debug(f"Processing test: {test_name}")
+
+ if test_name not in all_results:
+ outcome_data = test['outcomeDistribution']
+ if 'notSelected' in outcome_data:
+ outcome_data['not_selected'] = outcome_data.pop('notSelected')
+ outcome = TestOutcome(**outcome_data)
+ all_results[test_name] = TestResult(test_name, outcome, chunk_start)
+
+ # Collect build IDs by outcome
+ if 'buildScanIdsByOutcome' in test:
+ scan_ids = test['buildScanIdsByOutcome']
+
+ for outcome, ids in scan_ids.items():
+ if ids: # Only process if we have IDs
+ for build_id in ids:
+ build_ids.add(build_id)
+ test_container_results[test_name].append(
+ TestContainerResult(build_id=build_id, outcome=outcome)
+ )
+
+ chunk_start = chunk_end
+
+ logger.debug(f"Total unique build IDs collected: {len(build_ids)}")
+
+ # Fetch build information using the updated get_build_info method
+ builds = self.get_build_info(list(build_ids), project, test_type, threshold_days)
+ logger.debug(f"Retrieved {len(builds)} builds from API")
+ logger.debug(f"Retrieved build IDs: {sorted(builds.keys())}")
+
+ # Update test results with timeline information
+ for test_name, result in all_results.items():
+ logger.debug(f"\nProcessing timeline for test: {test_name}")
+ timeline = []
+ for container_result in test_container_results[test_name]:
+ logger.debug(f"Processing container result: {container_result}")
+ if container_result.build_id in builds:
+ build_info = builds[container_result.build_id]
+ timeline.append(TestTimelineEntry(
+ build_id=container_result.build_id,
+ timestamp=build_info.timestamp,
+ outcome=container_result.outcome
+ ))
+ else:
+ logger.warning(f"Build ID {container_result.build_id} not found in builds response")
+
+ # Sort timeline by timestamp
+ result.timeline = sorted(timeline, key=lambda x: x.timestamp)
+ logger.debug(f"Final timeline entries for {test_name}: {len(result.timeline)}")
+
+ # Print build details for debugging
+ logger.debug("Timeline entries:")
+ for entry in timeline:
+ logger.debug(f"Build ID: {entry.build_id}, Timestamp: {entry.timestamp}, Outcome: {entry.outcome}")
+
+ # Calculate recent failure rate
+ recent_cutoff = datetime.now(pytz.UTC) - timedelta(days=30)
+ recent_runs = [t for t in timeline if t.timestamp >= recent_cutoff]
+ if recent_runs:
+ recent_failures = sum(1 for t in recent_runs if t.outcome in ('failed', 'flaky'))
+ result.recent_failure_rate = recent_failures / len(recent_runs)
+
+ return list(all_results.values())
+
+ def get_defective_tests(self, results: List[TestResult]) -> Dict[str, TestResult]:
+ """
+ Analyze test results to find defective tests (failed or flaky)
+ """
+ defective_tests = {}
+
+ for result in results:
+ if result.outcome_distribution.failed > 0 or result.outcome_distribution.flaky > 0:
+ defective_tests[result.name] = result
+
+ return defective_tests
+
+ def get_long_quarantined_tests(self, results: List[TestResult], quarantine_threshold_days: int = 60) -> Dict[str, TestResult]:
+ """
+ Find tests that have been quarantined longer than the threshold.
+ These are candidates for removal or rewriting.
+
+ Args:
+ results: List of test results
+ quarantine_threshold_days: Number of days after which a quarantined test should be considered for removal/rewrite
+ """
+ long_quarantined = {}
+ current_time = datetime.now(pytz.UTC)
+
+ for result in results:
+ days_quarantined = (current_time - result.first_seen).days
+ if days_quarantined >= quarantine_threshold_days:
+ long_quarantined[result.name] = (result, days_quarantined)
+
+ return long_quarantined
+
+ def get_problematic_quarantined_tests(
+ self,
+ results: List[TestResult],
+ quarantine_threshold_days: int = 60,
+ min_failure_rate: float = 0.3,
+ recent_failure_threshold: float = 0.5
+ ) -> Dict[str, Dict]:
+ """Enhanced version that includes test case details"""
+ problematic_tests = {}
+ current_time = datetime.now(pytz.UTC)
+ chunk_start = current_time - timedelta(days=7) # Last 7 days for test cases
+
+ for result in results:
+ days_quarantined = (current_time - result.first_seen).days
+ if days_quarantined >= quarantine_threshold_days:
+ total_runs = result.outcome_distribution.total
+ if total_runs > 0:
+ problem_runs = result.outcome_distribution.failed + result.outcome_distribution.flaky
+ failure_rate = problem_runs / total_runs
+
+ if failure_rate >= min_failure_rate or result.recent_failure_rate >= recent_failure_threshold:
+ # Get detailed test case information
+ try:
+ test_cases = self.get_test_case_details(
+ result.name,
+ "kafka",
+ chunk_start,
+ current_time,
+ test_type="quarantinedTest"
+ )
+
+ problematic_tests[result.name] = {
+ 'container_result': result,
+ 'days_quarantined': days_quarantined,
+ 'failure_rate': failure_rate,
+ 'recent_failure_rate': result.recent_failure_rate,
+ 'test_cases': test_cases
+ }
+ except Exception as e:
+ logger.error(f"Error getting test case details for {result.name}: {str(e)}")
+
+ return problematic_tests
+
+ def get_test_case_details(
+ self,
+ container_name: str,
+ project: str,
+ chunk_start: datetime,
+ chunk_end: datetime,
+ test_type: str = "quarantinedTest"
+ ) -> List[TestCaseResult]:
+ """
+ Fetch detailed test case results for a specific container.
+
+ Args:
+ container_name: Name of the test container
+ project: The project name
+ chunk_start: Start time for the query
+ chunk_end: End time for the query
+ test_type: Type of tests to query (default: "quarantinedTest")
+ """
+ # Use the helper method to build the query, similar to get_test_results
+ query = self.build_query(project, chunk_start, chunk_end, test_type)
+
+ query_params = {
+ 'query': query,
+ 'testOutcomes': ['failed', 'flaky'],
+ 'container': container_name,
+ 'include': ['buildScanIds'], # Explicitly request build scan IDs
+ 'limit': 1000
+ }
+
+ try:
+ response = requests.get(
+ f'{self.base_url}/api/tests/cases',
+ headers=self.headers,
+ params=query_params
+ )
+ response.raise_for_status()
+
+ test_cases = []
+ content = response.json().get('content', [])
+
+ # Collect all build IDs first
+ build_ids = set()
+ for test in content:
+ if 'buildScanIdsByOutcome' in test:
+ for outcome_type, ids in test['buildScanIdsByOutcome'].items():
+ build_ids.update(ids)
+
+ # Get build info for all build IDs
+ builds = self.get_build_info(list(build_ids), project, test_type, 7) # 7 days for test cases
+
+ for test in content:
+ outcome_data = test['outcomeDistribution']
+ if 'notSelected' in outcome_data:
+ outcome_data['not_selected'] = outcome_data.pop('notSelected')
+ outcome = TestOutcome(**outcome_data)
+
+ test_case = TestCaseResult(
+ name=test['name'],
+ outcome_distribution=outcome,
+ first_seen=chunk_start,
+ container_name=container_name
+ )
+
+ # Add build information with proper timestamps
+ if 'buildScanIdsByOutcome' in test:
+ for outcome_type, build_ids in test['buildScanIdsByOutcome'].items():
+ for build_id in build_ids:
+ if build_id in builds:
+ build_info = builds[build_id]
+ test_case.timeline.append(
+ TestTimelineEntry(
+ build_id=build_id,
+ timestamp=build_info.timestamp,
+ outcome=outcome_type
+ )
+ )
+ else:
+ logger.warning(f"Build ID {build_id} not found for test case {test['name']}")
+
+ # Sort timeline by timestamp
+ test_case.timeline.sort(key=lambda x: x.timestamp)
+ test_cases.append(test_case)
+
+ return test_cases
+
+ except requests.exceptions.RequestException as e:
+ logger.error(f"Error fetching test case details for {container_name}: {str(e)}")
+ raise
+
+ def get_flaky_test_regressions(self, project: str, results: List[TestResult],
+ recent_days: int = 7, min_flaky_rate: float = 0.2) -> Dict[str, Dict]:
+ """
+ Identify tests that have recently started showing flaky behavior.
+
+ Args:
+ project: The project name
+ results: List of test results
+ recent_days: Number of days to consider for recent behavior
+ min_flaky_rate: Minimum flaky rate to consider a test as problematic
+ """
+ flaky_regressions = {}
+ current_time = datetime.now(pytz.UTC)
+ recent_cutoff = current_time - timedelta(days=recent_days)
+
+ for result in results:
+ # Skip tests with no timeline data
+ if not result.timeline:
+ continue
+
+ # Split timeline into recent and historical periods
+ recent_entries = [t for t in result.timeline if t.timestamp >= recent_cutoff]
+ historical_entries = [t for t in result.timeline if t.timestamp < recent_cutoff]
+
+ if not recent_entries or not historical_entries:
+ continue
+
+ # Calculate flaky rates
+ recent_flaky = sum(1 for t in recent_entries if t.outcome == 'flaky')
+ recent_total = len(recent_entries)
+ recent_flaky_rate = recent_flaky / recent_total if recent_total > 0 else 0
+
+ historical_flaky = sum(1 for t in historical_entries if t.outcome == 'flaky')
+ historical_total = len(historical_entries)
+ historical_flaky_rate = historical_flaky / historical_total if historical_total > 0 else 0
+
+ # Check if there's a significant increase in flakiness
+ if recent_flaky_rate >= min_flaky_rate and recent_flaky_rate > historical_flaky_rate * 1.5:
+ flaky_regressions[result.name] = {
+ 'result': result,
+ 'recent_flaky_rate': recent_flaky_rate,
+ 'historical_flaky_rate': historical_flaky_rate,
+ 'recent_executions': recent_entries,
+ 'historical_executions': historical_entries
+ }
+
+ return flaky_regressions
+
+ def get_cleared_tests(self, project: str, results: List[TestResult],
+ success_threshold: float = 0.7, min_executions: int = 5) -> Dict[str, Dict]:
+ """
+ Identify quarantined tests that are consistently passing and could be cleared.
+
+ Args:
+ project: The project name
+ results: List of test results
+ success_threshold: Required percentage of successful builds to be considered cleared
+ min_executions: Minimum number of executions required to make a determination
+ """
+ cleared_tests = {}
+ current_time = datetime.now(pytz.UTC)
+ chunk_start = current_time - timedelta(days=7) # Last 7 days for test cases
+
+ for result in results:
+ # Only consider tests with sufficient recent executions
+ recent_executions = result.timeline
+ if len(recent_executions) < min_executions:
+ continue
+
+ # Calculate success rate at class level
+ successful_runs = sum(1 for t in recent_executions
+ if t.outcome == 'passed')
+ success_rate = successful_runs / len(recent_executions)
+
+ # Check if the test meets clearing criteria at class level
+ if success_rate >= success_threshold:
+ # Verify no recent failures or flaky behavior
+ has_recent_issues = any(t.outcome in ['failed', 'flaky']
+ for t in recent_executions[-min_executions:])
+
+ if not has_recent_issues:
+ try:
+ # Get test case details
+ test_cases = self.get_test_case_details(
+ result.name,
+ project,
+ chunk_start,
+ current_time,
+ test_type="quarantinedTest"
+ )
+
+ # Only include if all test cases are also passing consistently
+ all_cases_passing = True
+ passing_test_cases = []
+
+ for test_case in test_cases:
+ case_total = test_case.outcome_distribution.total
+ if case_total >= min_executions:
+ case_success_rate = test_case.outcome_distribution.passed / case_total
+
+ # Check recent executions for the test case
+ recent_case_issues = any(t.outcome in ['failed', 'flaky']
+ for t in test_case.timeline[-min_executions:])
+
+ if case_success_rate >= success_threshold and not recent_case_issues:
+ passing_test_cases.append({
+ 'name': test_case.name,
+ 'success_rate': case_success_rate,
+ 'total_executions': case_total,
+ 'recent_executions': sorted(test_case.timeline,
+ key=lambda x: x.timestamp)[-min_executions:]
+ })
+ else:
+ all_cases_passing = False
+ break
+
+ if all_cases_passing and passing_test_cases:
+ cleared_tests[result.name] = {
+ 'result': result,
+ 'success_rate': success_rate,
+ 'total_executions': len(recent_executions),
+ 'successful_runs': successful_runs,
+ 'recent_executions': recent_executions[-min_executions:],
+ 'test_cases': passing_test_cases
+ }
+
+ except Exception as e:
+ logger.error(f"Error getting test case details for {result.name}: {str(e)}")
+
+ return cleared_tests
+
+ def update_cache(self, builds: Dict[str, BuildInfo]):
+ """
+ Update the build cache with new build information.
+
+ Args:
+ builds: Dictionary of build IDs to BuildInfo objects
+ """
+ current_time = datetime.now(pytz.UTC)
+
+ # Initialize cache if it doesn't exist
+ if not self.build_cache:
+ self.build_cache = BuildCache(current_time, {})
+
+ # Update builds and last update time
+ self.build_cache.builds.update(builds)
+ self.build_cache.last_update = current_time
+
+ # Save to all cache providers
+ self._save_cache()
+
+ logger.info(f"Updated cache with {len(builds)} builds")
+
+ def get_persistent_failing_tests(self, results: List[TestResult],
+ min_failure_rate: float = 0.2,
+ min_executions: int = 5) -> Dict[str, Dict]:
+ """
+ Identify tests that have been consistently failing/flaky over time.
+ Groups by test class and includes individual test cases.
+ """
+ persistent_failures = {}
+ current_time = datetime.now(pytz.UTC)
+ chunk_start = current_time - timedelta(days=7) # Last 7 days for test cases
+
+ # Group results by class
+ class_groups = {}
+ for result in results:
+ class_name = result.name.split('#')[0] # Get class name
+ if class_name not in class_groups:
+ class_groups[class_name] = []
+ class_groups[class_name].append(result)
+
+ # Analyze each class and its test cases
+ for class_name, class_results in class_groups.items():
+ class_total = sum(r.outcome_distribution.total for r in class_results)
+ class_problems = sum(r.outcome_distribution.failed + r.outcome_distribution.flaky
+ for r in class_results)
+
+ if class_total < min_executions:
+ continue
+
+ class_failure_rate = class_problems / class_total if class_total > 0 else 0
+
+ # Only include if class has significant failures
+ if class_failure_rate >= min_failure_rate:
+ try:
+ # Get detailed test case information using the same method as other reports
+ test_cases = self.get_test_case_details(
+ class_name,
+ "kafka",
+ chunk_start,
+ current_time,
+ test_type="test"
+ )
+
+ failing_test_cases = {}
+ for test_case in test_cases:
+ total_runs = test_case.outcome_distribution.total
+ if total_runs >= min_executions:
+ problem_runs = (test_case.outcome_distribution.failed +
+ test_case.outcome_distribution.flaky)
+ failure_rate = problem_runs / total_runs if total_runs > 0 else 0
+
+ if failure_rate >= min_failure_rate:
+ # Extract just the method name
+ method_name = test_case.name.split('.')[-1]
+ failing_test_cases[method_name] = {
+ 'result': test_case,
+ 'failure_rate': failure_rate,
+ 'total_executions': total_runs,
+ 'failed_executions': problem_runs,
+ 'timeline': sorted(test_case.timeline, key=lambda x: x.timestamp)
+ }
+
+ if failing_test_cases: # Only include classes that have problematic test cases
+ persistent_failures[class_name] = {
+ 'failure_rate': class_failure_rate,
+ 'total_executions': class_total,
+ 'failed_executions': class_problems,
+ 'test_cases': failing_test_cases
+ }
+
+ except Exception as e:
+ logger.error(f"Error getting test case details for {class_name}: {str(e)}")
+
+ return persistent_failures
+
+def get_develocity_class_link(class_name: str, threshold_days: int, test_type: str = None) -> str:
+ """
+ Generate Develocity link for a test class
+
+ Args:
+ class_name: Name of the test class
+ threshold_days: Number of days to look back in search
+ test_type: Type of test (e.g., "quarantinedTest", "test")
+ """
+ base_url = "https://ge.apache.org/scans/tests"
+ params = {
+ "search.rootProjectNames": "kafka",
+ "search.tags": "github,trunk",
+ "search.timeZoneId": "UTC",
+ "search.relativeStartTime": f"P{threshold_days}D",
+ "tests.container": class_name
+ }
+
+ if test_type:
+ params["search.tasks"] = test_type
+
+ return f"{base_url}?{'&'.join(f'{k}={requests.utils.quote(str(v))}' for k, v in params.items())}"
+
+def get_develocity_method_link(class_name: str, method_name: str, threshold_days: int, test_type: str = None) -> str:
+ """
+ Generate Develocity link for a test method
+
+ Args:
+ class_name: Name of the test class
+ method_name: Name of the test method
+ threshold_days: Number of days to look back in search
+ test_type: Type of test (e.g., "quarantinedTest", "test")
+ """
+ base_url = "https://ge.apache.org/scans/tests"
+
+ # Extract just the method name without the class prefix
+ if '.' in method_name:
+ method_name = method_name.split('.')[-1]
+
+ params = {
+ "search.rootProjectNames": "kafka",
+ "search.tags": "github,trunk",
+ "search.timeZoneId": "UTC",
+ "search.relativeStartTime": f"P{threshold_days}D",
+ "tests.container": class_name,
+ "tests.test": method_name
+ }
+
+ if test_type:
+ params["search.tasks"] = test_type
+
+ return f"{base_url}?{'&'.join(f'{k}={requests.utils.quote(str(v))}' for k, v in params.items())}"
+
+def print_most_problematic_tests(problematic_tests: Dict[str, Dict], threshold_days: int, test_type: str = None):
+ """Print a summary of the most problematic tests"""
+ print("\n## Most Problematic Tests")
+ if not problematic_tests:
+ print("No high-priority problematic tests found.")
+ return
+
+ print(f"Found {len(problematic_tests)} tests that have been quarantined for {threshold_days} days and are still failing frequently.")
+
+ # Print table with class and method information
+ print("\n
")
+
+ # Print detailed history
+ print("\n")
+ print("Detailed Execution History\n")
+
+ for class_name, class_details in sorted(persistent_failures.items(),
+ key=lambda x: x[1]['failure_rate'],
+ reverse=True):
+ print(f"\n### {class_name}")
+ print(f"* Overall Failure Rate: {class_details['failure_rate']:.2%}")
+ print(f"* Total Executions: {class_details['total_executions']}")
+ print(f"* Failed/Flaky Executions: {class_details['failed_executions']}")
+
+ for test_name, test_details in sorted(class_details['test_cases'].items(),
+ key=lambda x: x[1]['failure_rate'],
+ reverse=True):
+ print("\nRecent Executions:")
+ print("```")
+ print("Date/Time (UTC) Outcome Build ID")
+ print("-" * 44)
+ for entry in sorted(test_details['timeline'], key=lambda x: x.timestamp, reverse=True)[:5]:
+ date_str = entry.timestamp.strftime('%Y-%m-%d %H:%M')
+ print(f"{date_str:<17} {entry.outcome:<10} {entry.build_id}")
+ print("```")
+
+ print("")
+
+def print_cleared_tests(cleared_tests: Dict[str, Dict], threshold_days: int, test_type: str = None):
+ """Print tests that are ready to be unquarantined"""
+ print("\n## Cleared Tests (Ready for Unquarantine)")
+ if not cleared_tests:
+ print("No tests ready to be cleared from quarantine.")
+ return
+
+ # Calculate total number of test methods
+ total_methods = sum(len(details['test_cases']) for details in cleared_tests.values())
+
+ print(f"Found {len(cleared_tests)} test classes with {total_methods} test methods that have been consistently passing. "
+ f"These tests could be candidates for removing quarantine annotations at either class or method level.")
+
+ # Print table with class and method information
+ print("\n
")
+
+ # Print detailed history
+ print("\n")
+ print("Detailed Test Method History\n")
+
+ for test_name, details in sorted(cleared_tests.items(),
+ key=lambda x: x[1]['success_rate'],
+ reverse=True):
+ print(f"\n### {test_name}")
+ print(f"* Overall Success Rate: {details['success_rate']:.2%}")
+ print(f"* Total Executions: {details['total_executions']}")
+ print(f"* Consecutive Successful Runs: {details['successful_runs']}")
+
+ for test_case in details['test_cases']:
+ method_name = test_case['name'].split('.')[-1]
+ print(f"\n#### {method_name}")
+ print(f"* Success Rate: {test_case['success_rate']:.2%}")
+ print(f"* Total Runs: {test_case['total_executions']}")
+ print("\nRecent Executions:")
+ print("```")
+ print("Date/Time (UTC) Outcome Build ID")
+ print("-" * 44)
+ for entry in sorted(test_case['recent_executions'], key=lambda x: x.timestamp, reverse=True)[:5]:
+ date_str = entry.timestamp.strftime('%Y-%m-%d %H:%M')
+ print(f"{date_str:<17} {entry.outcome:<10} {entry.build_id}")
+ print("```")
+
+ print("")
+
+def main():
+ token = None
+ if os.environ.get("DEVELOCITY_ACCESS_TOKEN"):
+ token = os.environ.get("DEVELOCITY_ACCESS_TOKEN")
+ else:
+ print("No auth token was specified. You must set DEVELOCITY_ACCESS_TOKEN to your personal access token.")
+ exit(1)
+
+ # Configuration
+ BASE_URL = "https://ge.apache.org"
+ PROJECT = "kafka"
+ QUARANTINE_THRESHOLD_DAYS = 7
+ MIN_FAILURE_RATE = 0.1
+ RECENT_FAILURE_THRESHOLD = 0.5
+ SUCCESS_THRESHOLD = 0.7 # For cleared tests
+ MIN_FLAKY_RATE = 0.2 # For flaky regressions
+
+ analyzer = TestAnalyzer(BASE_URL, token)
+
+ try:
+ quarantined_builds = analyzer.get_build_info([], PROJECT, "quarantinedTest", 7, bypass_cache=True, fetch_all=True)
+ regular_builds = analyzer.get_build_info([], PROJECT, "test", 7, bypass_cache=True, fetch_all=True)
+
+ analyzer.update_cache(quarantined_builds)
+ analyzer.update_cache(regular_builds)
+
+ # Get test results
+ quarantined_results = analyzer.get_test_results(
+ PROJECT,
+ threshold_days=QUARANTINE_THRESHOLD_DAYS,
+ test_type="quarantinedTest"
+ )
+
+ regular_results = analyzer.get_test_results(
+ PROJECT,
+ threshold_days=7, # Last 7 days for regular tests
+ test_type="test"
+ )
+
+ # Generate reports
+ problematic_tests = analyzer.get_problematic_quarantined_tests(
+ quarantined_results,
+ QUARANTINE_THRESHOLD_DAYS,
+ MIN_FAILURE_RATE,
+ RECENT_FAILURE_THRESHOLD
+ )
+
+ flaky_regressions = analyzer.get_flaky_test_regressions(
+ PROJECT,
+ regular_results,
+ recent_days=7,
+ min_flaky_rate=MIN_FLAKY_RATE
+ )
+
+ cleared_tests = analyzer.get_cleared_tests(
+ PROJECT,
+ quarantined_results,
+ success_threshold=SUCCESS_THRESHOLD
+ )
+
+ # Get persistent failing tests (add after getting regular_results)
+ persistent_failures = analyzer.get_persistent_failing_tests(
+ regular_results,
+ min_failure_rate=0.2, # 20% failure rate threshold
+ min_executions=5
+ )
+
+ # Print report header
+ print(f"\n# Flaky Test Report for {datetime.now(pytz.UTC).strftime('%Y-%m-%d')}")
+ print(f"This report was run on {datetime.now(pytz.UTC).strftime('%Y-%m-%d %H:%M:%S')} UTC")
+
+ # Print each section
+ print_most_problematic_tests(problematic_tests, QUARANTINE_THRESHOLD_DAYS, test_type="quarantinedTest")
+ print_flaky_regressions(flaky_regressions, QUARANTINE_THRESHOLD_DAYS)
+ print_persistent_failing_tests(persistent_failures, QUARANTINE_THRESHOLD_DAYS)
+ print_cleared_tests(cleared_tests, QUARANTINE_THRESHOLD_DAYS, test_type="quarantinedTest")
+
+ except Exception as e:
+ logger.exception("Error occurred during report generation")
+ print(f"Error occurred: {str(e)}")
+
+if __name__ == "__main__":
+ # Configure logging
+ logging.basicConfig(
+ level=logging.INFO,
+ format='%(asctime)s - %(levelname)s - %(message)s',
+ handlers=[
+ logging.FileHandler("flaky_test_report.log")
+ ]
+ )
+ main()
diff --git a/.github/scripts/junit.py b/.github/scripts/junit.py
index 48d2f528a2631..b23f444720ba3 100644
--- a/.github/scripts/junit.py
+++ b/.github/scripts/junit.py
@@ -142,6 +142,7 @@ def parse_report(workspace_path, report_path, fp) -> Iterable[TestSuite]:
cur_suite: Optional[TestSuite] = None
partial_test_case = None
test_case_failed = False
+ test_case_skipped = False
for (event, elem) in xml.etree.ElementTree.iterparse(fp, events=["start", "end"]):
if event == "start":
if elem.tag == "testsuite":
@@ -158,6 +159,7 @@ def parse_report(workspace_path, report_path, fp) -> Iterable[TestSuite]:
test_time = float(elem.get("time", 0.0))
partial_test_case = partial(TestCase, test_name, class_name, test_time)
test_case_failed = False
+ test_case_skipped = False
elif elem.tag == "failure":
failure_message = elem.get("message")
if failure_message:
@@ -171,11 +173,12 @@ def parse_report(workspace_path, report_path, fp) -> Iterable[TestSuite]:
elif elem.tag == "skipped":
skipped = partial_test_case(None, None, None)
cur_suite.skipped_tests.append(skipped)
+ test_case_skipped = True
else:
pass
elif event == "end":
if elem.tag == "testcase":
- if not test_case_failed:
+ if not test_case_failed and not test_case_skipped:
passed = partial_test_case(None, None, None)
cur_suite.passed_tests.append(passed)
partial_test_case = None
@@ -303,7 +306,7 @@ def split_report_path(base_path: str, report_path: str) -> Tuple[str, str]:
logger.debug(f"Found skipped test: {skipped_test}")
skipped_table.append((simple_class_name, skipped_test.test_name))
- # Collect all tests that were run as part of quarantinedTest
+ # Only collect quarantined tests from the "quarantinedTest" task
if task == "quarantinedTest":
for test in all_suite_passed.values():
simple_class_name = test.class_name.split(".")[-1]
@@ -329,59 +332,89 @@ def split_report_path(base_path: str, report_path: str) -> Tuple[str, str]:
# The stdout (print) goes to the workflow step console output.
# The stderr (logger) is redirected to GITHUB_STEP_SUMMARY which becomes part of the HTML job summary.
report_url = get_env("JUNIT_REPORT_URL")
- report_md = f"Download [HTML report]({report_url})."
- summary = (f"{total_run} tests cases run in {duration}. "
+ if report_url:
+ report_md = f"Download [HTML report]({report_url})."
+ else:
+ report_md = "No report available. JUNIT_REPORT_URL was missing."
+ summary = (f"{total_run} tests cases run in {duration}.\n\n"
f"{total_success} {PASSED}, {total_failures} {FAILED}, "
- f"{total_flaky} {FLAKY}, {total_skipped} {SKIPPED}, and {total_errors} errors.")
+ f"{total_flaky} {FLAKY}, {total_skipped} {SKIPPED}, {len(quarantined_table)} {QUARANTINED}, and {total_errors} errors.")
print("## Test Summary\n")
- print(f"{summary} {report_md}\n")
+ print(f"{summary}\n\n{report_md}\n")
+
+ # Failed
if len(failed_table) > 0:
- logger.info(f"Found {len(failed_table)} test failures:")
- print("### Failed Tests\n")
+ print("")
+ print(f"Failed Tests {FAILED} ({len(failed_table)})\n")
print(f"| Module | Test | Message | Time |")
print(f"| ------ | ---- | ------- | ---- |")
+ logger.info(f"Found {len(failed_table)} test failures:")
for row in failed_table:
logger.info(f"{FAILED} {row[0]} > {row[1]}")
row_joined = " | ".join(row)
print(f"| {row_joined} |")
+ print("\n")
print("\n")
+
+ # Flaky
if len(flaky_table) > 0:
- logger.info(f"Found {len(flaky_table)} flaky test failures:")
- print("### Flaky Tests\n")
+ print("")
+ print(f"Flaky Tests {FLAKY} ({len(flaky_table)})\n")
print(f"| Module | Test | Message | Time |")
print(f"| ------ | ---- | ------- | ---- |")
+ logger.info(f"Found {len(flaky_table)} flaky test failures:")
for row in flaky_table:
logger.info(f"{FLAKY} {row[0]} > {row[1]}")
row_joined = " | ".join(row)
print(f"| {row_joined} |")
+ print("\n")
print("\n")
+
+ # Skipped
if len(skipped_table) > 0:
print("")
- print(f"{len(skipped_table)} Skipped Tests\n")
+ print(f"Skipped Tests {SKIPPED} ({len(skipped_table)})\n")
print(f"| Module | Test |")
print(f"| ------ | ---- |")
+ logger.debug(f"::group::Found {len(skipped_table)} skipped tests")
for row in skipped_table:
row_joined = " | ".join(row)
print(f"| {row_joined} |")
+ logger.debug(f"{row[0]} > {row[1]}")
print("\n")
+ logger.debug("::endgroup::")
+ print("\n")
+ # Quarantined
if len(quarantined_table) > 0:
- logger.info(f"Ran {len(quarantined_table)} quarantined test:")
print("")
- print(f"{len(quarantined_table)} Quarantined Tests\n")
+ print(f"Quarantined Tests {QUARANTINED} ({len(quarantined_table)})\n")
print(f"| Module | Test |")
print(f"| ------ | ---- |")
+ logger.debug(f"::group::Found {len(quarantined_table)} quarantined tests")
for row in quarantined_table:
- logger.info(f"{QUARANTINED} {row[0]} > {row[1]}")
row_joined = " | ".join(row)
print(f"| {row_joined} |")
+ logger.debug(f"{row[0]} > {row[1]}")
print("\n")
+ logger.debug("::endgroup::")
+
+ print("")
# Print special message if there was a timeout
- exit_code = get_env("GRADLE_EXIT_CODE", int)
- if exit_code == 124:
+ test_exit_code = get_env("GRADLE_TEST_EXIT_CODE", int)
+ quarantined_test_exit_code = get_env("GRADLE_QUARANTINED_TEST_EXIT_CODE", int)
+
+ if test_exit_code == 124 or quarantined_test_exit_code == 124:
+ # Special handling for timeouts. The exit code 124 is emitted by 'timeout' command used in build.yml.
+ # A watchdog script "thread-dump.sh" will use jstack to force a thread dump for any Gradle process
+ # still running after the timeout. We capture the exit codes of the two test tasks and pass them to
+ # this script. If either "test" or "quarantinedTest" fails due to timeout, we want to fail the overall build.
thread_dump_url = get_env("THREAD_DUMP_URL")
- logger.debug(f"Gradle command timed out. These are partial results!")
+ if test_exit_code == 124:
+ logger.debug(f"Gradle task for 'test' timed out. These are partial results!")
+ else:
+ logger.debug(f"Gradle task for 'quarantinedTest' timed out. These are partial results!")
logger.debug(summary)
if thread_dump_url:
print(f"\nThe JUnit tests were cancelled due to a timeout. Thread dumps were generated before the job was cancelled. "
@@ -390,7 +423,7 @@ def split_report_path(base_path: str, report_path: str) -> Tuple[str, str]:
else:
logger.debug(f"Failing this step because the tests timed out. Thread dumps were not archived, check logs in JUnit step.")
exit(1)
- elif exit_code in (0, 1):
+ elif test_exit_code in (0, 1):
logger.debug(summary)
if total_failures > 0:
logger.debug(f"Failing this step due to {total_failures} test failures")
@@ -401,5 +434,5 @@ def split_report_path(base_path: str, report_path: str) -> Tuple[str, str]:
else:
exit(0)
else:
- logger.debug(f"Gradle had unexpected exit code {exit_code}. Failing this step")
+ logger.debug(f"Gradle had unexpected exit code {test_exit_code}. Failing this step")
exit(1)
diff --git a/.github/scripts/requirements.txt b/.github/scripts/requirements.txt
index 900ce9a9b61bc..d59455f79dac6 100644
--- a/.github/scripts/requirements.txt
+++ b/.github/scripts/requirements.txt
@@ -13,3 +13,5 @@
# See the License for the specific language governing permissions and
# limitations under the License.
PyYAML~=6.0
+pytz==2024.2
+requests==2.32.3
diff --git a/.github/workflows/README.md b/.github/workflows/README.md
index f921ad78393ca..1087a3c1d6042 100644
--- a/.github/workflows/README.md
+++ b/.github/workflows/README.md
@@ -51,6 +51,67 @@ using this for very simple tasks such as applying labels or adding comments to P
_We must never run the untrusted PR code in the elevated `pull_request_target` context_
+## Our Workflows
+
+### Trunk Build
+
+The [ci.yml](ci.yml) is run when commits are pushed to trunk. This calls into [build.yml](build.yml)
+to run our main build. In the trunk build, we do not read from the Gradle cache,
+but we do write to it. Also, the test catalog is only updated from trunk builds.
+
+### PR Build
+
+Similar to trunk, this workflow starts in [ci.yml](ci.yml) and calls into [build.yml](build.yml).
+Unlike trunk, the PR builds _will_ utilize the Gradle cache.
+
+### PR Triage
+
+In order to get the attention of committers, we have a triage workflow for Pull Requests
+opened by non-committers. This workflow consists of two files:
+
+* [pr-update.yml](pr-update.yml) When a PR is created, add the `triage` label if
+ the PR was opened by a non-committer.
+* [pr-labels-cron.yml](pr-labels-cron.yml) Cron job to add `needs-attention` label to community
+ PRs that have not been reviewed after 7 days. Also includes a cron job to
+ remove the `triage` and `needs-attention` labels from PRs which have been reviewed.
+
+_The pr-update.yml workflow includes pull_request_target!_
+
+For committers to avoid having this label added, their membership in the ASF GitHub
+organization must be public. Here are the steps to take:
+
+* Navigate to the ASF organization's "People" page https://github.com/orgs/apache/people
+* Find yourself
+* Change "Organization Visibility" to Public
+
+Full documentation for this process can be found in GitHub's docs:
+https://docs.github.com/en/account-and-profile/setting-up-and-managing-your-personal-account-on-github/managing-your-membership-in-organizations/publicizing-or-hiding-organization-membership
+
+If you are a committer and do not want your membership in the ASF org listed as public,
+you will need to remove the `triage` label manually.
+
+### CI Approved
+
+Due to a combination of GitHub security and ASF's policy, we required explicit
+approval of workflows on PRs submitted by non-committers (and non-contributors).
+To simply this process, we have a `ci-approved` label which automatically approves
+these workflows.
+
+There are two files related to this workflow:
+
+* [pr-labeled.yml](pr-labeled.yml) approves a pending approval for PRs that have
+been labeled with `ci-approved`
+* [ci-requested.yml](ci-requested.yml) approves future workflow requests automatically
+if the PR has the `ci-approved` label
+
+_The pr-labeled.yml workflow includes pull_request_target!_
+
+### Stale PRs
+
+This one is straightforward. Using the "actions/stale" GitHub Action, we automatically
+label and eventually close PRs which have not had activity for some time. See the
+[stale.yml](stale.yml) workflow file for specifics.
+
## GitHub Actions Quirks
### Composite Actions
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index fb08bb2a9f510..d878a52ddac29 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -21,14 +21,10 @@ name: Check and Test
on:
workflow_call:
inputs:
- gradle-cache-read-only:
- description: "Should the Gradle cache be read-only?"
+ is-trunk:
+ description: "Is this a trunk build?"
default: true
type: boolean
- gradle-cache-write-only:
- description: "Should the Gradle cache be write-only?"
- default: false
- type: boolean
is-public-fork:
description: "Is this CI run from a public fork?"
default: true
@@ -105,8 +101,8 @@ jobs:
uses: ./.github/actions/setup-gradle
with:
java-version: 23
- gradle-cache-read-only: ${{ inputs.gradle-cache-read-only }}
- gradle-cache-write-only: ${{ inputs.gradle-cache-write-only }}
+ gradle-cache-read-only: ${{ !inputs.is-trunk }}
+ gradle-cache-write-only: ${{ inputs.is-trunk }}
develocity-access-key: ${{ secrets.GE_ACCESS_TOKEN }}
- name: Compile and validate
env:
@@ -117,7 +113,7 @@ jobs:
# --scan: Publish the build scan. This will only work on PRs from apache/kafka and trunk
# --no-scan: For public fork PRs, we won't attempt to publish the scan
run: |
- ./gradlew --build-cache --info $SCAN_ARG check -x test
+ ./gradlew --build-cache --info $SCAN_ARG check siteDocTar -x test
- name: Archive check reports
if: always()
uses: actions/upload-artifact@v4
@@ -137,6 +133,16 @@ jobs:
run: python .github/scripts/rat.py
env:
GITHUB_WORKSPACE: ${{ github.workspace }}
+ - name: Check generated documentation
+ # Check if there are any empty files under ./site-docs/generated, If any empty files are found, print an error
+ # message and list the empty files
+ run: |
+ tar zxvf core/build/distributions/kafka_2.13-$(./gradlew properties | grep version: | awk '{print $NF}' | head -n 1)-site-docs.tgz
+ if find ./site-docs/generated -type f -exec grep -L "." {} \; | grep -q "."; then
+ echo "One or more documentation files are empty!" >&2
+ find ./site-docs/generated -type f -exec grep -L "." {} \; >&2
+ exit 1
+ fi
test:
needs: [validate, load-catalog]
@@ -145,7 +151,9 @@ jobs:
strategy:
fail-fast: false
matrix:
- java: [ 23, 11 ] # If we change these, make sure to adjust ci-complete.yml
+ java: [ 23, 17 ] # If we change these, make sure to adjust ci-complete.yml
+ outputs:
+ timed-out: ${{ (steps.junit-test.outputs.gradle-exitcode == '124' || steps.junit-quarantined-test.outputs.gradle-exitcode == '124') }}
name: JUnit tests Java ${{ matrix.java }}
steps:
- name: Checkout code
@@ -158,39 +166,37 @@ jobs:
uses: ./.github/actions/setup-gradle
with:
java-version: ${{ matrix.java }}
- gradle-cache-read-only: ${{ inputs.gradle-cache-read-only }}
- gradle-cache-write-only: ${{ inputs.gradle-cache-write-only }}
+ gradle-cache-read-only: ${{ !inputs.is-trunk }}
+ gradle-cache-write-only: ${{ inputs.is-trunk }}
develocity-access-key: ${{ secrets.GE_ACCESS_TOKEN }}
# If the load-catalog job failed, we won't be able to download the artifact. Since we don't want this to fail
# the overall workflow, so we'll continue here without a test catalog.
- name: Load Test Catalog
+ id: load-test-catalog
uses: actions/download-artifact@v4
continue-on-error: true
with:
name: combined-test-catalog
- - name: Test
- # Gradle flags
- # --build-cache: Let Gradle restore the build cache
- # --no-scan: Don't attempt to publish the scan yet. We want to archive it first.
- # --continue: Keep running even if a test fails
- # -PcommitId Prevent the Git SHA being written into the jar files (which breaks caching)
+ - name: JUnit Quarantined Tests
+ id: junit-quarantined-test
+ uses: ./.github/actions/run-gradle
+ with:
+ test-task: quarantinedTest
+ timeout-minutes: 180
+ test-catalog-path: ${{ steps.load-test-catalog.outputs.download-path }}/combined-test-catalog.txt
+ build-scan-artifact-name: build-scan-quarantined-test-${{ matrix.java }}
+
+ - name: JUnit Tests
id: junit-test
- env:
- TIMEOUT_MINUTES: 180 # 3 hours
- run: |
- set +e
- ./.github/scripts/thread-dump.sh &
- timeout ${TIMEOUT_MINUTES}m ./gradlew --build-cache --continue --no-scan \
- -PtestLoggingEvents=started,passed,skipped,failed \
- -PmaxParallelForks=2 \
- -PmaxTestRetries=1 -PmaxTestRetryFailures=3 \
- -PmaxQuarantineTestRetries=3 -PmaxQuarantineTestRetryFailures=0 \
- -PcommitId=xxxxxxxxxxxxxxxx \
- quarantinedTest test
- exitcode="$?"
- echo "exitcode=$exitcode" >> $GITHUB_OUTPUT
+ uses: ./.github/actions/run-gradle
+ with:
+ test-task: test
+ timeout-minutes: 180 # 3 hours
+ test-catalog-path: ${{ steps.load-test-catalog.outputs.download-path }}/combined-test-catalog.txt
+ build-scan-artifact-name: build-scan-test-${{ matrix.java }}
+
- name: Archive JUnit HTML reports
uses: actions/upload-artifact@v4
id: junit-upload-artifact
@@ -200,6 +206,7 @@ jobs:
**/build/reports/tests/*
compression-level: 9
if-no-files-found: ignore
+
- name: Archive JUnit XML
uses: actions/upload-artifact@v4
with:
@@ -208,9 +215,10 @@ jobs:
build/junit-xml/**/*.xml
compression-level: 9
if-no-files-found: ignore
+
- name: Archive Thread Dumps
id: thread-dump-upload-artifact
- if: always() && steps.junit-test.outputs.exitcode == '124'
+ if: always() && (steps.junit-test.outputs.gradle-exitcode == '124' || steps.junit-quarantined-test.outputs.gradle-exitcode == '124')
uses: actions/upload-artifact@v4
with:
name: junit-thread-dumps-${{ matrix.java }}
@@ -218,13 +226,16 @@ jobs:
thread-dumps/*
compression-level: 9
if-no-files-found: ignore
+
- name: Parse JUnit tests
run: python .github/scripts/junit.py --export-test-catalog ./test-catalog >> $GITHUB_STEP_SUMMARY
env:
GITHUB_WORKSPACE: ${{ github.workspace }}
JUNIT_REPORT_URL: ${{ steps.junit-upload-artifact.outputs.artifact-url }}
THREAD_DUMP_URL: ${{ steps.thread-dump-upload-artifact.outputs.artifact-url }}
- GRADLE_EXIT_CODE: ${{ steps.junit-test.outputs.exitcode }}
+ GRADLE_TEST_EXIT_CODE: ${{ steps.junit-test.outputs.gradle-exitcode }}
+ GRADLE_QUARANTINED_TEST_EXIT_CODE: ${{ steps.junit-quarantined-test.outputs.gradle-exitcode }}
+
- name: Archive Test Catalog
if: ${{ always() && matrix.java == '23' }}
uses: actions/upload-artifact@v4
@@ -233,19 +244,11 @@ jobs:
path: test-catalog
compression-level: 9
if-no-files-found: ignore
- - name: Archive Build Scan
- if: always()
- uses: actions/upload-artifact@v4
- with:
- name: build-scan-test-${{ matrix.java }}
- path: ~/.gradle/build-scan-data
- compression-level: 9
- if-no-files-found: ignore
update-test-catalog:
name: Update Test Catalog
needs: test
- if: ${{ always() && !inputs.is-public-fork }}
+ if: ${{ always() && inputs.is-trunk && needs.test.outputs.timed-out == 'false' }}
runs-on: ubuntu-latest
permissions:
contents: write
diff --git a/.github/workflows/ci-complete.yml b/.github/workflows/ci-complete.yml
index 6478ae2c6daef..982eb7725e0a0 100644
--- a/.github/workflows/ci-complete.yml
+++ b/.github/workflows/ci-complete.yml
@@ -43,7 +43,8 @@ jobs:
strategy:
fail-fast: false
matrix:
- java: [ 23, 11 ]
+ java: [ 23, 17 ]
+ artifact-prefix: [ "build-scan-test-", "build-scan-quarantined-test-"]
steps:
- name: Env
run: printenv
@@ -66,7 +67,7 @@ jobs:
with:
github-token: ${{ github.token }}
run-id: ${{ github.event.workflow_run.id }}
- name: build-scan-test-${{ matrix.java }}
+ name: ${{ matrix.artifact-prefix }}${{ matrix.java }}
path: ~/.gradle/build-scan-data # This is where Gradle buffers unpublished build scan data when --no-scan is given
- name: Handle missing scan
if: ${{ steps.download-build-scan.outcome == 'failure' }}
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index dabec1a6eed6e..c02c47c7584be 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -19,6 +19,7 @@ on:
push:
branches:
- 'trunk'
+ - '4.0'
schedule:
- cron: '0 0 * * 6,0' # Run on Saturday and Sunday at midnight UTC
@@ -27,17 +28,17 @@ on:
types: [ opened, synchronize, ready_for_review, reopened ]
branches:
- 'trunk'
+ - '4.0'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
- cancel-in-progress: ${{ github.ref != 'refs/heads/trunk' }}
+ cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }}
jobs:
build:
uses: ./.github/workflows/build.yml
with:
- gradle-cache-read-only: ${{ github.ref != 'refs/heads/trunk' }}
- gradle-cache-write-only: ${{ github.ref == 'refs/heads/trunk' }}
+ is-trunk: ${{ github.ref == 'refs/heads/trunk' }}
is-public-fork: ${{ github.event.pull_request.head.repo.fork || false }}
secrets:
inherit
diff --git a/.github/workflows/docker_build_and_test.yml b/.github/workflows/docker_build_and_test.yml
index 0fe70ff023420..67acdf9fb7424 100644
--- a/.github/workflows/docker_build_and_test.yml
+++ b/.github/workflows/docker_build_and_test.yml
@@ -37,6 +37,11 @@ jobs:
uses: actions/setup-python@v5
with:
python-version: "3.10"
+ - name: Setup Docker Compose
+ run: |
+ sudo curl -L "https://github.com/docker/compose/releases/download/v2.30.3/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
+ sudo mv /usr/local/bin/docker-compose /usr/bin/docker-compose
+ sudo chmod +x /usr/bin/docker-compose
- name: Install dependencies
run: |
python -m pip install --upgrade pip
diff --git a/.github/workflows/docker_official_image_build_and_test.yml b/.github/workflows/docker_official_image_build_and_test.yml
index 419bfadc1416d..58866a19d6cab 100644
--- a/.github/workflows/docker_official_image_build_and_test.yml
+++ b/.github/workflows/docker_official_image_build_and_test.yml
@@ -36,6 +36,11 @@ jobs:
uses: actions/setup-python@v5
with:
python-version: "3.10"
+ - name: Setup Docker Compose
+ run: |
+ sudo curl -L "https://github.com/docker/compose/releases/download/v2.30.3/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
+ sudo mv /usr/local/bin/docker-compose /usr/bin/docker-compose
+ sudo chmod +x /usr/bin/docker-compose
- name: Install dependencies
run: |
python -m pip install --upgrade pip
diff --git a/.github/workflows/docker_scan.yml b/.github/workflows/docker_scan.yml
index 693b80c0626b6..a76916fffa916 100644
--- a/.github/workflows/docker_scan.yml
+++ b/.github/workflows/docker_scan.yml
@@ -26,7 +26,7 @@ jobs:
strategy:
matrix:
# This is an array of supported tags. Make sure this array only contains the supported tags
- supported_image_tag: ['latest', '3.7.1', "3.8.0", "3.8.1", "3.9.0"]
+ supported_image_tag: ['latest', '3.7.2', '3.8.1', '3.9.0']
steps:
- name: Run CVE scan
uses: aquasecurity/trivy-action@6e7b7d1fd3e4fef0c5fa8cce1229c54b2c9bd0d8 # v0.24.0
diff --git a/bin/zookeeper-server-start.sh b/.github/workflows/generate-reports.yml
old mode 100755
new mode 100644
similarity index 51%
rename from bin/zookeeper-server-start.sh
rename to .github/workflows/generate-reports.yml
index bd9c1142817c0..e2d52cea53a97
--- a/bin/zookeeper-server-start.sh
+++ b/.github/workflows/generate-reports.yml
@@ -1,4 +1,3 @@
-#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
@@ -14,31 +13,31 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-if [ $# -lt 1 ];
-then
- echo "USAGE: $0 [-daemon] zookeeper.properties"
- exit 1
-fi
-base_dir=$(dirname $0)
+name: Flaky Test Report
+on:
+ workflow_dispatch: # Let us run manually
-if [ "x$KAFKA_LOG4J_OPTS" = "x" ]; then
- export KAFKA_LOG4J_OPTS="-Dlog4j.configuration=file:$base_dir/../config/log4j.properties"
-fi
+ schedule:
+ - cron: '0 6 * * *' # Run daily at 6am UTC
-if [ "x$KAFKA_HEAP_OPTS" = "x" ]; then
- export KAFKA_HEAP_OPTS="-Xmx512M -Xms512M"
-fi
-
-EXTRA_ARGS=${EXTRA_ARGS-'-name zookeeper -loggc'}
-
-COMMAND=$1
-case $COMMAND in
- -daemon)
- EXTRA_ARGS="-daemon "$EXTRA_ARGS
- shift
- ;;
- *)
- ;;
-esac
-
-exec $base_dir/kafka-run-class.sh $EXTRA_ARGS org.apache.zookeeper.server.quorum.QuorumPeerMain "$@"
+jobs:
+ flaky-test-report:
+ name: Flaky Test Report
+ if : github.event.repository.fork == 'false'
+ permissions:
+ contents: read
+ runs-on: ubuntu-latest
+ steps:
+ - name: Env
+ run: printenv
+ env:
+ GITHUB_CONTEXT: ${{ toJson(github) }}
+ - name: Checkout code
+ uses: actions/checkout@v4
+ - name: Setup Python
+ uses: ./.github/actions/setup-python
+ - name: Run Report
+ env:
+ DEVELOCITY_ACCESS_TOKEN: ${{ secrets.DV_API_ACCESS }}
+ run: |
+ python ./.github/scripts/develocity_reports.py >> $GITHUB_STEP_SUMMARY
diff --git a/.github/workflows/pr-labeled.yml b/.github/workflows/pr-labeled.yml
index 34d5702ab1fce..87b39a659ec0c 100644
--- a/.github/workflows/pr-labeled.yml
+++ b/.github/workflows/pr-labeled.yml
@@ -44,10 +44,11 @@ jobs:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
PR_NUMBER: ${{ github.event.number }}
SHA: ${{ github.event.pull_request.head.sha }}
+ # Find the CI workflow run that is pending approval and approve it
run: |
set +e
echo "Found 'ci-approved' label on PR #$PR_NUMBER."
- RUN_ID=$(gh run list -L 1 -c $SHA -s action_required --json databaseId --jq '.[].databaseId')
+ RUN_ID=$(gh run list -L 1 -c $SHA -s action_required -w CI --json databaseId --jq '.[].databaseId')
if [ -z "$RUN_ID" ]; then
echo "No workflow run found for SHA $SHA";
exit 0;
@@ -64,4 +65,4 @@ jobs:
repository: ${{ github.repository }}
run_id: ${{ env.RUN_ID }}
pr_number: ${{ env.PR_NUMBER }}
- commit_sha: ${{ github.event.workflow_run.head_sha }}
+ commit_sha: ${{ github.event.pull_request.head.sha }}
diff --git a/.github/workflows/pr-labels-cron.yml b/.github/workflows/pr-labels-cron.yml
new file mode 100644
index 0000000000000..5faaca72ed36b
--- /dev/null
+++ b/.github/workflows/pr-labels-cron.yml
@@ -0,0 +1,92 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Fixup PR Labels
+
+on:
+ workflow_dispatch: # Let us run manually
+
+ schedule:
+ - cron: '0 3 * * *' # Run at 3:00 UTC nightly
+
+jobs:
+ # This job removes the triage and needs-attention labels from any PRs that have been reviewed. Once reviewed, it is
+ # assumed that a PR does _not_ need extra attention from the committers, so these labels can be removed.
+ fixup-pr-labels:
+ runs-on: ubuntu-latest
+ strategy:
+ matrix:
+ label: [triage, needs-attention]
+ steps:
+ - name: Env
+ run: printenv
+ env:
+ GITHUB_CONTEXT: ${{ toJson(github) }}
+ - name: Remove label
+ uses: actions/github-script@v7
+ continue-on-error: true
+ with:
+ github-token: ${{ secrets.GITHUB_TOKEN }}
+ script: |
+ console.log("Finding PRs with label '${{ matrix.label }}'");
+ github.paginate("GET /search/issues{?q}", {
+ q: "repo:apache/kafka label:${{ matrix.label }} is:pull-request"
+ })
+ .then((pulls) => {
+ pulls.forEach(pull => {
+ github.request("GET /repos/{owner}/{repo}/pulls/{pull_number}/reviews", {
+ owner: "apache",
+ repo: "kafka",
+ pull_number: pull.number,
+ headers: {
+ "X-GitHub-Api-Version": "2022-11-28"
+ }
+ }).then((resp) => {
+ console.log("Found " + resp.data.length + " reviews for PR " + pull.number);
+ if (resp.data.length > 0) {
+ console.log("Removing '${{ matrix.label }}' label from PR " + pull.number + " : " + pull.title);
+ github.rest.issues.removeLabel({
+ owner: "apache",
+ repo: "kafka",
+ issue_number: pull.number,
+ name: "${{ matrix.label }}"
+ });
+ }
+ });
+ });
+ });
+
+ # This job adds a 'needs-attention' label to any PR that has not been updated in 7 days and has been labeled with 'triage'.
+ # The idea here is to give committers a quick way to see which PRs have languished and need attention.
+ needs-attention:
+ runs-on: ubuntu-latest
+ needs: [fixup-pr-labels]
+ permissions:
+ issues: write
+ pull-requests: write
+ steps:
+ - uses: actions/stale@v9
+ with:
+ debug-only: ${{ inputs.dryRun || false }}
+ operations-per-run: ${{ inputs.operationsPerRun || 500 }}
+ days-before-stale: 7
+ days-before-close: -1
+ ignore-pr-updates: true
+ only-pr-labels: 'triage'
+ stale-pr-label: 'needs-attention'
+ stale-pr-message: |
+ A label of 'needs-attention' was automatically added to this PR in order to raise the
+ attention of the committers. Once this issue has been triaged, the `triage` label
+ should be removed to prevent this automation from happening again.
diff --git a/.github/workflows/pr-update.yml b/.github/workflows/pr-update.yml
index 31e0038705499..e1cd7214d6c36 100644
--- a/.github/workflows/pr-update.yml
+++ b/.github/workflows/pr-update.yml
@@ -25,9 +25,11 @@ on:
# * https://securitylab.github.com/resources/github-actions-preventing-pwn-requests/
pull_request_target:
types: [opened, reopened, synchronize]
+ branches:
+ - trunk
jobs:
- label_PRs:
+ add-labeler-labels:
name: Labeler
permissions:
contents: read
@@ -45,3 +47,24 @@ jobs:
PR_NUM: ${{github.event.number}}
run: |
./.github/scripts/label_small.sh
+
+ add-triage-label:
+ if: github.event.action == 'opened' || github.event.action == 'reopened'
+ name: Add triage label
+ runs-on: ubuntu-latest
+ permissions:
+ pull-requests: write
+ steps:
+ - name: Env
+ run: printenv
+ env:
+ GITHUB_CONTEXT: ${{ toJson(github) }}
+ # If the PR is from a non-committer, add triage label
+ - if: |
+ github.event.pull_request.author_association != 'MEMBER' &&
+ github.event.pull_request.author_association != 'OWNER'
+ env:
+ GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ GH_REPO: ${{ github.repository }}
+ NUMBER: ${{ github.event.pull_request.number }}
+ run: gh pr edit "$NUMBER" --add-label triage
diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml
index 816b754a968cd..9382d4173e94c 100644
--- a/.github/workflows/stale.yml
+++ b/.github/workflows/stale.yml
@@ -41,7 +41,7 @@ jobs:
- uses: actions/stale@v9
with:
debug-only: ${{ inputs.dryRun || false }}
- operations-per-run: ${{ inputs.operationsPerRun || 100 }}
+ operations-per-run: ${{ inputs.operationsPerRun || 500 }}
ascending: true
days-before-stale: 90
days-before-close: 30 # Since adding 'stale' will update the PR, days-before-close is relative to that.
diff --git a/LICENSE-binary b/LICENSE-binary
index 97e045fd7534e..c5f65ca15da1f 100644
--- a/LICENSE-binary
+++ b/LICENSE-binary
@@ -206,63 +206,64 @@ This project bundles some components that are also licensed under the Apache
License Version 2.0:
audience-annotations-0.12.0
-caffeine-2.9.3
+caffeine-3.1.1
commons-beanutils-1.9.4
-commons-cli-1.4
commons-collections-3.2.2
commons-digester-2.1
-commons-io-2.14.0
commons-lang3-3.12.0
commons-logging-1.3.2
commons-validator-1.9.0
-error_prone_annotations-2.10.0
+error_prone_annotations-2.14.0
jackson-annotations-2.16.2
jackson-core-2.16.2
jackson-databind-2.16.2
jackson-dataformat-csv-2.16.2
+jackson-dataformat-yaml-2.16.2
jackson-datatype-jdk8-2.16.2
+jackson-jakarta-rs-base-2.16.2
+jackson-jakarta-rs-json-provider-2.16.2
jackson-jaxrs-base-2.16.2
jackson-jaxrs-json-provider-2.16.2
-jackson-module-afterburner-2.16.2
+jackson-module-blackbird-2.16.2
+jackson-module-jakarta-xmlbind-annotations-2.16.2
jackson-module-jaxb-annotations-2.16.2
jackson-module-scala_2.13-2.16.2
-jakarta.validation-api-2.0.2
+jakarta.inject-api-2.0.1
+jakarta.validation-api-3.0.2
javassist-3.29.2-GA
-jetty-client-9.4.56.v20240826
+jetty-alpn-client-12.0.15
+jetty-client-12.0.15
jetty-continuation-9.4.56.v20240826
-jetty-http-9.4.56.v20240826
-jetty-io-9.4.56.v20240826
-jetty-security-9.4.56.v20240826
-jetty-server-9.4.56.v20240826
+jetty-ee10-servlet-12.0.15
+jetty-ee10-servlets-12.0.15
+jetty-http-12.0.15
+jetty-io-12.0.15
+jetty-security-12.0.15
+jetty-server-12.0.15
jetty-servlet-9.4.56.v20240826
jetty-servlets-9.4.56.v20240826
-jetty-util-9.4.56.v20240826
+jetty-session-12.0.15
+jetty-util-12.0.15
jetty-util-ajax-9.4.56.v20240826
jose4j-0.9.4
+log4j-api-2.24.1
+log4j-core-2.24.1
+log4j-core-test-2.24.1
+log4j-slf4j-impl-2.24.1
+log4j-1.2-api-2.24.1
lz4-java-1.8.0
maven-artifact-3.9.6
metrics-core-4.1.12.1
metrics-core-2.2.0
-netty-buffer-4.1.111.Final
-netty-codec-4.1.111.Final
-netty-common-4.1.111.Final
-netty-handler-4.1.111.Final
-netty-resolver-4.1.111.Final
-netty-transport-4.1.111.Final
-netty-transport-classes-epoll-4.1.111.Final
-netty-transport-native-epoll-4.1.111.Final
-netty-transport-native-unix-common-4.1.111.Final
opentelemetry-proto-1.0.0-alpha
plexus-utils-3.5.1
-reload4j-1.2.25
-rocksdbjni-7.9.2
+rocksdbjni-9.7.3
scala-library-2.13.15
scala-logging_2.13-3.9.5
scala-reflect-2.13.15
snappy-java-1.1.10.5
+snakeyaml-2.2
swagger-annotations-2.2.25
-zookeeper-3.8.4
-zookeeper-jute-3.8.4
===============================================================================
This product bundles various third-party components under other open source
@@ -273,36 +274,36 @@ See licenses/ for text of these licenses.
Eclipse Distribution License - v 1.0
see: licenses/eclipse-distribution-license-1.0
-jakarta.activation-api-1.2.2
-jakarta.xml.bind-api-2.3.3
+jakarta.activation-api-2.1.0
+jakarta.xml.bind-api-3.0.1
---------------------------------------
Eclipse Public License - v 2.0
see: licenses/eclipse-public-license-2.0
-jakarta.annotation-api-1.3.5
-jakarta.ws.rs-api-2.1.6
-hk2-api-2.6.1
-hk2-locator-2.6.1
-hk2-utils-2.6.1
+jakarta.annotation-api-2.1.1
+jakarta.ws.rs-api-3.1.0
+hk2-api-3.0.6
+hk2-locator-3.0.6
+hk2-utils-3.0.6
osgi-resource-locator-1.0.3
-aopalliance-repackaged-2.6.1
+aopalliance-repackaged-3.0.6
jakarta.inject-2.6.1
-jersey-client-2.39.1
-jersey-common-2.39.1
-jersey-container-servlet-2.39.1
-jersey-container-servlet-core-2.39.1
-jersey-hk2-2.39.1
-jersey-server-2.39.1
+jersey-client-3.1.9
+jersey-common-3.1.9
+jersey-container-servlet-3.1.9
+jersey-container-servlet-core-3.1.9
+jersey-hk2-3.1.9
+jersey-server-3.1.9
---------------------------------------
CDDL 1.1 + GPLv2 with classpath exception
see: licenses/CDDL+GPL-1.1
+jakarta.servlet-api-6.0.0
javax.activation-api-1.2.0
javax.annotation-api-1.3.2
javax.servlet-api-3.1.0
-javax.ws.rs-api-2.1.1
jaxb-api-2.3.1
activation-1.1.1
@@ -313,7 +314,6 @@ argparse4j-0.7.0, see: licenses/argparse-MIT
classgraph-4.8.173, see: licenses/classgraph-MIT
jopt-simple-5.0.4, see: licenses/jopt-simple-MIT
slf4j-api-1.7.36, see: licenses/slf4j-MIT
-slf4j-reload4j-1.7.36, see: licenses/slf4j-MIT
pcollections-4.0.1, see: licenses/pcollections-MIT
---------------------------------------
@@ -329,6 +329,7 @@ jline-3.25.1, see: licenses/jline-BSD-3-clause
jsr305-3.0.2, see: licenses/jsr305-BSD-3-clause
paranamer-2.8, see: licenses/paranamer-BSD-3-clause
protobuf-java-3.25.5, see: licenses/protobuf-java-BSD-3-clause
+jakarta.activation-2.0.1, see: licenses/jakarta-BSD-3-clause
---------------------------------------
Go License
diff --git a/NOTICE b/NOTICE
index c9806e02513a9..36f87db19a965 100644
--- a/NOTICE
+++ b/NOTICE
@@ -1,5 +1,5 @@
Apache Kafka
-Copyright 2024 The Apache Software Foundation.
+Copyright 2025 The Apache Software Foundation.
This product includes software developed at
The Apache Software Foundation (https://www.apache.org/).
@@ -15,9 +15,5 @@ The streams-scala (streams/streams-scala) module was donated by Lightbend and th
Copyright (C) 2018 Lightbend Inc.
Copyright (C) 2017-2018 Alexis Seigneurin.
-This project contains the following code copied from Apache Hadoop:
-clients/src/main/java/org/apache/kafka/common/utils/PureJavaCrc32C.java
-Some portions of this file Copyright (c) 2004-2006 Intel Corporation and licensed under the BSD license.
-
This project contains the following code copied from Apache Hive:
streams/src/main/java/org/apache/kafka/streams/state/internals/Murmur3.java
diff --git a/NOTICE-binary b/NOTICE-binary
index d3207a131e25b..50dabb33d61de 100644
--- a/NOTICE-binary
+++ b/NOTICE-binary
@@ -15,10 +15,6 @@ The streams-scala (streams/streams-scala) module was donated by Lightbend and th
Copyright (C) 2018 Lightbend Inc.
Copyright (C) 2017-2018 Alexis Seigneurin.
-This project contains the following code copied from Apache Hadoop:
-clients/src/main/java/org/apache/kafka/common/utils/PureJavaCrc32C.java
-Some portions of this file Copyright (c) 2004-2006 Intel Corporation and licensed under the BSD license.
-
This project contains the following code copied from Apache Hive:
streams/src/main/java/org/apache/kafka/streams/state/internals/Murmur3.java
@@ -600,45 +596,6 @@ limitations under the License.
This software includes projects with other licenses -- see `doc/LICENSE.md`.
-Apache ZooKeeper - Server
-Copyright 2008-2021 The Apache Software Foundation
-
-This product includes software developed at
-The Apache Software Foundation (http://www.apache.org/).
-
-
-Apache ZooKeeper - Jute
-Copyright 2008-2021 The Apache Software Foundation
-
-This product includes software developed at
-The Apache Software Foundation (http://www.apache.org/).
-
-
-The Netty Project
- =================
-
-Please visit the Netty web site for more information:
-
- * https://netty.io/
-
-Copyright 2014 The Netty Project
-
-The Netty Project licenses this file to you under the Apache License,
-version 2.0 (the "License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at:
-
- https://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-License for the specific language governing permissions and limitations
-under the License.
-
-Also, please refer to each LICENSE..txt file, which is located in
-the 'license' directory of the distribution file, for the license terms of the
-components that this product depends on.
-
-------------------------------------------------------------------------------
This product contains the extensions to Java Collections Framework which has
been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene:
diff --git a/README.md b/README.md
index e996365475b37..efcddbcd35966 100644
--- a/README.md
+++ b/README.md
@@ -1,13 +1,24 @@
-Apache Kafka
-=================
-See our [web site](https://kafka.apache.org) for details on the project.
+
+
+
+
+
+
+
+
+[](https://github.com/apache/kafka/actions/workflows/ci.yml?query=event%3Apush+branch%3Atrunk)
+[](https://github.com/apache/kafka/actions/workflows/generate-reports.yml?query=event%3Aschedule+branch%3Atrunk)
+
+[**Apache Kafka**](https://kafka.apache.org) is an open-source distributed event streaming platform used by thousands of
+
+companies for high-performance data pipelines, streaming analytics, data integration, and mission-critical applications.
You need to have [Java](http://www.oracle.com/technetwork/java/javase/downloads/index.html) installed.
-We build and test Apache Kafka with 11, 17 and 21. We set the `release` parameter in javac and scalac
-to `11` to ensure the generated binaries are compatible with Java 11 or higher (independently of the Java version
-used for compilation). Java 11 support for the broker and tools has been deprecated since Apache Kafka 3.7 and removal
-of both is planned for Apache Kafka 4.0.([KIP-1013](https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=284789510) for more details).
+We build and test Apache Kafka with 17 and 23. The `release` parameter in javac is set to `11` for the clients
+and streams modules, and `17` for the rest, ensuring compatibility with their respective
+minimum Java versions. Similarly, the `release` parameter in scalac is set to `11` for the streams modules and `17`
+for the rest.
Scala 2.13 is the only supported version in Apache Kafka.
@@ -52,10 +63,10 @@ Follow instructions in https://kafka.apache.org/quickstart
./gradlew clients:test --tests org.apache.kafka.clients.MetadataTest.testTimeToNextUpdate
### Running a particular unit/integration test with log4j output ###
-By default, there will be only small number of logs output while testing. You can adjust it by changing the `log4j.properties` file in the module's `src/test/resources` directory.
+By default, there will be only small number of logs output while testing. You can adjust it by changing the `log4j2.yaml` file in the module's `src/test/resources` directory.
-For example, if you want to see more logs for clients project tests, you can modify [the line](https://github.com/apache/kafka/blob/trunk/clients/src/test/resources/log4j.properties#L21) in `clients/src/test/resources/log4j.properties`
-to `log4j.logger.org.apache.kafka=INFO` and then run:
+For example, if you want to see more logs for clients project tests, you can modify [the line](https://github.com/apache/kafka/blob/trunk/clients/src/test/resources/log4j2.yaml#L35) in `clients/src/test/resources/log4j2.yaml`
+to `level: INFO` and then run:
./gradlew cleanTest clients:test --tests NetworkClientTest
@@ -94,12 +105,12 @@ fail due to code changes. You can just run:
./gradlew processMessages processTestMessages
-### Running a Kafka broker in KRaft mode
+### Running a Kafka broker
Using compiled files:
KAFKA_CLUSTER_ID="$(./bin/kafka-storage.sh random-uuid)"
- ./bin/kafka-storage.sh format -t $KAFKA_CLUSTER_ID -c config/kraft/reconfig-server.properties
+ ./bin/kafka-storage.sh format --standalone -t $KAFKA_CLUSTER_ID -c config/kraft/reconfig-server.properties
./bin/kafka-server-start.sh config/kraft/reconfig-server.properties
Using docker image:
@@ -123,10 +134,14 @@ Streams has multiple sub-projects, but you can run all the tests:
./gradlew tasks
### Building IDE project ####
-*Note that this is not strictly necessary (IntelliJ IDEA has good built-in support for Gradle projects, for example).*
+*Note Please ensure that JDK17 is used when developing Kafka.*
+
+IntelliJ supports Gradle natively and it will automatically check Java syntax and compatibility for each module, even if
+the Java version shown in the `Structure > Project Settings > Modules` may not be the correct one.
+
+When it comes to Eclipse, run:
./gradlew eclipse
- ./gradlew idea
The `eclipse` task has been configured to use `${project_dir}/build_eclipse` as Eclipse's build directory. Eclipse's default
build directory (`${project_dir}/bin`) clashes with Kafka's scripts directory and we don't use Gradle's build directory
@@ -161,6 +176,10 @@ Please note for this to work you should create/update user maven settings (typic
...
+### Installing all projects to the local Maven repository ###
+
+ ./gradlew -PskipSigning=true publishToMavenLocal
+
### Installing specific projects to the local Maven repository ###
./gradlew -PskipSigning=true :streams:publishToMavenLocal
@@ -179,6 +198,7 @@ You can run checkstyle using:
The checkstyle warnings will be found in `reports/checkstyle/reports/main.html` and `reports/checkstyle/reports/test.html` files in the
subproject build directories. They are also printed to the console. The build will fail if Checkstyle fails.
+For experiments (or regression testing purposes) add `-PcheckstyleVersion=X.y.z` switch (to override project-defined checkstyle version).
#### Spotless ####
The import order is a part of static check. please call `spotlessApply` to optimize the imports of Java codes before filing pull request.
diff --git a/Vagrantfile b/Vagrantfile
index a053be28d01dc..3a6b5a5afe1d9 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -55,11 +55,11 @@ ec2_iam_instance_profile_name = nil
ebs_volume_type = 'gp3'
-jdk_major = '8'
-jdk_full = '8u202-linux-x64'
+jdk_major = '17'
+jdk_full = '17-linux-x64'
local_config_file = File.join(File.dirname(__FILE__), "Vagrantfile.local")
-if File.exists?(local_config_file) then
+if File.exist?(local_config_file) then
eval(File.read(local_config_file), binding, "Vagrantfile.local")
end
diff --git a/bin/connect-distributed.sh b/bin/connect-distributed.sh
index b8088ad923451..2fd00f025dc51 100755
--- a/bin/connect-distributed.sh
+++ b/bin/connect-distributed.sh
@@ -22,8 +22,8 @@ fi
base_dir=$(dirname $0)
-if [ "x$KAFKA_LOG4J_OPTS" = "x" ]; then
- export KAFKA_LOG4J_OPTS="-Dlog4j.configuration=file:$base_dir/../config/connect-log4j.properties"
+if [ -z "$KAFKA_LOG4J_OPTS" ]; then
+ export KAFKA_LOG4J_OPTS="-Dlog4j2.configurationFile=$base_dir/../config/connect-log4j2.yaml"
fi
if [ "x$KAFKA_HEAP_OPTS" = "x" ]; then
diff --git a/bin/connect-mirror-maker.sh b/bin/connect-mirror-maker.sh
index 8e2b2e162daac..daf6e32042d7d 100755
--- a/bin/connect-mirror-maker.sh
+++ b/bin/connect-mirror-maker.sh
@@ -22,8 +22,8 @@ fi
base_dir=$(dirname $0)
-if [ "x$KAFKA_LOG4J_OPTS" = "x" ]; then
- export KAFKA_LOG4J_OPTS="-Dlog4j.configuration=file:$base_dir/../config/connect-log4j.properties"
+if [ -z "$KAFKA_LOG4J_OPTS" ]; then
+ export KAFKA_LOG4J_OPTS="-Dlog4j2.configurationFile=$base_dir/../config/connect-log4j2.yaml"
fi
if [ "x$KAFKA_HEAP_OPTS" = "x" ]; then
diff --git a/bin/connect-standalone.sh b/bin/connect-standalone.sh
index bef78d658fda9..d0d8b8bff32ba 100755
--- a/bin/connect-standalone.sh
+++ b/bin/connect-standalone.sh
@@ -22,8 +22,8 @@ fi
base_dir=$(dirname $0)
-if [ "x$KAFKA_LOG4J_OPTS" = "x" ]; then
- export KAFKA_LOG4J_OPTS="-Dlog4j.configuration=file:$base_dir/../config/connect-log4j.properties"
+if [ -z "$KAFKA_LOG4J_OPTS" ]; then
+ export KAFKA_LOG4J_OPTS="-Dlog4j2.configurationFile=$base_dir/../config/connect-log4j2.yaml"
fi
if [ "x$KAFKA_HEAP_OPTS" = "x" ]; then
diff --git a/bin/kafka-acls.sh b/bin/kafka-acls.sh
index 8fa65542e10bf..ffbb1e198109b 100755
--- a/bin/kafka-acls.sh
+++ b/bin/kafka-acls.sh
@@ -14,4 +14,4 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-exec $(dirname $0)/kafka-run-class.sh kafka.admin.AclCommand "$@"
+exec $(dirname $0)/kafka-run-class.sh org.apache.kafka.tools.AclCommand "$@"
diff --git a/bin/kafka-run-class.sh b/bin/kafka-run-class.sh
index b3291e461f2ba..8bd1b17623b12 100755
--- a/bin/kafka-run-class.sh
+++ b/bin/kafka-run-class.sh
@@ -23,7 +23,7 @@ fi
# WINDOWS_OS_FORMAT == 1 if Cygwin or MinGW is detected, else 0.
if [[ $(uname -a) =~ "CYGWIN" || $(uname -a) =~ "MINGW" || $(uname -a) =~ "MSYS" ]]; then
WINDOWS_OS_FORMAT=1
- export MSYS2_ARG_CONV_EXCL="-Xlog:gc*:file=;-Dlog4j.configuration=;$MSYS2_ARG_CONV_EXCL"
+ export MSYS2_ARG_CONV_EXCL="-Xlog:gc*:file=;-Dlog4j2.configurationFile=;$MSYS2_ARG_CONV_EXCL"
else
WINDOWS_OS_FORMAT=0
fi
@@ -116,14 +116,6 @@ else
CLASSPATH="$file":"$CLASSPATH"
fi
done
- if [ "$SHORT_VERSION_NO_DOTS" = "0100" ]; then
- CLASSPATH="/opt/kafka-$UPGRADE_KAFKA_STREAMS_TEST_VERSION/libs/zkclient-0.8.jar":"$CLASSPATH"
- CLASSPATH="/opt/kafka-$UPGRADE_KAFKA_STREAMS_TEST_VERSION/libs/zookeeper-3.4.6.jar":"$CLASSPATH"
- fi
- if [ "$SHORT_VERSION_NO_DOTS" = "0101" ]; then
- CLASSPATH="/opt/kafka-$UPGRADE_KAFKA_STREAMS_TEST_VERSION/libs/zkclient-0.9.jar":"$CLASSPATH"
- CLASSPATH="/opt/kafka-$UPGRADE_KAFKA_STREAMS_TEST_VERSION/libs/zookeeper-3.4.8.jar":"$CLASSPATH"
- fi
fi
for file in "$streams_dependant_clients_lib_dir"/rocksdb*.jar;
@@ -228,11 +220,18 @@ fi
# Log4j settings
if [ -z "$KAFKA_LOG4J_OPTS" ]; then
# Log to console. This is a tool.
- LOG4J_DIR="$base_dir/config/tools-log4j.properties"
+ LOG4J_DIR="$base_dir/config/tools-log4j2.yaml"
# If Cygwin is detected, LOG4J_DIR is converted to Windows format.
(( WINDOWS_OS_FORMAT )) && LOG4J_DIR=$(cygpath --path --mixed "${LOG4J_DIR}")
- KAFKA_LOG4J_OPTS="-Dlog4j.configuration=file:${LOG4J_DIR}"
+ KAFKA_LOG4J_OPTS="-Dlog4j2.configurationFile=${LOG4J_DIR}"
else
+ if echo "$KAFKA_LOG4J_OPTS" | grep -E "log4j\.[^[:space:]]+(\.properties|\.xml)$"; then
+ # Enable Log4j 1.x configuration compatibility mode for Log4j 2
+ export LOG4J_COMPATIBILITY=true
+ echo DEPRECATED: A Log4j 1.x configuration file has been detected, which is no longer recommended. >&2
+ echo To use a Log4j 2.x configuration, please see https://logging.apache.org/log4j/2.x/migrate-from-log4j1.html#Log4j2ConfigurationFormat for details about Log4j configuration file migration. >&2
+ echo You can also use the \$KAFKA_HOME/config/tools-log4j2.yaml file as a starting point. Make sure to remove the Log4j 1.x configuration after completing the migration. >&2
+ fi
# create logs directory
if [ ! -d "$LOG_DIR" ]; then
mkdir -p "$LOG_DIR"
diff --git a/bin/kafka-server-start.sh b/bin/kafka-server-start.sh
index 5a53126172de9..b98a8bebe7f85 100755
--- a/bin/kafka-server-start.sh
+++ b/bin/kafka-server-start.sh
@@ -21,8 +21,8 @@ then
fi
base_dir=$(dirname $0)
-if [ "x$KAFKA_LOG4J_OPTS" = "x" ]; then
- export KAFKA_LOG4J_OPTS="-Dlog4j.configuration=file:$base_dir/../config/log4j.properties"
+if [ -z "$KAFKA_LOG4J_OPTS" ]; then
+ export KAFKA_LOG4J_OPTS="-Dlog4j2.configurationFile=$base_dir/../config/log4j2.yaml"
fi
if [ "x$KAFKA_HEAP_OPTS" = "x" ]; then
diff --git a/shell/src/test/resources/log4j.properties b/bin/kafka-share-consumer-perf-test.sh
old mode 100644
new mode 100755
similarity index 78%
rename from shell/src/test/resources/log4j.properties
rename to bin/kafka-share-consumer-perf-test.sh
index a72a9693de2af..8e490ca02c430
--- a/shell/src/test/resources/log4j.properties
+++ b/bin/kafka-share-consumer-perf-test.sh
@@ -1,3 +1,4 @@
+#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
@@ -12,8 +13,8 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-log4j.rootLogger=DEBUG, stdout
-log4j.appender.stdout=org.apache.log4j.ConsoleAppender
-log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
-log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c:%L)%n
+if [ "x$KAFKA_HEAP_OPTS" = "x" ]; then
+ export KAFKA_HEAP_OPTS="-Xmx512M"
+fi
+exec $(dirname $0)/kafka-run-class.sh org.apache.kafka.tools.ShareConsumerPerformance "$@"
diff --git a/bin/kafka-verifiable-share-consumer.sh b/bin/kafka-verifiable-share-consumer.sh
new file mode 100755
index 0000000000000..e30178cebcafa
--- /dev/null
+++ b/bin/kafka-verifiable-share-consumer.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ "x$KAFKA_HEAP_OPTS" = "x" ]; then
+ export KAFKA_HEAP_OPTS="-Xmx512M"
+fi
+exec $(dirname $0)/kafka-run-class.sh org.apache.kafka.tools.VerifiableShareConsumer "$@"
diff --git a/bin/windows/connect-distributed.bat b/bin/windows/connect-distributed.bat
index 0535085bde507..396e5baf27801 100644
--- a/bin/windows/connect-distributed.bat
+++ b/bin/windows/connect-distributed.bat
@@ -27,7 +27,7 @@ popd
rem Log4j settings
IF ["%KAFKA_LOG4J_OPTS%"] EQU [""] (
- set KAFKA_LOG4J_OPTS=-Dlog4j.configuration=file:%BASE_DIR%/config/connect-log4j.properties
+ set KAFKA_LOG4J_OPTS=-Dlog4j2.configurationFile=%BASE_DIR%/config/connect-log4j2.yaml
)
"%~dp0kafka-run-class.bat" org.apache.kafka.connect.cli.ConnectDistributed %*
diff --git a/bin/windows/connect-standalone.bat b/bin/windows/connect-standalone.bat
index 12ebb21dc9a85..7ad655d5d39b0 100644
--- a/bin/windows/connect-standalone.bat
+++ b/bin/windows/connect-standalone.bat
@@ -27,7 +27,7 @@ popd
rem Log4j settings
IF ["%KAFKA_LOG4J_OPTS%"] EQU [""] (
- set KAFKA_LOG4J_OPTS=-Dlog4j.configuration=file:%BASE_DIR%/config/connect-log4j.properties
+ set KAFKA_LOG4J_OPTS=-Dlog4j2.configurationFile=%BASE_DIR%/config/connect-log4j2.yaml
)
"%~dp0kafka-run-class.bat" org.apache.kafka.connect.cli.ConnectStandalone %*
diff --git a/bin/windows/kafka-acls.bat b/bin/windows/kafka-acls.bat
index 8f0be85c0455a..12c4a9a69a7e5 100644
--- a/bin/windows/kafka-acls.bat
+++ b/bin/windows/kafka-acls.bat
@@ -14,4 +14,4 @@ rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
rem See the License for the specific language governing permissions and
rem limitations under the License.
-"%~dp0kafka-run-class.bat" kafka.admin.AclCommand %*
+"%~dp0kafka-run-class.bat" org.apache.kafka.tools.AclCommand %*
diff --git a/bin/windows/kafka-run-class.bat b/bin/windows/kafka-run-class.bat
index a163ccd0a7c08..ca151e5df96ed 100755
--- a/bin/windows/kafka-run-class.bat
+++ b/bin/windows/kafka-run-class.bat
@@ -116,8 +116,17 @@ IF ["%LOG_DIR%"] EQU [""] (
rem Log4j settings
IF ["%KAFKA_LOG4J_OPTS%"] EQU [""] (
- set KAFKA_LOG4J_OPTS=-Dlog4j.configuration=file:%BASE_DIR%/config/tools-log4j.properties
+ set KAFKA_LOG4J_OPTS=-Dlog4j2.configurationFile=file:%BASE_DIR%/config/tools-log4j2.yaml
) ELSE (
+ rem Check if Log4j 1.x configuration options are present in KAFKA_LOG4J_OPTS
+ echo %KAFKA_LOG4J_OPTS% | findstr /r /c:"log4j\.[^ ]*(\.properties|\.xml)$" >nul
+ IF %ERRORLEVEL% == 0 (
+ rem Enable Log4j 1.x configuration compatibility mode for Log4j 2
+ set LOG4J_COMPATIBILITY=true
+ echo DEPRECATED: A Log4j 1.x configuration file has been detected, which is no longer recommended. >&2
+ echo To use a Log4j 2.x configuration, please see https://logging.apache.org/log4j/2.x/migrate-from-log4j1.html#Log4j2ConfigurationFormat for details about Log4j configuration file migration. >&2
+ echo You can also use the %BASE_DIR%/config/tool-log4j2.yaml file as a starting point. Make sure to remove the Log4j 1.x configuration after completing the migration. >&2
+ )
rem create logs directory
IF not exist "%LOG_DIR%" (
mkdir "%LOG_DIR%"
diff --git a/bin/windows/kafka-server-start.bat b/bin/windows/kafka-server-start.bat
index 8624eda9ff089..6cf0533bd05b5 100644
--- a/bin/windows/kafka-server-start.bat
+++ b/bin/windows/kafka-server-start.bat
@@ -21,7 +21,7 @@ IF [%1] EQU [] (
SetLocal
IF ["%KAFKA_LOG4J_OPTS%"] EQU [""] (
- set KAFKA_LOG4J_OPTS=-Dlog4j.configuration=file:%~dp0../../config/log4j.properties
+ set KAFKA_LOG4J_OPTS=-Dlog4j2.configurationFile=%~dp0../../config/log4j2.yaml
)
IF ["%KAFKA_HEAP_OPTS%"] EQU [""] (
rem detect OS architecture
diff --git a/bin/windows/zookeeper-server-stop.bat b/bin/windows/kafka-share-consumer-perf-test.bat
similarity index 85%
rename from bin/windows/zookeeper-server-stop.bat
rename to bin/windows/kafka-share-consumer-perf-test.bat
index 8b57dd8d63069..5bfcb1ed11bcf 100644
--- a/bin/windows/zookeeper-server-stop.bat
+++ b/bin/windows/kafka-share-consumer-perf-test.bat
@@ -14,4 +14,7 @@ rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
rem See the License for the specific language governing permissions and
rem limitations under the License.
-wmic process where (commandline like "%%zookeeper%%" and not name="wmic.exe") delete
+SetLocal
+set KAFKA_HEAP_OPTS=-Xmx512M -Xms512M
+"%~dp0kafka-run-class.bat" org.apache.kafka.tools.ShareConsumerPerformance %*
+EndLocal
diff --git a/bin/windows/zookeeper-server-start.bat b/bin/windows/zookeeper-server-start.bat
deleted file mode 100644
index f201a585135d2..0000000000000
--- a/bin/windows/zookeeper-server-start.bat
+++ /dev/null
@@ -1,30 +0,0 @@
-@echo off
-rem Licensed to the Apache Software Foundation (ASF) under one or more
-rem contributor license agreements. See the NOTICE file distributed with
-rem this work for additional information regarding copyright ownership.
-rem The ASF licenses this file to You under the Apache License, Version 2.0
-rem (the "License"); you may not use this file except in compliance with
-rem the License. You may obtain a copy of the License at
-rem
-rem http://www.apache.org/licenses/LICENSE-2.0
-rem
-rem Unless required by applicable law or agreed to in writing, software
-rem distributed under the License is distributed on an "AS IS" BASIS,
-rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-rem See the License for the specific language governing permissions and
-rem limitations under the License.
-
-IF [%1] EQU [] (
- echo USAGE: %0 zookeeper.properties
- EXIT /B 1
-)
-
-SetLocal
-IF ["%KAFKA_LOG4J_OPTS%"] EQU [""] (
- set KAFKA_LOG4J_OPTS=-Dlog4j.configuration=file:%~dp0../../config/log4j.properties
-)
-IF ["%KAFKA_HEAP_OPTS%"] EQU [""] (
- set KAFKA_HEAP_OPTS=-Xmx512M -Xms512M
-)
-"%~dp0kafka-run-class.bat" org.apache.zookeeper.server.quorum.QuorumPeerMain %*
-EndLocal
diff --git a/bin/windows/zookeeper-shell.bat b/bin/windows/zookeeper-shell.bat
deleted file mode 100644
index f1c86c430c170..0000000000000
--- a/bin/windows/zookeeper-shell.bat
+++ /dev/null
@@ -1,22 +0,0 @@
-@echo off
-rem Licensed to the Apache Software Foundation (ASF) under one or more
-rem contributor license agreements. See the NOTICE file distributed with
-rem this work for additional information regarding copyright ownership.
-rem The ASF licenses this file to You under the Apache License, Version 2.0
-rem (the "License"); you may not use this file except in compliance with
-rem the License. You may obtain a copy of the License at
-rem
-rem http://www.apache.org/licenses/LICENSE-2.0
-rem
-rem Unless required by applicable law or agreed to in writing, software
-rem distributed under the License is distributed on an "AS IS" BASIS,
-rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-rem See the License for the specific language governing permissions and
-rem limitations under the License.
-
-IF [%1] EQU [] (
- echo USAGE: %0 zookeeper_host:port[/path] [-zk-tls-config-file file] [args...]
- EXIT /B 1
-)
-
-"%~dp0kafka-run-class.bat" org.apache.zookeeper.ZooKeeperMainWithTlsSupportForKafka -server %*
diff --git a/bin/zookeeper-server-stop.sh b/bin/zookeeper-server-stop.sh
deleted file mode 100755
index 11665f32707f8..0000000000000
--- a/bin/zookeeper-server-stop.sh
+++ /dev/null
@@ -1,35 +0,0 @@
-#!/bin/bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-SIGNAL=${SIGNAL:-TERM}
-
-OSNAME=$(uname -s)
-if [[ "$OSNAME" == "OS/390" ]]; then
- if [ -z $JOBNAME ]; then
- JOBNAME="ZKEESTRT"
- fi
- PIDS=$(ps -A -o pid,jobname,comm | grep -i $JOBNAME | grep java | grep -v grep | awk '{print $1}')
-elif [[ "$OSNAME" == "OS400" ]]; then
- PIDS=$(ps -Af | grep java | grep -i QuorumPeerMain | grep -v grep | awk '{print $2}')
-else
- PIDS=$(ps ax | grep java | grep -i QuorumPeerMain | grep -v grep | awk '{print $1}')
-fi
-
-if [ -z "$PIDS" ]; then
- echo "No zookeeper server to stop"
- exit 1
-else
- kill -s $SIGNAL $PIDS
-fi
diff --git a/bin/zookeeper-shell.sh b/bin/zookeeper-shell.sh
deleted file mode 100755
index 2f1d0f2c61670..0000000000000
--- a/bin/zookeeper-shell.sh
+++ /dev/null
@@ -1,23 +0,0 @@
-#!/bin/bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-if [ $# -lt 1 ];
-then
- echo "USAGE: $0 zookeeper_host:port[/path] [-zk-tls-config-file file] [args...]"
- exit 1
-fi
-
-exec $(dirname $0)/kafka-run-class.sh org.apache.zookeeper.ZooKeeperMainWithTlsSupportForKafka -server "$@"
diff --git a/build.gradle b/build.gradle
index 20eeb3eff4714..25340f0a9eed5 100644
--- a/build.gradle
+++ b/build.gradle
@@ -47,7 +47,10 @@ plugins {
ext {
gradleVersion = versions.gradle
- minJavaVersion = 11
+ minClientJavaVersion = 11
+ minNonClientJavaVersion = 17
+ modulesNeedingJava11 = [":clients", ":generator", ":streams", ":streams:test-utils", ":streams-scala", ":test-common:test-common-runtime"]
+
buildVersionFileName = "kafka-version.properties"
defaultMaxHeapSize = "2g"
@@ -113,28 +116,48 @@ ext {
commitId = determineCommitId()
- configureJavaCompiler = { name, options ->
+ configureJavaCompiler = { name, options, projectPath ->
// -parameters generates arguments with parameter names in TestInfo#getDisplayName.
// ref: https://github.com/junit-team/junit5/blob/4c0dddad1b96d4a20e92a2cd583954643ac56ac0/junit-jupiter-params/src/main/java/org/junit/jupiter/params/ParameterizedTest.java#L161-L164
- if (name == "compileTestJava" || name == "compileTestScala") {
+
+ def releaseVersion = modulesNeedingJava11.any { projectPath == it } ? minClientJavaVersion : minNonClientJavaVersion
+
+ options.compilerArgs << "-encoding" << "UTF-8"
+ options.release = releaseVersion
+
+ if (name in ["compileTestJava", "compileTestScala"]) {
options.compilerArgs << "-parameters"
- options.compilerArgs += ["--release", String.valueOf(minJavaVersion)]
- } else if (name == "compileJava" || name == "compileScala") {
- options.compilerArgs << "-Xlint:all"
- if (!project.path.startsWith(":connect") && !project.path.startsWith(":storage"))
- options.compilerArgs << "-Xlint:-rawtypes"
- options.compilerArgs << "-encoding" << "UTF-8"
+ } else if (name in ["compileJava", "compileScala"]) {
options.compilerArgs << "-Xlint:-rawtypes"
+ options.compilerArgs << "-Xlint:all"
options.compilerArgs << "-Xlint:-serial"
options.compilerArgs << "-Xlint:-try"
options.compilerArgs << "-Werror"
- options.compilerArgs += ["--release", String.valueOf(minJavaVersion)]
}
}
runtimeTestLibs = [
- libs.slf4jReload4j,
+ libs.slf4jLog4j2,
libs.junitPlatformLanucher,
+ project(":test-common:test-common-runtime")
+ ]
+
+ log4jRuntimeLibs = [
+ libs.slf4jLog4j2,
+ libs.log4j1Bridge2Api,
+ libs.jacksonDatabindYaml
+ ]
+
+ log4j2Libs = [
+ libs.log4j2Api,
+ libs.log4j2Core
+ ]
+
+ testLog4j2Libs = [
+ libs.slf4jApi,
+ libs.slf4jLog4j2,
+ libs.log4j2Api,
+ libs.log4j2Core
]
}
@@ -171,13 +194,7 @@ allprojects {
// ensure we have a single version in the classpath despite transitive dependencies
libs.scalaLibrary,
libs.scalaReflect,
- libs.jacksonAnnotations,
- // be explicit about the Netty dependency version instead of relying on the version set by
- // ZooKeeper (potentially older and containing CVEs)
- libs.nettyHandler,
- libs.nettyTransportNativeEpoll,
- // be explicit about the reload4j version instead of relying on the transitive versions
- libs.reload4j
+ libs.jacksonAnnotations
)
}
}
@@ -196,6 +213,11 @@ allprojects {
options.links "https://docs.oracle.com/en/java/javase/${JavaVersion.current().majorVersion}/docs/api/"
}
+ tasks.withType(Checkstyle) {
+ minHeapSize = "200m"
+ maxHeapSize = "1g"
+ }
+
clean {
delete "${projectDir}/src/generated"
delete "${projectDir}/src/generated-test"
@@ -224,7 +246,7 @@ static def projectToJUnitXmlPath(project) {
projectNames.push(p.name)
p = p.parent
if (p.name == "kafka") {
- break;
+ break
}
}
return projectNames.join("/")
@@ -320,7 +342,7 @@ subprojects {
}
tasks.withType(JavaCompile) {
- configureJavaCompiler(name, options)
+ configureJavaCompiler(name, options, project.path)
}
if (shouldPublish) {
@@ -483,6 +505,8 @@ subprojects {
// KAFKA-17433 Used by deflake.yml github action to repeat individual tests
systemProperty("kafka.cluster.test.repeat", project.findProperty("kafka.cluster.test.repeat"))
+ systemProperty("kafka.test.catalog.file", project.findProperty("kafka.test.catalog.file"))
+ systemProperty("kafka.test.run.quarantined", "false")
testLogging {
events = userTestLoggingEvents ?: testLoggingEvents
@@ -537,6 +561,7 @@ subprojects {
task quarantinedTest(type: Test, dependsOn: compileJava) {
ext {
isGithubActions = System.getenv('GITHUB_ACTIONS') != null
+ hadFailure = false // Used to track if any tests failed, see afterSuite below
}
// Disable caching and up-to-date for this task. We always want quarantined tests
@@ -546,13 +571,15 @@ subprojects {
outputs.cacheIf { false }
maxParallelForks = maxTestForks
- ignoreFailures = userIgnoreFailures
+ ignoreFailures = userIgnoreFailures || ext.isGithubActions
maxHeapSize = defaultMaxHeapSize
jvmArgs = defaultJvmArgs
// KAFKA-17433 Used by deflake.yml github action to repeat individual tests
systemProperty("kafka.cluster.test.repeat", project.findProperty("kafka.cluster.test.repeat"))
+ systemProperty("kafka.test.catalog.file", project.findProperty("kafka.test.catalog.file"))
+ systemProperty("kafka.test.run.quarantined", "true")
testLogging {
events = userTestLoggingEvents ?: testLoggingEvents
@@ -564,7 +591,6 @@ subprojects {
useJUnitPlatform {
includeEngines 'junit-jupiter'
- includeTags 'flaky'
}
develocity {
@@ -574,6 +600,13 @@ subprojects {
}
}
+ // As we process results, check if there were any test failures.
+ afterSuite { desc, result ->
+ if (result.resultType == TestResult.ResultType.FAILURE) {
+ ext.hadFailure = true
+ }
+ }
+
// This closure will copy JUnit XML files out of the sub-project's build directory and into
// a top-level build/junit-xml directory. This is necessary to avoid reporting on tests which
// were not run, but instead were restored via FROM-CACHE. See KAFKA-17479 for more details.
@@ -587,6 +620,11 @@ subprojects {
ant.include(name: "**/*.xml")
}
}
+ // If there were any test failures, we want to fail the task to prevent the failures
+ // from being cached.
+ if (ext.hadFailure) {
+ throw new GradleException("Failing this task since '${project.name}:${name}' had test failures.")
+ }
}
}
}
@@ -726,7 +764,7 @@ subprojects {
}
tasks.withType(ScalaCompile) {
-
+ def releaseVersion = modulesNeedingJava11.any { project.path == it } ? minClientJavaVersion : minNonClientJavaVersion
scalaCompileOptions.keepAliveMode = userKeepAliveMode
scalaCompileOptions.additionalParameters = [
@@ -770,10 +808,14 @@ subprojects {
scalaCompileOptions.additionalParameters += ["-opt-warnings", "-Xlint:strict-unsealed-patmat"]
// Scala 2.13.2 introduces compiler warnings suppression, which is a pre-requisite for -Xfatal-warnings
scalaCompileOptions.additionalParameters += ["-Xfatal-warnings"]
+ scalaCompileOptions.additionalParameters += ["--release", String.valueOf(releaseVersion)]
- scalaCompileOptions.additionalParameters += ["-release", String.valueOf(minJavaVersion)]
+ // Gradle does not support the `release` configuration when performing joint Java-Scala compilation.
+ // For more details, refer to https://github.com/gradle/gradle/issues/13762.
+ // As a result, we need to explicitly configure the Scala compiler with this setting.
+ options.compilerArgs += ["--release", String.valueOf(releaseVersion)]
- configureJavaCompiler(name, options)
+ configureJavaCompiler(name, options, project.path)
configure(scalaCompileOptions.forkOptions) {
memoryMaximumSize = defaultMaxHeapSize
@@ -932,24 +974,22 @@ project(':server') {
implementation project(':group-coordinator')
implementation project(':transaction-coordinator')
implementation project(':raft')
- implementation libs.metrics
+ implementation project(':share-coordinator')
implementation libs.jacksonDatabind
-
+ implementation libs.metrics
implementation libs.slf4jApi
- compileOnly libs.reload4j
-
testImplementation project(':clients').sourceSets.test.output
testImplementation libs.mockitoCore
testImplementation libs.junitJupiter
- testImplementation libs.slf4jReload4j
+ testImplementation testLog4j2Libs
- testRuntimeOnly libs.junitPlatformLanucher
+ testRuntimeOnly runtimeTestLibs
}
task createVersionFile() {
- def receiptFile = file("$buildDir/kafka/$buildVersionFileName")
+ def receiptFile = file("${layout.buildDirectory.get().asFile.path}/kafka/$buildVersionFileName")
inputs.property "commitId", commitId
inputs.property "version", version
outputs.file receiptFile
@@ -969,13 +1009,13 @@ project(':server') {
jar {
dependsOn createVersionFile
- from("$buildDir") {
+ from("${layout.buildDirectory.get().asFile.path}") {
include "kafka/$buildVersionFileName"
}
}
clean.doFirst {
- delete "$buildDir/kafka/"
+ delete "${layout.buildDirectory.get().asFile.path}/kafka/"
}
checkstyle {
@@ -987,35 +1027,6 @@ project(':server') {
}
}
-project(':share') {
- base {
- archivesName = "kafka-share"
- }
-
- dependencies {
- implementation project(':server-common')
-
- testImplementation project(':clients').sourceSets.test.output
- testImplementation project(':server-common').sourceSets.test.output
-
- implementation libs.slf4jApi
-
- testImplementation libs.junitJupiter
- testImplementation libs.mockitoCore
- testImplementation libs.slf4jReload4j
-
- testRuntimeOnly libs.junitPlatformLanucher
- }
-
- checkstyle {
- configProperties = checkstyleConfigProperties("import-control-share.xml")
- }
-
- javadoc {
- enabled = false
- }
-}
-
project(':core') {
apply plugin: 'scala'
@@ -1033,12 +1044,29 @@ project(':core') {
archivesName = "kafka_${versions.baseScala}"
}
+ configurations {
+ // manually excludes some unnecessary dependencies
+ implementation.exclude module: 'javax'
+ implementation.exclude module: 'jline'
+ implementation.exclude module: 'jms'
+ implementation.exclude module: 'jmxri'
+ implementation.exclude module: 'jmxtools'
+ implementation.exclude module: 'mail'
+ // To prevent a UniqueResourceException due the same resource existing in both
+ // org.apache.directory.api/api-all and org.apache.directory.api/api-ldap-schema-data
+ testImplementation.exclude module: 'api-ldap-schema-data'
+ releaseOnly
+ }
+
dependencies {
+ releaseOnly log4jRuntimeLibs
// `core` is often used in users' tests, define the following dependencies as `api` for backwards compatibility
// even though the `core` module doesn't expose any public API
api project(':clients')
api libs.scalaLibrary
+ compileOnly log4j2Libs
+
implementation project(':server-common')
implementation project(':group-coordinator:group-coordinator-api')
implementation project(':group-coordinator')
@@ -1050,7 +1078,6 @@ project(':core') {
implementation project(':storage')
implementation project(':server')
implementation project(':coordinator-common')
- implementation project(':share')
implementation project(':share-coordinator')
implementation libs.argparse4j
@@ -1066,22 +1093,6 @@ project(':core') {
implementation libs.scalaReflect
implementation libs.scalaLogging
implementation libs.slf4jApi
- implementation libs.commonsIo // ZooKeeper dependency. Do not use, this is going away.
- implementation(libs.zookeeper) {
- // Dropwizard Metrics are required by ZooKeeper as of v3.6.0,
- // but the library should *not* be used in Kafka code
- implementation libs.dropwizardMetrics
- exclude module: 'slf4j-log4j12'
- exclude module: 'log4j'
- // Both Kafka and Zookeeper use slf4j. ZooKeeper moved from log4j to logback in v3.8.0, but Kafka relies on reload4j.
- // We are removing Zookeeper's dependency on logback so we have a singular logging backend.
- exclude module: 'logback-classic'
- exclude module: 'logback-core'
- }
- // ZooKeeperMain depends on commons-cli but declares the dependency as `provided`
- implementation libs.commonsCli
-
- compileOnly libs.reload4j
testImplementation project(':clients').sourceSets.test.output
testImplementation project(':group-coordinator').sourceSets.test.output
@@ -1091,7 +1102,6 @@ project(':core') {
testImplementation project(':server-common').sourceSets.test.output
testImplementation project(':storage:storage-api').sourceSets.test.output
testImplementation project(':server').sourceSets.test.output
- testImplementation project(':share').sourceSets.test.output
testImplementation project(':test-common')
testImplementation project(':test-common:test-common-api')
testImplementation libs.bcpkix
@@ -1111,10 +1121,10 @@ project(':core') {
testImplementation libs.apachedsMavibotPartition
testImplementation libs.apachedsJdbmPartition
testImplementation libs.junitJupiter
- testImplementation libs.slf4jReload4j
testImplementation libs.caffeine
+ testImplementation testLog4j2Libs
- testRuntimeOnly libs.junitPlatformLanucher
+ testRuntimeOnly runtimeTestLibs
}
if (userEnableTestCoverage) {
@@ -1123,33 +1133,18 @@ project(':core') {
if (versions.baseScala == '2.13') {
scoverageScalaVersion = '2.13.9' // there's no newer 2.13 artifact, org.scoverage:scalac-scoverage-plugin_2.13.9:2.0.11 is the latest as of now
}
- reportDir = file("${rootProject.buildDir}/scoverage")
+ reportDir = file("${layout.buildDirectory.get().asFile.path}/scoverage")
highlighting = false
minimumRate = 0.0
}
}
- configurations {
- // manually excludes some unnecessary dependencies
- implementation.exclude module: 'javax'
- implementation.exclude module: 'jline'
- implementation.exclude module: 'jms'
- implementation.exclude module: 'jmxri'
- implementation.exclude module: 'jmxtools'
- implementation.exclude module: 'mail'
- // To prevent a UniqueResourceException due the same resource existing in both
- // org.apache.directory.api/api-all and org.apache.directory.api/api-ldap-schema-data
- testImplementation.exclude module: 'api-ldap-schema-data'
- }
-
tasks.create(name: "copyDependantLibs", type: Copy) {
- from (configurations.compileClasspath) {
- include('reload4j*jar')
- }
from (configurations.runtimeClasspath) {
exclude('kafka-clients*')
}
- into "$buildDir/dependant-libs-${versions.scala}"
+ from (configurations.releaseOnly)
+ into "${layout.buildDirectory.get().asFile.path}/dependant-libs-${versions.scala}"
duplicatesStrategy 'exclude'
}
@@ -1258,6 +1253,7 @@ project(':core') {
from "$rootDir/NOTICE-binary" rename {String filename -> filename.replace("-binary", "")}
from(configurations.runtimeClasspath) { into("libs/") }
from(configurations.archives.artifacts.files) { into("libs/") }
+ from(configurations.releaseOnly) { into("libs/") }
from(project.siteDocsTar) { into("site-docs/") }
from(project(':tools').jar) { into("libs/") }
from(project(':tools').configurations.runtimeClasspath) { into("libs/") }
@@ -1308,10 +1304,10 @@ project(':core') {
from (configurations.testRuntimeClasspath) {
include('*.jar')
}
- into "$buildDir/dependant-testlibs"
+ into "${layout.buildDirectory.get().asFile.path}/dependant-testlibs"
//By default gradle does not handle test dependencies between the sub-projects
//This line is to include clients project test jar to dependant-testlibs
- from (project(':clients').testJar ) { "$buildDir/dependant-testlibs" }
+ from (project(':clients').testJar ) { "${layout.buildDirectory.get().asFile.path}/dependant-testlibs" }
duplicatesStrategy 'exclude'
}
@@ -1358,16 +1354,17 @@ project(':metadata') {
implementation libs.jacksonDatabind
implementation libs.jacksonJDK8Datatypes
implementation libs.metrics
- compileOnly libs.reload4j
+ implementation libs.slf4jApi
+
+ testImplementation testLog4j2Libs
testImplementation libs.junitJupiter
testImplementation libs.jqwik
testImplementation libs.mockitoCore
- testImplementation libs.slf4jReload4j
testImplementation project(':clients').sourceSets.test.output
testImplementation project(':raft').sourceSets.test.output
testImplementation project(':server-common').sourceSets.test.output
- testRuntimeOnly libs.junitPlatformLanucher
+ testRuntimeOnly runtimeTestLibs
generator project(':generator')
}
@@ -1423,7 +1420,7 @@ project(':group-coordinator:group-coordinator-api') {
}
task createVersionFile() {
- def receiptFile = file("$buildDir/kafka/$buildVersionFileName")
+ def receiptFile = file("${layout.buildDirectory.get().asFile.path}/kafka/$buildVersionFileName")
inputs.property "commitId", commitId
inputs.property "version", version
outputs.file receiptFile
@@ -1443,13 +1440,13 @@ project(':group-coordinator:group-coordinator-api') {
jar {
dependsOn createVersionFile
- from("$buildDir") {
+ from("${layout.buildDirectory.get().asFile.path}") {
include "kafka/$buildVersionFileName"
}
}
clean.doFirst {
- delete "$buildDir/kafka/"
+ delete "${layout.buildDirectory.get().asFile.path}/kafka/"
}
javadoc {
@@ -1479,16 +1476,18 @@ project(':group-coordinator') {
implementation project(':coordinator-common')
implementation libs.jacksonDatabind
implementation libs.jacksonJDK8Datatypes
- implementation libs.slf4jApi
implementation libs.metrics
implementation libs.hdrHistogram
implementation libs.re2j
+ implementation libs.slf4jApi
testImplementation project(':clients').sourceSets.test.output
testImplementation project(':server-common').sourceSets.test.output
testImplementation project(':coordinator-common').sourceSets.test.output
+ testImplementation libs.jacksonDatabindYaml
testImplementation libs.junitJupiter
testImplementation libs.mockitoCore
+ testImplementation testLog4j2Libs
testRuntimeOnly runtimeTestLibs
@@ -1522,7 +1521,8 @@ project(':group-coordinator') {
args = [ "-p", "org.apache.kafka.coordinator.group.generated",
"-o", "${projectDir}/build/generated/main/java/org/apache/kafka/coordinator/group/generated",
"-i", "src/main/resources/common/message",
- "-m", "MessageDataGenerator", "JsonConverterGenerator"
+ "-m", "MessageDataGenerator", "JsonConverterGenerator",
+ "-t", "CoordinatorRecordTypeGenerator", "CoordinatorRecordJsonConvertersGenerator"
]
inputs.dir("src/main/resources/common/message")
.withPropertyName("messages")
@@ -1536,6 +1536,7 @@ project(':group-coordinator') {
}
project(':test-common') {
+ // Test framework stuff. Implementations that support test-common-api
base {
archivesName = "kafka-test-common"
}
@@ -1547,10 +1548,13 @@ project(':test-common') {
implementation project(':raft')
implementation project(':storage')
implementation project(':server-common')
+ implementation libs.jacksonDatabindYaml
implementation libs.slf4jApi
+
testImplementation libs.junitJupiter
testImplementation libs.mockitoCore
-
+ testImplementation testLog4j2Libs
+
testRuntimeOnly runtimeTestLibs
}
@@ -1564,11 +1568,11 @@ project(':test-common') {
}
project(':test-common:test-common-api') {
+ // Interfaces, config classes, and other test APIs
base {
archivesName = "kafka-test-common-api"
}
-
dependencies {
implementation project(':clients')
implementation project(':core')
@@ -1583,6 +1587,7 @@ project(':test-common:test-common-api') {
testImplementation libs.junitJupiter
testImplementation libs.mockitoCore
+ testImplementation testLog4j2Libs
testRuntimeOnly runtimeTestLibs
}
@@ -1596,6 +1601,29 @@ project(':test-common:test-common-api') {
}
}
+project(':test-common:test-common-runtime') {
+ // Runtime-only test code including JUnit extentions
+ base {
+ archivesName = "kafka-test-common-runtime"
+ }
+
+ dependencies {
+ implementation libs.junitPlatformLanucher
+ implementation libs.junitJupiterApi
+ implementation libs.junitJupiter
+ implementation libs.slf4jApi
+ testImplementation testLog4j2Libs
+ }
+
+ checkstyle {
+ configProperties = checkstyleConfigProperties("import-control-test-common-api.xml")
+ }
+
+ javadoc {
+ enabled = false
+ }
+}
+
project(':transaction-coordinator') {
base {
archivesName = "kafka-transaction-coordinator"
@@ -1609,13 +1637,16 @@ project(':transaction-coordinator') {
implementation libs.jacksonDatabind
implementation project(':clients')
implementation project(':server-common')
+ implementation project(':coordinator-common')
implementation libs.slf4jApi
+ testImplementation testLog4j2Libs
testImplementation libs.junitJupiter
testImplementation libs.mockitoCore
testImplementation project(':clients').sourceSets.test.output
testImplementation project(':test-common')
testImplementation project(':test-common:test-common-api')
+
testRuntimeOnly runtimeTestLibs
generator project(':generator')
@@ -1679,6 +1710,7 @@ project(':coordinator-common') {
testImplementation project(':server-common').sourceSets.test.output
testImplementation libs.junitJupiter
testImplementation libs.mockitoCore
+ testImplementation testLog4j2Libs
testRuntimeOnly runtimeTestLibs
}
@@ -1706,17 +1738,16 @@ project(':share-coordinator') {
implementation project(':clients')
implementation project(':coordinator-common')
implementation project(':metadata')
- implementation project(':server')
implementation project(':server-common')
- implementation project(':share')
- implementation libs.slf4jApi
implementation libs.metrics
+ implementation libs.slf4jApi
testImplementation project(':clients').sourceSets.test.output
testImplementation project(':server-common').sourceSets.test.output
testImplementation project(':coordinator-common').sourceSets.test.output
testImplementation libs.junitJupiter
testImplementation libs.mockitoCore
+ testImplementation testLog4j2Libs
testRuntimeOnly runtimeTestLibs
@@ -1746,7 +1777,8 @@ project(':share-coordinator') {
args = [ "-p", "org.apache.kafka.coordinator.share.generated",
"-o", "${projectDir}/build/generated/main/java/org/apache/kafka/coordinator/share/generated",
"-i", "src/main/resources/common/message",
- "-m", "MessageDataGenerator", "JsonConverterGenerator"
+ "-m", "MessageDataGenerator", "JsonConverterGenerator",
+ "-t", "CoordinatorRecordTypeGenerator", "CoordinatorRecordJsonConvertersGenerator"
]
inputs.dir("src/main/resources/common/message")
.withPropertyName("messages")
@@ -1786,10 +1818,17 @@ project(':generator') {
implementation libs.argparse4j
implementation libs.jacksonDatabind
implementation libs.jacksonJDK8Datatypes
- implementation libs.jacksonJaxrsJsonProvider
+ implementation libs.jacksonJakartarsJsonProvider
+
+ implementation 'org.eclipse.jgit:org.eclipse.jgit:6.4.0.202211300538-r'
+ // SSH support for JGit based on Apache MINA sshd
+ implementation 'org.eclipse.jgit:org.eclipse.jgit.ssh.apache:6.4.0.202211300538-r'
+ // GPG support for JGit based on BouncyCastle (commit signing)
+ implementation 'org.eclipse.jgit:org.eclipse.jgit.gpg.bc:6.4.0.202211300538-r'
+
testImplementation libs.junitJupiter
- testRuntimeOnly libs.junitPlatformLanucher
+ testRuntimeOnly runtimeTestLibs
}
javadoc {
@@ -1811,9 +1850,9 @@ project(':clients') {
implementation libs.zstd
implementation libs.lz4
implementation libs.snappy
- implementation libs.slf4jApi
implementation libs.opentelemetryProto
implementation libs.protobuf
+ implementation libs.slf4jApi
// libraries which should be added as runtime dependencies in generated pom.xml should be defined here:
shadowed libs.zstd
@@ -1825,23 +1864,28 @@ project(':clients') {
compileOnly libs.jacksonJDK8Datatypes
compileOnly libs.jose4j // for SASL/OAUTHBEARER JWT validation; only used by broker
+
testImplementation libs.bcpkix
- testImplementation libs.jacksonJaxrsJsonProvider
+ testImplementation libs.jacksonJakartarsJsonProvider
testImplementation libs.jose4j
testImplementation libs.junitJupiter
- testImplementation libs.reload4j
+ testImplementation libs.spotbugs
testImplementation libs.mockitoCore
testImplementation libs.mockitoJunitJupiter // supports MockitoExtension
+ testImplementation testLog4j2Libs
+
+ testCompileOnly libs.bndlib
testRuntimeOnly libs.jacksonDatabind
testRuntimeOnly libs.jacksonJDK8Datatypes
testRuntimeOnly runtimeTestLibs
+ testRuntimeOnly log4jRuntimeLibs
generator project(':generator')
}
task createVersionFile() {
- def receiptFile = file("$buildDir/kafka/$buildVersionFileName")
+ def receiptFile = file("${layout.buildDirectory.get().asFile.path}/kafka/$buildVersionFileName")
inputs.property "commitId", commitId
inputs.property "version", version
outputs.file receiptFile
@@ -1881,7 +1925,7 @@ project(':clients') {
exclude "**/google/protobuf/*.proto"
}
- from("$buildDir") {
+ from("${layout.buildDirectory.get().asFile.path}") {
include "kafka/$buildVersionFileName"
}
@@ -1895,7 +1939,7 @@ project(':clients') {
}
clean.doFirst {
- delete "$buildDir/kafka/"
+ delete "${layout.buildDirectory.get().asFile.path}/kafka/"
}
task processMessages(type:JavaExec) {
@@ -1988,16 +2032,18 @@ project(':raft') {
dependencies {
implementation project(':server-common')
implementation project(':clients')
- implementation libs.slf4jApi
implementation libs.jacksonDatabind
+ implementation libs.slf4jApi
testImplementation project(':server-common')
testImplementation project(':server-common').sourceSets.test.output
testImplementation project(':clients')
testImplementation project(':clients').sourceSets.test.output
+ testImplementation libs.jacksonDatabindYaml
testImplementation libs.junitJupiter
testImplementation libs.mockitoCore
testImplementation libs.jqwik
+ testImplementation testLog4j2Libs
testRuntimeOnly runtimeTestLibs
@@ -2005,7 +2051,7 @@ project(':raft') {
}
task createVersionFile() {
- def receiptFile = file("$buildDir/kafka/$buildVersionFileName")
+ def receiptFile = file("${layout.buildDirectory.get().asFile.path}/kafka/$buildVersionFileName")
inputs.property "commitId", commitId
inputs.property "version", version
outputs.file receiptFile
@@ -2055,7 +2101,7 @@ project(':raft') {
jar {
dependsOn createVersionFile
- from("$buildDir") {
+ from("${layout.buildDirectory.get().asFile.path}") {
include "kafka/$buildVersionFileName"
}
}
@@ -2067,7 +2113,7 @@ project(':raft') {
}
clean.doFirst {
- delete "$buildDir/kafka/"
+ delete "${layout.buildDirectory.get().asFile.path}/kafka/"
}
javadoc {
@@ -2082,23 +2128,24 @@ project(':server-common') {
dependencies {
api project(':clients')
- implementation libs.slf4jApi
implementation libs.metrics
implementation libs.joptSimple
implementation libs.jacksonDatabind
implementation libs.pcollections
+ implementation libs.slf4jApi
testImplementation project(':clients')
testImplementation project(':clients').sourceSets.test.output
+ testImplementation libs.jacksonDatabindYaml
testImplementation libs.junitJupiter
testImplementation libs.mockitoCore
+ testImplementation testLog4j2Libs
testRuntimeOnly runtimeTestLibs
- testRuntimeOnly project(":test-common")
}
task createVersionFile() {
- def receiptFile = file("$buildDir/kafka/$buildVersionFileName")
+ def receiptFile = file("${layout.buildDirectory.get().asFile.path}/kafka/$buildVersionFileName")
inputs.property "commitId", commitId
inputs.property "version", version
outputs.file receiptFile
@@ -2118,13 +2165,13 @@ project(':server-common') {
jar {
dependsOn createVersionFile
- from("$buildDir") {
+ from("${layout.buildDirectory.get().asFile.path}") {
include "kafka/$buildVersionFileName"
}
}
clean.doFirst {
- delete "$buildDir/kafka/"
+ delete "${layout.buildDirectory.get().asFile.path}/kafka/"
}
checkstyle {
@@ -2151,12 +2198,13 @@ project(':storage:storage-api') {
testImplementation project(':clients').sourceSets.test.output
testImplementation libs.junitJupiter
testImplementation libs.mockitoCore
+ testImplementation testLog4j2Libs
testRuntimeOnly runtimeTestLibs
}
task createVersionFile() {
- def receiptFile = file("$buildDir/kafka/$buildVersionFileName")
+ def receiptFile = file("${layout.buildDirectory.get().asFile.path}/kafka/$buildVersionFileName")
inputs.property "commitId", commitId
inputs.property "version", version
outputs.file receiptFile
@@ -2176,13 +2224,13 @@ project(':storage:storage-api') {
jar {
dependsOn createVersionFile
- from("$buildDir") {
+ from("${layout.buildDirectory.get().asFile.path}") {
include "kafka/$buildVersionFileName"
}
}
clean.doFirst {
- delete "$buildDir/kafka/"
+ delete "${layout.buildDirectory.get().asFile.path}/kafka/"
}
javadoc {
@@ -2207,7 +2255,6 @@ project(':storage') {
implementation project(':storage:storage-api')
implementation project(':server-common')
implementation project(':clients')
- implementation project(':transaction-coordinator')
implementation(libs.caffeine) {
exclude group: 'org.checkerframework', module: 'checker-qual'
}
@@ -2224,9 +2271,11 @@ project(':storage') {
testImplementation project(':server-common')
testImplementation project(':server-common').sourceSets.test.output
testImplementation libs.hamcrest
+ testImplementation libs.jacksonDatabindYaml
testImplementation libs.junitJupiter
testImplementation libs.mockitoCore
testImplementation libs.bcpkix
+ testImplementation testLog4j2Libs
testRuntimeOnly runtimeTestLibs
@@ -2234,7 +2283,7 @@ project(':storage') {
}
task createVersionFile() {
- def receiptFile = file("$buildDir/kafka/$buildVersionFileName")
+ def receiptFile = file("${layout.buildDirectory.get().asFile.path}/kafka/$buildVersionFileName")
inputs.property "commitId", commitId
inputs.property "version", version
outputs.file receiptFile
@@ -2299,13 +2348,13 @@ project(':storage') {
jar {
dependsOn createVersionFile
- from("$buildDir") {
+ from("${layout.buildDirectory.get().asFile.path}") {
include "kafka/$buildVersionFileName"
}
}
clean.doFirst {
- delete "$buildDir/kafka/"
+ delete "${layout.buildDirectory.get().asFile.path}/kafka/"
}
javadoc {
@@ -2325,11 +2374,12 @@ project(':tools:tools-api') {
dependencies {
implementation project(':clients')
testImplementation libs.junitJupiter
- testRuntimeOnly libs.junitPlatformLanucher
+ testImplementation testLog4j2Libs
+ testRuntimeOnly runtimeTestLibs
}
task createVersionFile() {
- def receiptFile = file("$buildDir/kafka/$buildVersionFileName")
+ def receiptFile = file("${layout.buildDirectory.get().asFile.path}/kafka/$buildVersionFileName")
inputs.property "commitId", commitId
inputs.property "version", version
outputs.file receiptFile
@@ -2349,13 +2399,13 @@ project(':tools:tools-api') {
jar {
dependsOn createVersionFile
- from("$buildDir") {
+ from("${layout.buildDirectory.get().asFile.path}") {
include "kafka/$buildVersionFileName"
}
}
clean.doFirst {
- delete "$buildDir/kafka/"
+ delete "${layout.buildDirectory.get().asFile.path}/kafka/"
}
javadoc {
@@ -2368,7 +2418,13 @@ project(':tools') {
archivesName = "kafka-tools"
}
+ configurations {
+ releaseOnly
+ }
+
dependencies {
+ releaseOnly log4jRuntimeLibs
+
implementation project(':clients')
implementation project(':metadata')
implementation project(':storage')
@@ -2380,17 +2436,18 @@ project(':tools') {
implementation project(':group-coordinator')
implementation project(':coordinator-common')
implementation project(':share-coordinator')
- implementation project(':share')
implementation libs.argparse4j
implementation libs.jacksonDatabind
implementation libs.jacksonDataformatCsv
implementation libs.jacksonJDK8Datatypes
- implementation libs.slf4jApi
- implementation libs.slf4jReload4j
implementation libs.joptSimple
+ implementation libs.slf4jApi
+ implementation libs.re2j
implementation libs.jose4j // for SASL/OAUTHBEARER JWT validation
- implementation libs.jacksonJaxrsJsonProvider
+ implementation libs.jacksonJakartarsJsonProvider
+
+ compileOnly libs.spotbugs
testImplementation project(':clients')
testImplementation project(':clients').sourceSets.test.output
@@ -2417,17 +2474,16 @@ project(':tools') {
testImplementation(libs.jfreechart) {
exclude group: 'junit', module: 'junit'
}
- testImplementation libs.reload4j
testImplementation libs.apachedsCoreApi
testImplementation libs.apachedsInterceptorKerberos
testImplementation libs.apachedsProtocolShared
testImplementation libs.apachedsProtocolKerberos
testImplementation libs.apachedsProtocolLdap
testImplementation libs.apachedsLdifPartition
+ testImplementation testLog4j2Libs
- testRuntimeOnly libs.junitPlatformLanucher
+ testRuntimeOnly runtimeTestLibs
testRuntimeOnly libs.hamcrest
- testRuntimeOnly project(':test-common')
}
javadoc {
@@ -2438,7 +2494,8 @@ project(':tools') {
from (configurations.runtimeClasspath) {
exclude('kafka-clients*')
}
- into "$buildDir/dependant-libs-${versions.scala}"
+ from (configurations.releaseOnly)
+ into "${layout.buildDirectory.get().asFile.path}/dependant-libs-${versions.scala}"
duplicatesStrategy 'exclude'
}
@@ -2458,23 +2515,34 @@ project(':trogdor') {
implementation libs.jacksonDatabind
implementation libs.jacksonJDK8Datatypes
implementation libs.slf4jApi
- runtimeOnly libs.reload4j
- implementation libs.jacksonJaxrsJsonProvider
+ implementation libs.jacksonJakartarsJsonProvider
implementation libs.jerseyContainerServlet
implementation libs.jerseyHk2
implementation libs.jaxbApi // Jersey dependency that was available in the JDK before Java 9
implementation libs.activation // Jersey dependency that was available in the JDK before Java 9
- implementation libs.jettyServer
- implementation libs.jettyServlet
- implementation libs.jettyServlets
+ implementation (libs.jettyServer) {
+ exclude group: 'org.slf4j', module: 'slf4j-api'
+ }
+ implementation (libs.jettyServlet) {
+ exclude group: 'org.slf4j', module: 'slf4j-api'
+ }
+ implementation (libs.jettyServlets) {
+ exclude group: 'org.slf4j', module: 'slf4j-api'
+ }
+
+ implementation project(':group-coordinator')
+ implementation project(':group-coordinator:group-coordinator-api')
testImplementation project(':clients')
- testImplementation libs.junitJupiter
testImplementation project(':clients').sourceSets.test.output
+ testImplementation project(':group-coordinator')
+ testImplementation libs.junitJupiter
testImplementation libs.mockitoCore
+ testImplementation testLog4j2Libs
testRuntimeOnly runtimeTestLibs
+ testRuntimeOnly libs.junitPlatformLanucher
}
javadoc {
@@ -2485,7 +2553,7 @@ project(':trogdor') {
from (configurations.runtimeClasspath) {
exclude('kafka-clients*')
}
- into "$buildDir/dependant-libs-${versions.scala}"
+ into "${layout.buildDirectory.get().asFile.path}/dependant-libs-${versions.scala}"
duplicatesStrategy 'exclude'
}
@@ -2512,14 +2580,16 @@ project(':shell') {
implementation project(':raft')
implementation libs.jose4j // for SASL/OAUTHBEARER JWT validation
- implementation libs.jacksonJaxrsJsonProvider
+ implementation libs.jacksonJakartarsJsonProvider
testImplementation project(':clients')
testImplementation project(':clients').sourceSets.test.output
testImplementation project(':core')
testImplementation project(':server-common')
testImplementation project(':server-common').sourceSets.test.output
+ testImplementation libs.jacksonDatabindYaml
testImplementation libs.junitJupiter
+ testImplementation testLog4j2Libs
testRuntimeOnly runtimeTestLibs
}
@@ -2532,7 +2602,7 @@ project(':shell') {
from (configurations.runtimeClasspath) {
include('jline-*jar')
}
- into "$buildDir/dependant-libs-${versions.scala}"
+ into "${layout.buildDirectory.get().asFile.path}/dependant-libs-${versions.scala}"
duplicatesStrategy 'exclude'
}
@@ -2557,26 +2627,27 @@ project(':streams') {
// `org.rocksdb.Options` is part of Kafka Streams public api via `RocksDBConfigSetter`
api libs.rocksDBJni
- implementation libs.slf4jApi
implementation libs.jacksonAnnotations
implementation libs.jacksonDatabind
+ implementation libs.slf4jApi
// testCompileOnly prevents streams from exporting a dependency on test-utils, which would cause a dependency cycle
testCompileOnly project(':streams:test-utils')
+ testCompileOnly libs.bndlib
- testImplementation project(':metadata')
testImplementation project(':clients').sourceSets.test.output
- testImplementation libs.reload4j
+ testImplementation libs.jacksonDatabindYaml
testImplementation libs.junitJupiter
testImplementation libs.bcpkix
testImplementation libs.hamcrest
testImplementation libs.mockitoCore
testImplementation libs.mockitoJunitJupiter // supports MockitoExtension
testImplementation libs.junitPlatformSuiteEngine // supports suite test
- testImplementation project(':group-coordinator')
+ testImplementation testLog4j2Libs
testRuntimeOnly project(':streams:test-utils')
testRuntimeOnly runtimeTestLibs
+ testRuntimeOnly log4jRuntimeLibs
generator project(':generator')
}
@@ -2621,12 +2692,12 @@ project(':streams') {
from (configurations.runtimeClasspath) {
exclude('kafka-clients*')
}
- into "$buildDir/dependant-libs-${versions.scala}"
+ into "${layout.buildDirectory.get().asFile.path}/dependant-libs-${versions.scala}"
duplicatesStrategy 'exclude'
}
task createStreamsVersionFile() {
- def receiptFile = file("$buildDir/kafka/$buildStreamsVersionFileName")
+ def receiptFile = file("${layout.buildDirectory.get().asFile.path}/kafka/$buildStreamsVersionFileName")
inputs.property "commitId", commitId
inputs.property "version", version
outputs.file receiptFile
@@ -2645,7 +2716,7 @@ project(':streams') {
jar {
dependsOn 'createStreamsVersionFile'
- from("$buildDir") {
+ from("${layout.buildDirectory.get().asFile.path}") {
include "kafka/$buildStreamsVersionFileName"
}
dependsOn 'copyDependantLibs'
@@ -2665,12 +2736,9 @@ project(':streams') {
task testAll(
dependsOn: [
':streams:test',
- ':streams:integration-tests',
+ ':streams:integration-tests:test',
':streams:test-utils:test',
':streams:streams-scala:test',
- ':streams:upgrade-system-tests-0100:test',
- ':streams:upgrade-system-tests-0101:test',
- ':streams:upgrade-system-tests-0102:test',
':streams:upgrade-system-tests-0110:test',
':streams:upgrade-system-tests-10:test',
':streams:upgrade-system-tests-11:test',
@@ -2692,6 +2760,7 @@ project(':streams') {
':streams:upgrade-system-tests-36:test',
':streams:upgrade-system-tests-37:test',
':streams:upgrade-system-tests-38:test',
+ ':streams:upgrade-system-tests-39:test',
':streams:examples:test'
]
)
@@ -2712,8 +2781,11 @@ project(':streams:streams-scala') {
testImplementation project(':clients').sourceSets.test.output
testImplementation project(':streams:test-utils')
+ testImplementation libs.jacksonDatabindYaml
testImplementation libs.junitJupiter
testImplementation libs.mockitoJunitJupiter // supports MockitoExtension
+ testImplementation testLog4j2Libs
+
testRuntimeOnly runtimeTestLibs
}
@@ -2729,7 +2801,7 @@ project(':streams:streams-scala') {
from (configurations.runtimeClasspath) {
exclude('kafka-streams*')
}
- into "$buildDir/dependant-libs-${versions.scala}"
+ into "${layout.buildDirectory.get().asFile.path}/dependant-libs-${versions.scala}"
duplicatesStrategy 'exclude'
}
@@ -2755,6 +2827,9 @@ project(':streams:integration-tests') {
}
dependencies {
+ implementation libs.slf4jApi
+ implementation libs.scalaLibrary
+
testImplementation project(':clients').sourceSets.test.output
testImplementation project(':group-coordinator')
testImplementation project(':server')
@@ -2771,8 +2846,7 @@ project(':streams:integration-tests') {
testImplementation libs.junitJupiter
testImplementation libs.junitPlatformSuiteEngine // supports suite test
testImplementation libs.mockitoCore
- testImplementation libs.reload4j
- testImplementation libs.slf4jApi
+ testImplementation testLog4j2Libs
testImplementation project(':streams:test-utils')
testRuntimeOnly runtimeTestLibs
@@ -2811,9 +2885,11 @@ project(':streams:test-utils') {
implementation libs.slf4jApi
testImplementation project(':clients').sourceSets.test.output
+ testImplementation libs.jacksonDatabindYaml
testImplementation libs.junitJupiter
testImplementation libs.mockitoCore
testImplementation libs.hamcrest
+ testImplementation testLog4j2Libs
testRuntimeOnly runtimeTestLibs
}
@@ -2822,7 +2898,7 @@ project(':streams:test-utils') {
from (configurations.runtimeClasspath) {
exclude('kafka-streams*')
}
- into "$buildDir/dependant-libs-${versions.scala}"
+ into "${layout.buildDirectory.get().asFile.path}/dependant-libs-${versions.scala}"
duplicatesStrategy 'exclude'
}
@@ -2839,21 +2915,17 @@ project(':streams:examples') {
dependencies {
// this dependency should be removed after we unify data API
- implementation(project(':connect:json')) {
- // this transitive dependency is not used in Streams, and it breaks SBT builds
- exclude module: 'javax.ws.rs-api'
- }
-
+ implementation(project(':connect:json'))
implementation project(':streams')
-
- implementation libs.slf4jReload4j
+ implementation libs.slf4jApi
testImplementation project(':streams:test-utils')
testImplementation project(':clients').sourceSets.test.output // for org.apache.kafka.test.IntegrationTest
testImplementation libs.junitJupiter
testImplementation libs.hamcrest
+ testImplementation testLog4j2Libs
- testRuntimeOnly libs.junitPlatformLanucher
+ testRuntimeOnly runtimeTestLibs
}
javadoc {
@@ -2864,7 +2936,7 @@ project(':streams:examples') {
from (configurations.runtimeClasspath) {
exclude('kafka-streams*')
}
- into "$buildDir/dependant-libs-${versions.scala}"
+ into "${layout.buildDirectory.get().asFile.path}/dependant-libs-${versions.scala}"
duplicatesStrategy 'exclude'
}
@@ -2873,57 +2945,6 @@ project(':streams:examples') {
}
}
-project(':streams:upgrade-system-tests-0100') {
- base {
- archivesName = "kafka-streams-upgrade-system-tests-0100"
- }
-
- dependencies {
- testImplementation(libs.kafkaStreams_0100) {
- exclude group: 'org.slf4j', module: 'slf4j-log4j12'
- exclude group: 'log4j', module: 'log4j'
- }
- testRuntimeOnly libs.junitJupiter
- }
-
- systemTestLibs {
- dependsOn testJar
- }
-}
-
-project(':streams:upgrade-system-tests-0101') {
- base {
- archivesName = "kafka-streams-upgrade-system-tests-0101"
- }
-
- dependencies {
- testImplementation(libs.kafkaStreams_0101) {
- exclude group: 'org.slf4j', module: 'slf4j-log4j12'
- exclude group: 'log4j', module: 'log4j'
- }
- testRuntimeOnly libs.junitJupiter
- }
-
- systemTestLibs {
- dependsOn testJar
- }
-}
-
-project(':streams:upgrade-system-tests-0102') {
- base {
- archivesName = "kafka-streams-upgrade-system-tests-0102"
- }
-
- dependencies {
- testImplementation libs.kafkaStreams_0102
- testRuntimeOnly libs.junitJupiter
- }
-
- systemTestLibs {
- dependsOn testJar
- }
-}
-
project(':streams:upgrade-system-tests-0110') {
base{
archivesName = "kafka-streams-upgrade-system-tests-0110"
@@ -3239,6 +3260,21 @@ project(':streams:upgrade-system-tests-38') {
}
}
+project(':streams:upgrade-system-tests-39') {
+ base {
+ archivesName = "kafka-streams-upgrade-system-tests-39"
+ }
+
+ dependencies {
+ testImplementation libs.kafkaStreams_39
+ testRuntimeOnly libs.junitJupiter
+ }
+
+ systemTestLibs {
+ dependsOn testJar
+ }
+}
+
project(':jmh-benchmarks') {
apply plugin: 'io.github.goooler.shadow'
@@ -3262,6 +3298,7 @@ project(':jmh-benchmarks') {
implementation project(':metadata')
implementation project(':storage')
implementation project(':streams')
+ implementation project(':transaction-coordinator')
implementation project(':core')
implementation project(':connect:api')
implementation project(':connect:transforms')
@@ -3276,8 +3313,8 @@ project(':jmh-benchmarks') {
implementation libs.jacksonDatabind
implementation libs.metrics
implementation libs.mockitoCore
- implementation libs.slf4jReload4j
implementation libs.scalaLibrary
+ implementation libs.slf4jApi
}
tasks.withType(JavaCompile) {
@@ -3319,12 +3356,12 @@ project(':connect:api') {
dependencies {
api project(':clients')
+ implementation libs.jakartaRsApi
implementation libs.slf4jApi
- runtimeOnly libs.reload4j
- implementation libs.jaxrsApi
testImplementation libs.junitJupiter
testImplementation project(':clients').sourceSets.test.output
+ testImplementation testLog4j2Libs
testRuntimeOnly runtimeTestLibs
}
@@ -3338,7 +3375,7 @@ project(':connect:api') {
exclude('kafka-clients*')
exclude('connect-*')
}
- into "$buildDir/dependant-libs"
+ into "${layout.buildDirectory.get().asFile.path}/dependant-libs"
duplicatesStrategy 'exclude'
}
@@ -3356,11 +3393,10 @@ project(':connect:transforms') {
api project(':connect:api')
implementation libs.slf4jApi
- runtimeOnly libs.reload4j
testImplementation libs.junitJupiter
-
testImplementation project(':clients').sourceSets.test.output
+ testImplementation testLog4j2Libs
testRuntimeOnly runtimeTestLibs
}
@@ -3374,7 +3410,7 @@ project(':connect:transforms') {
exclude('kafka-clients*')
exclude('connect-*')
}
- into "$buildDir/dependant-libs"
+ into "${layout.buildDirectory.get().asFile.path}/dependant-libs"
duplicatesStrategy 'exclude'
}
@@ -3393,13 +3429,12 @@ project(':connect:json') {
api libs.jacksonDatabind
api libs.jacksonJDK8Datatypes
- api libs.jacksonAfterburner
+ api libs.jacksonBlackbird
implementation libs.slf4jApi
- runtimeOnly libs.reload4j
testImplementation libs.junitJupiter
-
+ testImplementation testLog4j2Libs
testImplementation project(':clients').sourceSets.test.output
testRuntimeOnly runtimeTestLibs
@@ -3414,7 +3449,7 @@ project(':connect:json') {
exclude('kafka-clients*')
exclude('connect-*')
}
- into "$buildDir/dependant-libs"
+ into "${layout.buildDirectory.get().asFile.path}/dependant-libs"
duplicatesStrategy 'exclude'
}
@@ -3440,26 +3475,39 @@ project(':connect:runtime') {
api project(':connect:json')
api project(':connect:transforms')
+ compileOnly log4j2Libs
+
implementation libs.slf4jApi
- implementation libs.reload4j
implementation libs.jose4j // for SASL/OAUTHBEARER JWT validation
implementation libs.jacksonAnnotations
- implementation libs.jacksonJaxrsJsonProvider
+ implementation libs.jacksonJakartarsJsonProvider
implementation libs.jerseyContainerServlet
implementation libs.jerseyHk2
implementation libs.jaxbApi // Jersey dependency that was available in the JDK before Java 9
implementation libs.activation // Jersey dependency that was available in the JDK before Java 9
- implementation libs.jettyServer
- implementation libs.jettyServlet
- implementation libs.jettyServlets
- implementation libs.jettyClient
+ implementation (libs.jettyServer) {
+ exclude group: 'org.slf4j', module: 'slf4j-api'
+ }
+ implementation (libs.jettyServlet) {
+ exclude group: 'org.slf4j', module: 'slf4j-api'
+ }
+ implementation (libs.jettyServlets) {
+ exclude group: 'org.slf4j', module: 'slf4j-api'
+ }
+ implementation (libs.jettyClient) {
+ exclude group: 'org.slf4j', module: 'slf4j-api'
+ }
implementation libs.classgraph
implementation libs.mavenArtifact
implementation libs.swaggerAnnotations
+ compileOnly libs.bndlib
+ compileOnly libs.spotbugs
+
// We use this library to generate OpenAPI docs for the REST API, but we don't want or need it at compile
// or run time. So, we add it to a separate configuration, which we use later on during docs generation
- swagger libs.swaggerJaxrs2
+ swagger libs.jakartaServletApi
+ swagger libs.jaxrs2Jakarta
testImplementation project(':clients').sourceSets.test.output
testImplementation project(':core')
@@ -3475,10 +3523,14 @@ project(':connect:runtime') {
testImplementation project(':server-common').sourceSets.test.output
testImplementation project(':test-common:test-common-api')
+ testImplementation libs.jacksonDatabindYaml
testImplementation libs.junitJupiter
testImplementation libs.mockitoCore
testImplementation libs.mockitoJunitJupiter
testImplementation libs.httpclient
+ testImplementation testLog4j2Libs
+
+ testCompileOnly libs.bndlib
testRuntimeOnly libs.bcpkix
testRuntimeOnly runtimeTestLibs
@@ -3493,7 +3545,7 @@ project(':connect:runtime') {
exclude('kafka-clients*')
exclude('connect-*')
}
- into "$buildDir/dependant-libs"
+ into "${layout.buildDirectory.get().asFile.path}/dependant-libs"
duplicatesStrategy 'exclude'
}
@@ -3545,7 +3597,7 @@ project(':connect:runtime') {
task setVersionInOpenAPISpec(type: Copy) {
from "$rootDir/gradle/openapi.template"
- into "$buildDir/resources/docs"
+ into "${layout.buildDirectory.get().asFile.path}/resources/docs"
rename ('openapi.template', 'openapi.yaml')
expand(kafkaVersion: "$rootProject.version")
}
@@ -3558,7 +3610,7 @@ project(':connect:runtime') {
outputFormat = 'YAML'
prettyPrint = 'TRUE'
sortOutput = 'TRUE'
- openApiFile = file("$buildDir/resources/docs/openapi.yaml")
+ openApiFile = file("${layout.buildDirectory.get().asFile.path}/resources/docs/openapi.yaml")
resourcePackages = ['org.apache.kafka.connect.runtime.rest.resources']
if( !generatedDocsDir.exists() ) { generatedDocsDir.mkdirs() }
outputDir = file(generatedDocsDir)
@@ -3574,10 +3626,11 @@ project(':connect:file') {
dependencies {
implementation project(':connect:api')
implementation libs.slf4jApi
- runtimeOnly libs.reload4j
+ testImplementation testLog4j2Libs
testImplementation libs.junitJupiter
testImplementation libs.mockitoCore
+
testImplementation project(':clients').sourceSets.test.output
testImplementation project(':connect:runtime')
testImplementation project(':connect:runtime').sourceSets.test.output
@@ -3597,7 +3650,7 @@ project(':connect:file') {
exclude('kafka-clients*')
exclude('connect-*')
}
- into "$buildDir/dependant-libs"
+ into "${layout.buildDirectory.get().asFile.path}/dependant-libs"
duplicatesStrategy 'exclude'
}
@@ -3613,14 +3666,15 @@ project(':connect:basic-auth-extension') {
dependencies {
implementation project(':connect:api')
+
implementation libs.slf4jApi
- runtimeOnly libs.reload4j
- implementation libs.jaxrsApi
+ implementation libs.jakartaRsApi
implementation libs.jaxAnnotationApi
testImplementation libs.bcpkix
testImplementation libs.mockitoCore
testImplementation libs.junitJupiter
+ testImplementation testLog4j2Libs
testImplementation project(':clients').sourceSets.test.output
testRuntimeOnly libs.jerseyContainerServlet
@@ -3636,7 +3690,7 @@ project(':connect:basic-auth-extension') {
exclude('kafka-clients*')
exclude('connect-*')
}
- into "$buildDir/dependant-libs"
+ into "${layout.buildDirectory.get().asFile.path}/dependant-libs"
duplicatesStrategy 'exclude'
}
@@ -3657,23 +3711,30 @@ project(':connect:mirror') {
implementation project(':clients')
implementation libs.argparse4j
- implementation libs.jacksonAnnotations
implementation libs.slf4jApi
- runtimeOnly libs.reload4j
implementation libs.jacksonAnnotations
- implementation libs.jacksonJaxrsJsonProvider
+ implementation libs.jacksonJakartarsJsonProvider
implementation libs.jerseyContainerServlet
implementation libs.jerseyHk2
implementation libs.jaxbApi // Jersey dependency that was available in the JDK before Java 9
implementation libs.activation // Jersey dependency that was available in the JDK before Java 9
- implementation libs.jettyServer
- implementation libs.jettyServlet
- implementation libs.jettyServlets
- implementation libs.jettyClient
+ implementation (libs.jettyServer) {
+ exclude group: 'org.slf4j', module: 'slf4j-api'
+ }
+ implementation (libs.jettyServlet) {
+ exclude group: 'org.slf4j', module: 'slf4j-api'
+ }
+ implementation (libs.jettyServlets) {
+ exclude group: 'org.slf4j', module: 'slf4j-api'
+ }
+ implementation (libs.jettyClient) {
+ exclude group: 'org.slf4j', module: 'slf4j-api'
+ }
implementation libs.swaggerAnnotations
+ testImplementation testLog4j2Libs
testImplementation libs.junitJupiter
- testImplementation libs.reload4j
+ testImplementation libs.bndlib
testImplementation libs.mockitoCore
testImplementation project(':clients').sourceSets.test.output
testImplementation project(':connect:runtime').sourceSets.test.output
@@ -3682,6 +3743,7 @@ project(':connect:mirror') {
testImplementation project(':server')
testImplementation project(':server-common').sourceSets.test.output
+
testRuntimeOnly project(':connect:runtime')
testRuntimeOnly libs.bcpkix
testRuntimeOnly runtimeTestLibs
@@ -3696,7 +3758,7 @@ project(':connect:mirror') {
exclude('kafka-clients*')
exclude('connect-*')
}
- into "$buildDir/dependant-libs"
+ into "${layout.buildDirectory.get().asFile.path}/dependant-libs"
duplicatesStrategy 'exclude'
}
@@ -3741,8 +3803,8 @@ project(':connect:mirror-client') {
dependencies {
implementation project(':clients')
implementation libs.slf4jApi
- runtimeOnly libs.reload4j
+ testImplementation testLog4j2Libs
testImplementation libs.junitJupiter
testImplementation project(':clients').sourceSets.test.output
@@ -3758,7 +3820,7 @@ project(':connect:mirror-client') {
exclude('kafka-clients*')
exclude('connect-*')
}
- into "$buildDir/dependant-libs"
+ into "${layout.buildDirectory.get().asFile.path}/dependant-libs"
duplicatesStrategy 'exclude'
}
@@ -3778,6 +3840,8 @@ project(':connect:test-plugins') {
implementation project(':server-common')
implementation libs.slf4jApi
implementation libs.jacksonDatabind
+
+ testImplementation testLog4j2Libs
}
}
diff --git a/checkstyle/checkstyle.xml b/checkstyle/checkstyle.xml
index 6b8aedb6e4700..fa1d5873a2cbc 100644
--- a/checkstyle/checkstyle.xml
+++ b/checkstyle/checkstyle.xml
@@ -39,7 +39,9 @@
-
+
+
+
diff --git a/checkstyle/import-control-core.xml b/checkstyle/import-control-core.xml
index a8dc78160e3e1..3cfd0ce663cc1 100644
--- a/checkstyle/import-control-core.xml
+++ b/checkstyle/import-control-core.xml
@@ -114,7 +114,14 @@
-
+
+
+
+
+
+
+
+
@@ -136,7 +143,7 @@
-
+
@@ -146,6 +153,14 @@
+
+
+
+
+
+
+
+
diff --git a/checkstyle/import-control-jmh-benchmarks.xml b/checkstyle/import-control-jmh-benchmarks.xml
index 6840d786926d3..4469ccf3bbeb9 100644
--- a/checkstyle/import-control-jmh-benchmarks.xml
+++ b/checkstyle/import-control-jmh-benchmarks.xml
@@ -58,6 +58,7 @@
+
diff --git a/checkstyle/import-control-server-common.xml b/checkstyle/import-control-server-common.xml
index c576ea76f9a6c..ac8706686617c 100644
--- a/checkstyle/import-control-server-common.xml
+++ b/checkstyle/import-control-server-common.xml
@@ -98,6 +98,15 @@
+
+
+
+
+
+
+
+
+
diff --git a/checkstyle/import-control-share.xml b/checkstyle/import-control-share.xml
deleted file mode 100644
index 8abc1b9f7b779..0000000000000
--- a/checkstyle/import-control-share.xml
+++ /dev/null
@@ -1,62 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
diff --git a/checkstyle/import-control-storage.xml b/checkstyle/import-control-storage.xml
index 2792ad844ee77..fecc55f98ea11 100644
--- a/checkstyle/import-control-storage.xml
+++ b/checkstyle/import-control-storage.xml
@@ -44,6 +44,7 @@
+
diff --git a/checkstyle/import-control-test-common.xml b/checkstyle/import-control-test-common.xml
index 9fe7f4d4844da..9520c0b21b86e 100644
--- a/checkstyle/import-control-test-common.xml
+++ b/checkstyle/import-control-test-common.xml
@@ -24,4 +24,5 @@
+
diff --git a/checkstyle/import-control-transaction-coordinator.xml b/checkstyle/import-control-transaction-coordinator.xml
index 4a61407cfa988..f4045907257a3 100644
--- a/checkstyle/import-control-transaction-coordinator.xml
+++ b/checkstyle/import-control-transaction-coordinator.xml
@@ -34,6 +34,7 @@
+
diff --git a/checkstyle/import-control.xml b/checkstyle/import-control.xml
index 9465f016c8a6a..921db8162e33a 100644
--- a/checkstyle/import-control.xml
+++ b/checkstyle/import-control.xml
@@ -201,7 +201,7 @@
-
+
@@ -225,6 +225,7 @@
+
@@ -289,6 +290,8 @@
+
+
@@ -307,6 +310,10 @@
+
+
+
+
@@ -330,6 +337,7 @@
+
@@ -358,8 +366,8 @@
-
-
+
+
@@ -368,6 +376,7 @@
+
@@ -378,6 +387,7 @@
+
@@ -387,7 +397,7 @@
-
+
@@ -415,7 +425,6 @@
-
@@ -442,7 +451,6 @@
-
@@ -525,7 +533,7 @@
-
+
@@ -549,10 +557,11 @@
+
-
-
+
+
@@ -562,29 +571,30 @@
+
-
+
-
-
+
+
-
+
+
-
-
+
@@ -616,8 +626,8 @@
-
-
+
+
@@ -629,7 +639,7 @@
-
+
diff --git a/checkstyle/suppressions.xml b/checkstyle/suppressions.xml
index 819b53ddc4a6b..ebff3f9bc8e8a 100644
--- a/checkstyle/suppressions.xml
+++ b/checkstyle/suppressions.xml
@@ -47,6 +47,7 @@
+
@@ -57,7 +58,7 @@
+ files="(AbstractFetch|Sender|SenderTest|ConsumerCoordinator|KafkaConsumer|KafkaProducer|Utils|TransactionManager|TransactionManagerTest|KafkaAdminClient|NetworkClient|Admin|RaftClientTestContext|TestingMetricsInterceptingAdminClient).java"/>
+ files="(NetworkClient|FieldSpec|KafkaProducer).java"/>
+ files="(KafkaConsumer|ConsumerCoordinator|AbstractFetch|KafkaProducer|AbstractRequest|AbstractResponse|TransactionManager|Admin|KafkaAdminClient|MockAdminClient|KafkaNetworkChannelTest).java"/>
@@ -101,10 +102,10 @@
files="(AbstractFetch|ClientTelemetryReporter|ConsumerCoordinator|CommitRequestManager|FetchCollector|OffsetFetcherUtils|KafkaProducer|Sender|ConfigDef|KerberosLogin|AbstractRequest|AbstractResponse|Selector|SslFactory|SslTransportLayer|SaslClientAuthenticator|SaslClientCallbackHandler|SaslServerAuthenticator|AbstractCoordinator|TransactionManager|AbstractStickyAssignor|DefaultSslEngineFactory|Authorizer|RecordAccumulator|MemoryRecords|FetchSessionHandler|MockAdminClient).java"/>
+ files="(AbstractRequest|AbstractResponse|KerberosLogin|WorkerSinkTaskTest|TransactionManagerTest|SenderTest|KafkaAdminClient|ConsumerCoordinatorTest|KafkaAdminClientTest).java"/>
+ files="(ConsumerCoordinator|BufferPool|MetricName|Node|ConfigDef|RecordBatch|SslFactory|SslTransportLayer|MetadataResponse|KerberosLogin|Selector|Sender|Serdes|TokenInformation|Agent|PluginUtils|MiniTrogdorCluster|TasksRequest|KafkaProducer|AbstractStickyAssignor|Authorizer|FetchSessionHandler|RecordAccumulator|Shell).java"/>
@@ -119,7 +120,7 @@
files="(Sender|Fetcher|FetchRequestManager|OffsetFetcher|KafkaConsumer|Metrics|RequestResponse|TransactionManager|KafkaAdminClient|Message|KafkaProducer)Test.java"/>
+ files="(ConsumerCoordinator|KafkaConsumer|RequestResponse|Fetcher|FetchRequestManager|KafkaAdminClient|Message|KafkaProducer|NetworkClient)Test.java"/>
@@ -186,6 +187,9 @@
+
+
@@ -268,11 +272,11 @@
+ files="(AclCommand|ConsoleConsumer|DefaultMessageFormatter|StreamsResetter|ProducerPerformance|Agent).java"/>
+ files="(AclCommand|DefaultMessageFormatter|ProducerPerformance|StreamsResetter|Agent|TransactionalMessageCopier|ReplicaVerificationTool|LineMessageReader).java"/>
+
+ files="(ConfigurationControlManager|PartitionRegistration|PartitionChangeBuilder|ScramParser).java"/>
-
+
+
diff --git a/clients/src/main/java/org/apache/kafka/clients/ApiVersions.java b/clients/src/main/java/org/apache/kafka/clients/ApiVersions.java
index 64341e1054f7f..e00755e994542 100644
--- a/clients/src/main/java/org/apache/kafka/clients/ApiVersions.java
+++ b/clients/src/main/java/org/apache/kafka/clients/ApiVersions.java
@@ -16,13 +16,8 @@
*/
package org.apache.kafka.clients;
-import org.apache.kafka.common.protocol.ApiKeys;
-import org.apache.kafka.common.record.RecordBatch;
-import org.apache.kafka.common.requests.ProduceRequest;
-
import java.util.HashMap;
import java.util.Map;
-import java.util.Optional;
/**
* Maintains node api versions for access outside of NetworkClient (which is where the information is derived).
@@ -33,7 +28,6 @@
public class ApiVersions {
private final Map nodeApiVersions = new HashMap<>();
- private byte maxUsableProduceMagic = RecordBatch.CURRENT_MAGIC_VALUE;
// The maximum finalized feature epoch of all the node api versions.
private long maxFinalizedFeaturesEpoch = -1;
@@ -50,7 +44,6 @@ public static class FinalizedFeaturesInfo {
public synchronized void update(String nodeId, NodeApiVersions nodeApiVersions) {
this.nodeApiVersions.put(nodeId, nodeApiVersions);
- this.maxUsableProduceMagic = computeMaxUsableProduceMagic();
if (maxFinalizedFeaturesEpoch < nodeApiVersions.finalizedFeaturesEpoch()) {
this.maxFinalizedFeaturesEpoch = nodeApiVersions.finalizedFeaturesEpoch();
this.finalizedFeatures = nodeApiVersions.finalizedFeatures();
@@ -59,7 +52,6 @@ public synchronized void update(String nodeId, NodeApiVersions nodeApiVersions)
public synchronized void remove(String nodeId) {
this.nodeApiVersions.remove(nodeId);
- this.maxUsableProduceMagic = computeMaxUsableProduceMagic();
}
public synchronized NodeApiVersions get(String nodeId) {
@@ -74,19 +66,4 @@ public synchronized FinalizedFeaturesInfo getFinalizedFeaturesInfo() {
return new FinalizedFeaturesInfo(maxFinalizedFeaturesEpoch, finalizedFeatures);
}
- private byte computeMaxUsableProduceMagic() {
- // use a magic version which is supported by all brokers to reduce the chance that
- // we will need to convert the messages when they are ready to be sent.
- Optional knownBrokerNodesMinRequiredMagicForProduce = this.nodeApiVersions.values().stream()
- .filter(versions -> versions.apiVersion(ApiKeys.PRODUCE) != null) // filter out Raft controller nodes
- .map(versions -> ProduceRequest.requiredMagicForVersion(versions.latestUsableVersion(ApiKeys.PRODUCE)))
- .min(Byte::compare);
- return (byte) Math.min(RecordBatch.CURRENT_MAGIC_VALUE,
- knownBrokerNodesMinRequiredMagicForProduce.orElse(RecordBatch.CURRENT_MAGIC_VALUE));
- }
-
- public synchronized byte maxUsableProduceMagic() {
- return maxUsableProduceMagic;
- }
-
}
diff --git a/clients/src/main/java/org/apache/kafka/clients/ClientUtils.java b/clients/src/main/java/org/apache/kafka/clients/ClientUtils.java
index c2c2fca17d1b7..ad4e2d94e6887 100644
--- a/clients/src/main/java/org/apache/kafka/clients/ClientUtils.java
+++ b/clients/src/main/java/org/apache/kafka/clients/ClientUtils.java
@@ -118,7 +118,7 @@ public static ChannelBuilder createChannelBuilder(AbstractConfig config, Time ti
SecurityProtocol securityProtocol = SecurityProtocol.forName(config.getString(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG));
String clientSaslMechanism = config.getString(SaslConfigs.SASL_MECHANISM);
return ChannelBuilders.clientChannelBuilder(securityProtocol, JaasContext.Type.CLIENT, config, null,
- clientSaslMechanism, time, true, logContext);
+ clientSaslMechanism, time, logContext);
}
static List resolve(String host, HostResolver hostResolver) throws UnknownHostException {
@@ -247,6 +247,7 @@ public static NetworkClient createNetworkClient(AbstractConfig config,
logContext,
hostResolver,
clientTelemetrySender,
+ config.getLong(CommonClientConfigs.METADATA_RECOVERY_REBOOTSTRAP_TRIGGER_MS_CONFIG),
MetadataRecoveryStrategy.forName(config.getString(CommonClientConfigs.METADATA_RECOVERY_STRATEGY_CONFIG))
);
} catch (Throwable t) {
@@ -274,4 +275,4 @@ public static ClusterResourceListeners configureClusterResourceListeners(List
return clusterResourceListeners;
}
-}
\ No newline at end of file
+}
diff --git a/clients/src/main/java/org/apache/kafka/clients/CommonClientConfigs.java b/clients/src/main/java/org/apache/kafka/clients/CommonClientConfigs.java
index 1a5bb595d6d39..aa3b5c9d628c9 100644
--- a/clients/src/main/java/org/apache/kafka/clients/CommonClientConfigs.java
+++ b/clients/src/main/java/org/apache/kafka/clients/CommonClientConfigs.java
@@ -130,8 +130,7 @@ public class CommonClientConfigs {
public static final String METRICS_CONTEXT_PREFIX = "metrics.context.";
public static final String SECURITY_PROTOCOL_CONFIG = "security.protocol";
- public static final String SECURITY_PROTOCOL_DOC = "Protocol used to communicate with brokers. Valid values are: " +
- String.join(", ", SecurityProtocol.names()) + ".";
+ public static final String SECURITY_PROTOCOL_DOC = "Protocol used to communicate with brokers.";
public static final String DEFAULT_SECURITY_PROTOCOL = "PLAINTEXT";
public static final String SOCKET_CONNECTION_SETUP_TIMEOUT_MS_CONFIG = "socket.connection.setup.timeout.ms";
@@ -207,7 +206,8 @@ public class CommonClientConfigs {
+ "to the broker. If no heartbeats are received by the broker before the expiration of this session timeout, "
+ "then the broker will remove this client from the group and initiate a rebalance. Note that the value "
+ "must be in the allowable range as configured in the broker configuration by group.min.session.timeout.ms "
- + "and group.max.session.timeout.ms.";
+ + "and group.max.session.timeout.ms. Note that this configuration is not supported when group.protocol "
+ + "is set to \"consumer\".";
public static final String HEARTBEAT_INTERVAL_MS_CONFIG = "heartbeat.interval.ms";
public static final String HEARTBEAT_INTERVAL_MS_DOC = "The expected time between heartbeats to the consumer "
@@ -230,8 +230,15 @@ public class CommonClientConfigs {
"Brokers appear unavailable when disconnected and no current retry attempt is in-progress. " +
"Consider increasing reconnect.backoff.ms and reconnect.backoff.max.ms and " +
"decreasing socket.connection.setup.timeout.ms and socket.connection.setup.timeout.max.ms " +
- "for the client.";
- public static final String DEFAULT_METADATA_RECOVERY_STRATEGY = MetadataRecoveryStrategy.NONE.name;
+ "for the client. Rebootstrap is also triggered if connection cannot be established to any of the brokers for " +
+ "metadata.recovery.rebootstrap.trigger.ms milliseconds or if server requests rebootstrap.";
+ public static final String DEFAULT_METADATA_RECOVERY_STRATEGY = MetadataRecoveryStrategy.REBOOTSTRAP.name;
+
+ public static final String METADATA_RECOVERY_REBOOTSTRAP_TRIGGER_MS_CONFIG = "metadata.recovery.rebootstrap.trigger.ms";
+ public static final String METADATA_RECOVERY_REBOOTSTRAP_TRIGGER_MS_DOC = "If a client configured to rebootstrap using " +
+ "metadata.recovery.strategy=rebootstrap is unable to obtain metadata from any of the brokers in the last known " +
+ "metadata for this interval, client repeats the bootstrap process using bootstrap.servers configuration.";
+ public static final long DEFAULT_METADATA_RECOVERY_REBOOTSTRAP_TRIGGER_MS = 300 * 1000;
/**
* Postprocess the configuration so that exponential backoff is disabled when reconnect backoff
diff --git a/clients/src/main/java/org/apache/kafka/clients/Metadata.java b/clients/src/main/java/org/apache/kafka/clients/Metadata.java
index ece1a25adca19..b60156aae0066 100644
--- a/clients/src/main/java/org/apache/kafka/clients/Metadata.java
+++ b/clients/src/main/java/org/apache/kafka/clients/Metadata.java
@@ -294,7 +294,7 @@ public Map topicIds() {
public synchronized LeaderAndEpoch currentLeader(TopicPartition topicPartition) {
Optional maybeMetadata = partitionMetadataIfCurrent(topicPartition);
- if (!maybeMetadata.isPresent())
+ if (maybeMetadata.isEmpty())
return new LeaderAndEpoch(Optional.empty(), Optional.ofNullable(lastSeenLeaderEpochs.get(topicPartition)));
MetadataResponse.PartitionMetadata partitionMetadata = maybeMetadata.get();
@@ -392,7 +392,7 @@ public synchronized Set updatePartitionLeadership(Map updatePartitionLeadership(Map(State.ACTIVE);
this.telemetrySender = (clientTelemetrySender != null) ? new TelemetrySender(clientTelemetrySender) : null;
+ this.rebootstrapTriggerMs = rebootstrapTriggerMs;
this.metadataRecoveryStrategy = metadataRecoveryStrategy;
}
@@ -401,6 +448,8 @@ public void close(String nodeId) {
long now = time.milliseconds();
cancelInFlightRequests(nodeId, now, null, false);
connectionStates.remove(nodeId);
+ apiVersions.remove(nodeId);
+ nodesNeedingApiVersionsFetch.remove(nodeId);
}
/**
@@ -608,6 +657,7 @@ public List poll(long timeout, long now) {
handleInitiateApiVersionRequests(updatedNow);
handleTimedOutConnections(responses, updatedNow);
handleTimedOutRequests(responses, updatedNow);
+ handleRebootstrap(responses, updatedNow);
completeResponses(responses);
return responses;
@@ -997,7 +1047,6 @@ private void handleApiVersionsResponse(List responses,
NodeApiVersions nodeVersionInfo = new NodeApiVersions(
apiVersionsResponse.data().apiKeys(),
apiVersionsResponse.data().supportedFeatures(),
- apiVersionsResponse.data().zkMigrationReady(),
apiVersionsResponse.data().finalizedFeatures(),
apiVersionsResponse.data().finalizedFeaturesEpoch());
apiVersions.update(node, nodeVersionInfo);
@@ -1065,6 +1114,20 @@ private void handleInitiateApiVersionRequests(long now) {
}
}
+ private void handleRebootstrap(List responses, long now) {
+ if (metadataRecoveryStrategy == MetadataRecoveryStrategy.REBOOTSTRAP && metadataUpdater.needsRebootstrap(now, rebootstrapTriggerMs)) {
+ this.metadataUpdater.fetchNodes().forEach(node -> {
+ String nodeId = node.idString();
+ this.selector.close(nodeId);
+ if (connectionStates.isConnecting(nodeId) || connectionStates.isConnected(nodeId)) {
+ log.info("Disconnecting from node {} due to client rebootstrap.", nodeId);
+ processDisconnection(responses, nodeId, now, ChannelState.LOCAL_CLOSE);
+ }
+ });
+ metadataUpdater.rebootstrap(now);
+ }
+ }
+
/**
* Initiate a connection to the given node
* @param node the node to connect to
@@ -1116,6 +1179,15 @@ class DefaultMetadataUpdater implements MetadataUpdater {
// Defined if there is a request in progress, null otherwise
private InProgressData inProgress;
+ /*
+ * The time in wall-clock milliseconds when we started attempts to fetch metadata. If empty,
+ * metadata has not been requested. This is the start time based on which rebootstrap is
+ * triggered if metadata is not obtained for the configured rebootstrap trigger interval.
+ * Set to Optional.of(0L) to force rebootstrap immediately.
+ */
+ private Optional metadataAttemptStartMs = Optional.empty();
+
+
DefaultMetadataUpdater(Metadata metadata) {
this.metadata = metadata;
this.inProgress = null;
@@ -1146,6 +1218,9 @@ public long maybeUpdate(long now) {
return metadataTimeout;
}
+ if (metadataAttemptStartMs.isEmpty())
+ metadataAttemptStartMs = Optional.of(now);
+
// Beware that the behavior of this method and the computation of timeouts for poll() are
// highly dependent on the behavior of leastLoadedNode.
LeastLoadedNode leastLoadedNode = leastLoadedNode(now);
@@ -1153,7 +1228,7 @@ public long maybeUpdate(long now) {
// Rebootstrap if needed and configured.
if (metadataRecoveryStrategy == MetadataRecoveryStrategy.REBOOTSTRAP
&& !leastLoadedNode.hasNodeAvailableOrConnectionReady()) {
- metadata.rebootstrap();
+ rebootstrap(now);
leastLoadedNode = leastLoadedNode(now);
}
@@ -1219,23 +1294,42 @@ public void handleSuccessfulResponse(RequestHeader requestHeader, long now, Meta
if (!errors.isEmpty())
log.warn("The metadata response from the cluster reported a recoverable issue with correlation id {} : {}", requestHeader.correlationId(), errors);
- // When talking to the startup phase of a broker, it is possible to receive an empty metadata set, which
- // we should retry later.
- if (response.brokers().isEmpty()) {
+ if (metadataRecoveryStrategy == MetadataRecoveryStrategy.REBOOTSTRAP && response.topLevelError() == Errors.REBOOTSTRAP_REQUIRED) {
+ log.info("Rebootstrap requested by server.");
+ initiateRebootstrap();
+ } else if (response.brokers().isEmpty()) {
+ // When talking to the startup phase of a broker, it is possible to receive an empty metadata set, which
+ // we should retry later.
log.trace("Ignoring empty metadata response with correlation id {}.", requestHeader.correlationId());
this.metadata.failedUpdate(now);
} else {
this.metadata.update(inProgress.requestVersion, response, inProgress.isPartialUpdate, now);
+ metadataAttemptStartMs = Optional.empty();
}
inProgress = null;
}
+ @Override
+ public boolean needsRebootstrap(long now, long rebootstrapTriggerMs) {
+ return metadataAttemptStartMs.filter(startMs -> now - startMs > rebootstrapTriggerMs).isPresent();
+ }
+
+ @Override
+ public void rebootstrap(long now) {
+ metadata.rebootstrap();
+ metadataAttemptStartMs = Optional.of(now);
+ }
+
@Override
public void close() {
this.metadata.close();
}
+ private void initiateRebootstrap() {
+ metadataAttemptStartMs = Optional.of(0L); // to force rebootstrap
+ }
+
/**
* Add a metadata request to the list of sends if we can make one
*/
@@ -1317,7 +1411,7 @@ private long maybeUpdate(long now, Node node) {
if (canSendRequest(nodeConnectionId, now)) {
Optional> requestOpt = clientTelemetrySender.createRequest();
- if (!requestOpt.isPresent())
+ if (requestOpt.isEmpty())
return Long.MAX_VALUE;
AbstractRequest.Builder request = requestOpt.get();
diff --git a/clients/src/main/java/org/apache/kafka/clients/NetworkClientUtils.java b/clients/src/main/java/org/apache/kafka/clients/NetworkClientUtils.java
index e044fd48ee3e1..3e161b1e9939b 100644
--- a/clients/src/main/java/org/apache/kafka/clients/NetworkClientUtils.java
+++ b/clients/src/main/java/org/apache/kafka/clients/NetworkClientUtils.java
@@ -71,6 +71,17 @@ public static boolean awaitReady(KafkaClient client, Node node, Time time, long
throw new IOException("Connection to " + node + " failed.");
}
long pollTimeout = timeoutMs - (attemptStartTime - startTime); // initialize in this order to avoid overflow
+
+ // If the network client is waiting to send data for some reason (eg. throttling or retry backoff),
+ // polling longer than that is potentially dangerous as the producer will not attempt to send
+ // any pending requests.
+ long waitingTime = client.pollDelayMs(node, startTime);
+ if (waitingTime > 0 && pollTimeout > waitingTime) {
+ // Block only until the next-scheduled time that it's okay to send data to the producer,
+ // wake up, and try again. This is the way.
+ pollTimeout = waitingTime;
+ }
+
client.poll(pollTimeout, attemptStartTime);
if (client.authenticationException(node) != null)
throw client.authenticationException(node);
diff --git a/clients/src/main/java/org/apache/kafka/clients/NodeApiVersions.java b/clients/src/main/java/org/apache/kafka/clients/NodeApiVersions.java
index 43bd9125a1493..b2eaa481f557e 100644
--- a/clients/src/main/java/org/apache/kafka/clients/NodeApiVersions.java
+++ b/clients/src/main/java/org/apache/kafka/clients/NodeApiVersions.java
@@ -49,8 +49,6 @@ public class NodeApiVersions {
private final Map supportedFeatures;
- private final boolean zkMigrationEnabled;
-
private final Map finalizedFeatures;
private final long finalizedFeaturesEpoch;
@@ -83,7 +81,7 @@ public static NodeApiVersions create(Collection overrides) {
}
if (!exists) apiVersions.add(ApiVersionsResponse.toApiVersion(apiKey));
}
- return new NodeApiVersions(apiVersions, Collections.emptyList(), false, Collections.emptyList(), -1);
+ return new NodeApiVersions(apiVersions, Collections.emptyList(), Collections.emptyList(), -1);
}
@@ -104,16 +102,14 @@ public static NodeApiVersions create(short apiKey, short minVersion, short maxVe
public NodeApiVersions(
Collection nodeApiVersions,
- Collection nodeSupportedFeatures,
- boolean zkMigrationEnabled
+ Collection nodeSupportedFeatures
) {
- this(nodeApiVersions, nodeSupportedFeatures, zkMigrationEnabled, Collections.emptyList(), -1);
+ this(nodeApiVersions, nodeSupportedFeatures, Collections.emptyList(), -1);
}
public NodeApiVersions(
Collection nodeApiVersions,
Collection nodeSupportedFeatures,
- boolean zkMigrationEnabled,
Collection nodeFinalizedFeatures,
long finalizedFeaturesEpoch
) {
@@ -133,8 +129,6 @@ public NodeApiVersions(
new SupportedVersionRange(supportedFeature.minVersion(), supportedFeature.maxVersion()));
}
this.supportedFeatures = Collections.unmodifiableMap(supportedFeaturesBuilder);
- this.zkMigrationEnabled = zkMigrationEnabled;
-
this.finalizedFeaturesEpoch = finalizedFeaturesEpoch;
this.finalizedFeatures = new HashMap<>();
for (ApiVersionsResponseData.FinalizedFeatureKey finalizedFeature : nodeFinalizedFeatures) {
@@ -264,10 +258,6 @@ public Map supportedFeatures() {
return supportedFeatures;
}
- public boolean zkMigrationEnabled() {
- return zkMigrationEnabled;
- }
-
public Map finalizedFeatures() {
return finalizedFeatures;
}
diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/Admin.java b/clients/src/main/java/org/apache/kafka/clients/admin/Admin.java
index dd3b92cf44a21..6c743dec048a8 100644
--- a/clients/src/main/java/org/apache/kafka/clients/admin/Admin.java
+++ b/clients/src/main/java/org/apache/kafka/clients/admin/Admin.java
@@ -486,41 +486,6 @@ default DescribeConfigsResult describeConfigs(Collection resourc
*/
DescribeConfigsResult describeConfigs(Collection resources, DescribeConfigsOptions options);
- /**
- * Update the configuration for the specified resources with the default options.
- *
- * This is a convenience method for {@link #alterConfigs(Map, AlterConfigsOptions)} with default options.
- * See the overload for more details.
- *
- * This operation is supported by brokers with version 0.11.0.0 or higher.
- *
- * @param configs The resources with their configs (topic is the only resource type with configs that can
- * be updated currently)
- * @return The AlterConfigsResult
- * @deprecated Since 2.3. Use {@link #incrementalAlterConfigs(Map)}.
- */
- @Deprecated
- default AlterConfigsResult alterConfigs(Map configs) {
- return alterConfigs(configs, new AlterConfigsOptions());
- }
-
- /**
- * Update the configuration for the specified resources with the default options.
- *
- * Updates are not transactional so they may succeed for some resources while fail for others. The configs for
- * a particular resource are updated atomically.
- *
- * This operation is supported by brokers with version 0.11.0.0 or higher.
- *
- * @param configs The resources with their configs (topic is the only resource type with configs that can
- * be updated currently)
- * @param options The options to use when describing configs
- * @return The AlterConfigsResult
- * @deprecated Since 2.3. Use {@link #incrementalAlterConfigs(Map, AlterConfigsOptions)}.
- */
- @Deprecated
- AlterConfigsResult alterConfigs(Map configs, AlterConfigsOptions options);
-
/**
* Incrementally updates the configuration for the specified resources with default options.
*
@@ -938,9 +903,7 @@ default ListConsumerGroupsResult listConsumerGroups() {
* @return The ListConsumerGroupOffsetsResult
*/
default ListConsumerGroupOffsetsResult listConsumerGroupOffsets(String groupId, ListConsumerGroupOffsetsOptions options) {
- @SuppressWarnings("deprecation")
- ListConsumerGroupOffsetsSpec groupSpec = new ListConsumerGroupOffsetsSpec()
- .topicPartitions(options.topicPartitions());
+ ListConsumerGroupOffsetsSpec groupSpec = new ListConsumerGroupOffsetsSpec();
// We can use the provided options with the batched API, which uses topic partitions from
// the group spec and ignores any topic partitions set in the options.
@@ -1480,14 +1443,23 @@ default DescribeFeaturesResult describeFeatures() {
* error code for each supplied {@link FeatureUpdate}, and the code indicates if the update
* succeeded or failed in the controller.
*
- *
Downgrade of feature version level is not a regular operation/intent. It is only allowed
- * in the controller if the {@link FeatureUpdate} has the allowDowngrade flag set. Setting this
- * flag conveys user intent to attempt downgrade of a feature max version level. Note that
- * despite the allowDowngrade flag being set, certain downgrades may be rejected by the
- * controller if it is deemed impossible.
- *
Deletion of a finalized feature version is not a regular operation/intent. It could be
- * done by setting the allowDowngrade flag to true in the {@link FeatureUpdate}, and, setting
- * the max version level to a value less than 1.
+ *
Downgrading a feature version level is not a common operation and should only be
+ * performed when necessary. It is permitted only if the {@link FeatureUpdate} specifies the
+ * {@code upgradeType} as either {@link FeatureUpdate.UpgradeType#SAFE_DOWNGRADE} or
+ * {@link FeatureUpdate.UpgradeType#UNSAFE_DOWNGRADE}.
+ *
+ *
{@code SAFE_DOWNGRADE}: Allows downgrades that do not lead to metadata loss.
+ *
{@code UNSAFE_DOWNGRADE}: Permits downgrades that might result in metadata loss.
+ *
+ * Note that even with these settings, certain downgrades may still be rejected by the controller
+ * if they are considered unsafe or impossible.
+ *
Deleting a finalized feature version is also not a common operation. To delete a feature,
+ * set the {@code maxVersionLevel} to zero and specify the {@code upgradeType} as either
+ * {@link FeatureUpdate.UpgradeType#SAFE_DOWNGRADE} or
+ * {@link FeatureUpdate.UpgradeType#UNSAFE_DOWNGRADE}.
+ *
The {@link FeatureUpdate.UpgradeType#UPGRADE} type cannot be used when the
+ * {@code maxVersionLevel} is zero. Attempting to do so will result in an
+ * {@link IllegalArgumentException}.
*
*
* The following exceptions can be anticipated when calling {@code get()} on the futures
@@ -1548,8 +1520,7 @@ default DescribeMetadataQuorumResult describeMetadataQuorum() {
/**
* Unregister a broker.
*
- * This operation does not have any effect on partition assignments. It is supported
- * only on Kafka clusters which use Raft to store metadata, rather than ZooKeeper.
+ * This operation does not have any effect on partition assignments.
*
* This is a convenience method for {@link #unregisterBroker(int, UnregisterBrokerOptions)}
*
@@ -1565,8 +1536,7 @@ default UnregisterBrokerResult unregisterBroker(int brokerId) {
/**
* Unregister a broker.
*
- * This operation does not have any effect on partition assignments. It is supported
- * only on Kafka clusters which use Raft to store metadata, rather than ZooKeeper.
+ * This operation does not have any effect on partition assignments.
*
* The following exceptions can be anticipated when calling {@code get()} on the future from the
* returned {@link UnregisterBrokerResult}:
@@ -1574,8 +1544,7 @@ default UnregisterBrokerResult unregisterBroker(int brokerId) {
*
{@link org.apache.kafka.common.errors.TimeoutException}
* If the request timed out before the describe operation could finish.
*
{@link org.apache.kafka.common.errors.UnsupportedVersionException}
- * If the software is too old to support the unregistration API, or if the
- * cluster is not using Raft to store metadata.
+ * If the software is too old to support the unregistration API.
*
*
*
@@ -1825,23 +1794,25 @@ default DescribeShareGroupsResult describeShareGroups(Collection groupId
}
/**
- * List the share groups available in the cluster.
+ * List the share group offsets available in the cluster for the specified share groups.
*
- * @param options The options to use when listing the share groups.
- * @return The ListShareGroupsResult.
+ * @param groupSpecs Map of share group ids to a spec that specifies the topic partitions of the group to list offsets for.
+ * @param options The options to use when listing the share group offsets.
+ * @return The ListShareGroupOffsetsResult
*/
- ListShareGroupsResult listShareGroups(ListShareGroupsOptions options);
+ ListShareGroupOffsetsResult listShareGroupOffsets(Map groupSpecs, ListShareGroupOffsetsOptions options);
/**
- * List the share groups available in the cluster with the default options.
- *
- * This is a convenience method for {@link #listShareGroups(ListShareGroupsOptions)} with default options.
- * See the overload for more details.
+ * List the share group offsets available in the cluster for the specified share groups with the default options.
+ *
+ *
This is a convenience method for {@link #listShareGroupOffsets(Map, ListShareGroupOffsetsOptions)}
+ * to list offsets of all partitions for the specified share groups with default options.
*
- * @return The ListShareGroupsResult.
+ * @param groupSpecs Map of share group ids to a spec that specifies the topic partitions of the group to list offsets for.
+ * @return The ListShareGroupOffsetsResult
*/
- default ListShareGroupsResult listShareGroups() {
- return listShareGroups(new ListShareGroupsOptions());
+ default ListShareGroupOffsetsResult listShareGroupOffsets(Map groupSpecs) {
+ return listShareGroupOffsets(groupSpecs, new ListShareGroupOffsetsOptions());
}
/**
diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/AdminClientConfig.java b/clients/src/main/java/org/apache/kafka/clients/admin/AdminClientConfig.java
index 2a00f450c5ca7..a87af6be154a5 100644
--- a/clients/src/main/java/org/apache/kafka/clients/admin/AdminClientConfig.java
+++ b/clients/src/main/java/org/apache/kafka/clients/admin/AdminClientConfig.java
@@ -141,6 +141,10 @@ public class AdminClientConfig extends AbstractConfig {
public static final String METADATA_RECOVERY_STRATEGY_DOC = CommonClientConfigs.METADATA_RECOVERY_STRATEGY_DOC;
public static final String DEFAULT_METADATA_RECOVERY_STRATEGY = CommonClientConfigs.DEFAULT_METADATA_RECOVERY_STRATEGY;
+ public static final String METADATA_RECOVERY_REBOOTSTRAP_TRIGGER_MS_CONFIG = CommonClientConfigs.METADATA_RECOVERY_REBOOTSTRAP_TRIGGER_MS_CONFIG;
+ public static final String METADATA_RECOVERY_REBOOTSTRAP_TRIGGER_MS_DOC = CommonClientConfigs.METADATA_RECOVERY_REBOOTSTRAP_TRIGGER_MS_DOC;
+ public static final long DEFAULT_METADATA_RECOVERY_REBOOTSTRAP_TRIGGER_MS = CommonClientConfigs.DEFAULT_METADATA_RECOVERY_REBOOTSTRAP_TRIGGER_MS;
+
/**
* security.providers
*/
@@ -270,7 +274,13 @@ public class AdminClientConfig extends AbstractConfig {
ConfigDef.CaseInsensitiveValidString
.in(Utils.enumOptions(MetadataRecoveryStrategy.class)),
Importance.LOW,
- METADATA_RECOVERY_STRATEGY_DOC);
+ METADATA_RECOVERY_STRATEGY_DOC)
+ .define(METADATA_RECOVERY_REBOOTSTRAP_TRIGGER_MS_CONFIG,
+ Type.LONG,
+ DEFAULT_METADATA_RECOVERY_REBOOTSTRAP_TRIGGER_MS,
+ atLeast(0),
+ Importance.LOW,
+ METADATA_RECOVERY_REBOOTSTRAP_TRIGGER_MS_DOC);
}
@Override
diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/AlterConfigsOptions.java b/clients/src/main/java/org/apache/kafka/clients/admin/AlterConfigsOptions.java
index 198a4eab62a70..b2e85b9e813e4 100644
--- a/clients/src/main/java/org/apache/kafka/clients/admin/AlterConfigsOptions.java
+++ b/clients/src/main/java/org/apache/kafka/clients/admin/AlterConfigsOptions.java
@@ -22,7 +22,7 @@
import java.util.Map;
/**
- * Options for {@link Admin#incrementalAlterConfigs(Map)} and {@link Admin#alterConfigs(Map)}.
+ * Options for {@link Admin#incrementalAlterConfigs(Map)}.
*
* The API of this class is evolving, see {@link Admin} for details.
*/
diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/AlterConfigsResult.java b/clients/src/main/java/org/apache/kafka/clients/admin/AlterConfigsResult.java
index 29056ce29403d..007b4422b364d 100644
--- a/clients/src/main/java/org/apache/kafka/clients/admin/AlterConfigsResult.java
+++ b/clients/src/main/java/org/apache/kafka/clients/admin/AlterConfigsResult.java
@@ -24,7 +24,7 @@
import java.util.Map;
/**
- * The result of the {@link Admin#alterConfigs(Map)} call.
+ * The result of the {@link Admin#incrementalAlterConfigs(Map, AlterConfigsOptions)} call.
*
* The API of this class is evolving, see {@link Admin} for details.
*/
diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/ConsumerGroupDescription.java b/clients/src/main/java/org/apache/kafka/clients/admin/ConsumerGroupDescription.java
index 13ec5965eedbb..b2c44d4fbeb63 100644
--- a/clients/src/main/java/org/apache/kafka/clients/admin/ConsumerGroupDescription.java
+++ b/clients/src/main/java/org/apache/kafka/clients/admin/ConsumerGroupDescription.java
@@ -18,14 +18,16 @@
package org.apache.kafka.clients.admin;
import org.apache.kafka.common.ConsumerGroupState;
+import org.apache.kafka.common.GroupState;
import org.apache.kafka.common.GroupType;
import org.apache.kafka.common.Node;
import org.apache.kafka.common.acl.AclOperation;
-import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
+import java.util.List;
import java.util.Objects;
+import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
@@ -38,10 +40,16 @@ public class ConsumerGroupDescription {
private final Collection members;
private final String partitionAssignor;
private final GroupType type;
- private final ConsumerGroupState state;
+ private final GroupState groupState;
private final Node coordinator;
private final Set authorizedOperations;
+ private final Optional groupEpoch;
+ private final Optional targetAssignmentEpoch;
+ /**
+ * @deprecated Since 4.0. Use {@link #ConsumerGroupDescription(String, boolean, Collection, String, GroupType, GroupState, Node, Set, Optional, Optional)} instead.
+ */
+ @Deprecated
public ConsumerGroupDescription(String groupId,
boolean isSimpleConsumerGroup,
Collection members,
@@ -51,6 +59,10 @@ public ConsumerGroupDescription(String groupId,
this(groupId, isSimpleConsumerGroup, members, partitionAssignor, state, coordinator, Collections.emptySet());
}
+ /**
+ * @deprecated Since 4.0. Use {@link #ConsumerGroupDescription(String, boolean, Collection, String, GroupType, GroupState, Node, Set, Optional, Optional)} instead.
+ */
+ @Deprecated
public ConsumerGroupDescription(String groupId,
boolean isSimpleConsumerGroup,
Collection members,
@@ -61,6 +73,10 @@ public ConsumerGroupDescription(String groupId,
this(groupId, isSimpleConsumerGroup, members, partitionAssignor, GroupType.CLASSIC, state, coordinator, authorizedOperations);
}
+ /**
+ * @deprecated Since 4.0. Use {@link #ConsumerGroupDescription(String, boolean, Collection, String, GroupType, GroupState, Node, Set, Optional, Optional)} instead.
+ */
+ @Deprecated
public ConsumerGroupDescription(String groupId,
boolean isSimpleConsumerGroup,
Collection members,
@@ -71,13 +87,36 @@ public ConsumerGroupDescription(String groupId,
Set authorizedOperations) {
this.groupId = groupId == null ? "" : groupId;
this.isSimpleConsumerGroup = isSimpleConsumerGroup;
- this.members = members == null ? Collections.emptyList() :
- Collections.unmodifiableList(new ArrayList<>(members));
+ this.members = members == null ? Collections.emptyList() : List.copyOf(members);
+ this.partitionAssignor = partitionAssignor == null ? "" : partitionAssignor;
+ this.type = type;
+ this.groupState = GroupState.parse(state.toString());
+ this.coordinator = coordinator;
+ this.authorizedOperations = authorizedOperations;
+ this.groupEpoch = Optional.empty();
+ this.targetAssignmentEpoch = Optional.empty();
+ }
+
+ public ConsumerGroupDescription(String groupId,
+ boolean isSimpleConsumerGroup,
+ Collection members,
+ String partitionAssignor,
+ GroupType type,
+ GroupState groupState,
+ Node coordinator,
+ Set authorizedOperations,
+ Optional groupEpoch,
+ Optional targetAssignmentEpoch) {
+ this.groupId = groupId == null ? "" : groupId;
+ this.isSimpleConsumerGroup = isSimpleConsumerGroup;
+ this.members = members == null ? Collections.emptyList() : List.copyOf(members);
this.partitionAssignor = partitionAssignor == null ? "" : partitionAssignor;
this.type = type;
- this.state = state;
+ this.groupState = groupState;
this.coordinator = coordinator;
this.authorizedOperations = authorizedOperations;
+ this.groupEpoch = groupEpoch;
+ this.targetAssignmentEpoch = targetAssignmentEpoch;
}
@Override
@@ -90,14 +129,17 @@ public boolean equals(final Object o) {
Objects.equals(members, that.members) &&
Objects.equals(partitionAssignor, that.partitionAssignor) &&
type == that.type &&
- state == that.state &&
+ groupState == that.groupState &&
Objects.equals(coordinator, that.coordinator) &&
- Objects.equals(authorizedOperations, that.authorizedOperations);
+ Objects.equals(authorizedOperations, that.authorizedOperations) &&
+ Objects.equals(groupEpoch, that.groupEpoch) &&
+ Objects.equals(targetAssignmentEpoch, that.targetAssignmentEpoch);
}
@Override
public int hashCode() {
- return Objects.hash(groupId, isSimpleConsumerGroup, members, partitionAssignor, type, state, coordinator, authorizedOperations);
+ return Objects.hash(groupId, isSimpleConsumerGroup, members, partitionAssignor, type, groupState, coordinator,
+ authorizedOperations, groupEpoch, targetAssignmentEpoch);
}
/**
@@ -138,9 +180,18 @@ public GroupType type() {
/**
* The consumer group state, or UNKNOWN if the state is too new for us to parse.
+ * @deprecated Since 4.0. Use {@link #groupState()} instead.
*/
+ @Deprecated
public ConsumerGroupState state() {
- return state;
+ return ConsumerGroupState.parse(groupState.toString());
+ }
+
+ /**
+ * The group state, or UNKNOWN if the state is too new for us to parse.
+ */
+ public GroupState groupState() {
+ return groupState;
}
/**
@@ -157,6 +208,24 @@ public Set authorizedOperations() {
return authorizedOperations;
}
+ /**
+ * The epoch of the consumer group.
+ * The optional is set to an integer if it is a {@link GroupType#CONSUMER} group, and to empty if it
+ * is a {@link GroupType#CLASSIC} group.
+ */
+ public Optional groupEpoch() {
+ return groupEpoch;
+ }
+
+ /**
+ * The epoch of the target assignment.
+ * The optional is set to an integer if it is a {@link GroupType#CONSUMER} group, and to empty if it
+ * is a {@link GroupType#CLASSIC} group.
+ */
+ public Optional targetAssignmentEpoch() {
+ return targetAssignmentEpoch;
+ }
+
@Override
public String toString() {
return "(groupId=" + groupId +
@@ -164,9 +233,11 @@ public String toString() {
", members=" + members.stream().map(MemberDescription::toString).collect(Collectors.joining(",")) +
", partitionAssignor=" + partitionAssignor +
", type=" + type +
- ", state=" + state +
+ ", groupState=" + groupState +
", coordinator=" + coordinator +
", authorizedOperations=" + authorizedOperations +
+ ", groupEpoch=" + groupEpoch.orElse(null) +
+ ", targetAssignmentEpoch=" + targetAssignmentEpoch.orElse(null) +
")";
}
}
diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/ConsumerGroupListing.java b/clients/src/main/java/org/apache/kafka/clients/admin/ConsumerGroupListing.java
index 96a3ecce9d000..34b9f08f10e06 100644
--- a/clients/src/main/java/org/apache/kafka/clients/admin/ConsumerGroupListing.java
+++ b/clients/src/main/java/org/apache/kafka/clients/admin/ConsumerGroupListing.java
@@ -18,6 +18,7 @@
package org.apache.kafka.clients.admin;
import org.apache.kafka.common.ConsumerGroupState;
+import org.apache.kafka.common.GroupState;
import org.apache.kafka.common.GroupType;
import java.util.Objects;
@@ -29,28 +30,30 @@
public class ConsumerGroupListing {
private final String groupId;
private final boolean isSimpleConsumerGroup;
- private final Optional state;
+ private final Optional groupState;
private final Optional type;
/**
* Create an instance with the specified parameters.
*
- * @param groupId Group Id
- * @param isSimpleConsumerGroup If consumer group is simple or not.
+ * @param groupId Group Id.
+ * @param isSimpleConsumerGroup If consumer group is simple or not.
*/
public ConsumerGroupListing(String groupId, boolean isSimpleConsumerGroup) {
- this(groupId, isSimpleConsumerGroup, Optional.empty(), Optional.empty());
+ this(groupId, Optional.empty(), Optional.empty(), isSimpleConsumerGroup);
}
/**
* Create an instance with the specified parameters.
*
- * @param groupId Group Id
- * @param isSimpleConsumerGroup If consumer group is simple or not.
- * @param state The state of the consumer group
+ * @param groupId Group Id.
+ * @param isSimpleConsumerGroup If consumer group is simple or not.
+ * @param state The state of the consumer group.
+ * @deprecated Since 4.0. Use {@link #ConsumerGroupListing(String, Optional, boolean)} instead.
*/
+ @Deprecated
public ConsumerGroupListing(String groupId, boolean isSimpleConsumerGroup, Optional state) {
- this(groupId, isSimpleConsumerGroup, state, Optional.empty());
+ this(groupId, Objects.requireNonNull(state).map(state0 -> GroupState.parse(state0.toString())), Optional.empty(), isSimpleConsumerGroup);
}
/**
@@ -60,17 +63,51 @@ public ConsumerGroupListing(String groupId, boolean isSimpleConsumerGroup, Optio
* @param isSimpleConsumerGroup If consumer group is simple or not.
* @param state The state of the consumer group.
* @param type The type of the consumer group.
+ * @deprecated Since 4.0. Use {@link #ConsumerGroupListing(String, Optional, Optional, boolean)} instead.
*/
+ @Deprecated
public ConsumerGroupListing(
String groupId,
boolean isSimpleConsumerGroup,
Optional state,
Optional type
+ ) {
+ this(groupId, Objects.requireNonNull(state).map(state0 -> GroupState.parse(state0.toString())), type, isSimpleConsumerGroup);
+ }
+
+ /**
+ * Create an instance with the specified parameters.
+ *
+ * @param groupId Group Id.
+ * @param groupState The state of the consumer group.
+ * @param isSimpleConsumerGroup If consumer group is simple or not.
+ */
+ public ConsumerGroupListing(
+ String groupId,
+ Optional groupState,
+ boolean isSimpleConsumerGroup
+ ) {
+ this(groupId, groupState, Optional.empty(), isSimpleConsumerGroup);
+ }
+
+ /**
+ * Create an instance with the specified parameters.
+ *
+ * @param groupId Group Id.
+ * @param groupState The state of the consumer group.
+ * @param type The type of the consumer group.
+ * @param isSimpleConsumerGroup If consumer group is simple or not.
+ */
+ public ConsumerGroupListing(
+ String groupId,
+ Optional groupState,
+ Optional type,
+ boolean isSimpleConsumerGroup
) {
this.groupId = groupId;
- this.isSimpleConsumerGroup = isSimpleConsumerGroup;
- this.state = Objects.requireNonNull(state);
+ this.groupState = Objects.requireNonNull(groupState);
this.type = Objects.requireNonNull(type);
+ this.isSimpleConsumerGroup = isSimpleConsumerGroup;
}
/**
@@ -87,11 +124,20 @@ public boolean isSimpleConsumerGroup() {
return isSimpleConsumerGroup;
}
+ /**
+ * Group state
+ */
+ public Optional groupState() {
+ return groupState;
+ }
+
/**
* Consumer Group state
+ * @deprecated Since 4.0. Use {@link #groupState()} instead.
*/
+ @Deprecated
public Optional state() {
- return state;
+ return groupState.map(state0 -> ConsumerGroupState.parse(state0.toString()));
}
/**
@@ -108,14 +154,14 @@ public String toString() {
return "(" +
"groupId='" + groupId + '\'' +
", isSimpleConsumerGroup=" + isSimpleConsumerGroup +
- ", state=" + state +
+ ", groupState=" + groupState +
", type=" + type +
')';
}
@Override
public int hashCode() {
- return Objects.hash(groupId, isSimpleConsumerGroup(), state, type);
+ return Objects.hash(groupId, isSimpleConsumerGroup(), groupState, type);
}
@Override
@@ -125,7 +171,7 @@ public boolean equals(Object o) {
ConsumerGroupListing that = (ConsumerGroupListing) o;
return isSimpleConsumerGroup() == that.isSimpleConsumerGroup() &&
Objects.equals(groupId, that.groupId) &&
- Objects.equals(state, that.state) &&
+ Objects.equals(groupState, that.groupState) &&
Objects.equals(type, that.type);
}
}
diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/CreateDelegationTokenOptions.java b/clients/src/main/java/org/apache/kafka/clients/admin/CreateDelegationTokenOptions.java
index 13540b0299cfc..8e5c24aa6fc1f 100644
--- a/clients/src/main/java/org/apache/kafka/clients/admin/CreateDelegationTokenOptions.java
+++ b/clients/src/main/java/org/apache/kafka/clients/admin/CreateDelegationTokenOptions.java
@@ -31,7 +31,7 @@
*/
@InterfaceStability.Evolving
public class CreateDelegationTokenOptions extends AbstractOptions {
- private long maxLifeTimeMs = -1;
+ private long maxLifetimeMs = -1;
private List renewers = new LinkedList<>();
private KafkaPrincipal owner = null;
@@ -53,12 +53,29 @@ public Optional owner() {
return Optional.ofNullable(owner);
}
- public CreateDelegationTokenOptions maxlifeTimeMs(long maxLifeTimeMs) {
- this.maxLifeTimeMs = maxLifeTimeMs;
+ /**
+ * @deprecated Since 4.0 and should not be used any longer.
+ */
+ @Deprecated
+ public CreateDelegationTokenOptions maxlifeTimeMs(long maxLifetimeMs) {
+ this.maxLifetimeMs = maxLifetimeMs;
return this;
}
+ public CreateDelegationTokenOptions maxLifetimeMs(long maxLifetimeMs) {
+ this.maxLifetimeMs = maxLifetimeMs;
+ return this;
+ }
+
+ /**
+ * @deprecated Since 4.0 and should not be used any longer.
+ */
+ @Deprecated
public long maxlifeTimeMs() {
- return maxLifeTimeMs;
+ return maxLifetimeMs;
+ }
+
+ public long maxLifetimeMs() {
+ return maxLifetimeMs;
}
}
diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/DeleteConsumerGroupsResult.java b/clients/src/main/java/org/apache/kafka/clients/admin/DeleteConsumerGroupsResult.java
index 90ddbd0582c49..5c1e60b1a6176 100644
--- a/clients/src/main/java/org/apache/kafka/clients/admin/DeleteConsumerGroupsResult.java
+++ b/clients/src/main/java/org/apache/kafka/clients/admin/DeleteConsumerGroupsResult.java
@@ -42,7 +42,7 @@ public class DeleteConsumerGroupsResult {
*/
public Map> deletedGroups() {
Map> deletedGroups = new HashMap<>(futures.size());
- futures.forEach((key, future) -> deletedGroups.put(key, future));
+ deletedGroups.putAll(futures);
return deletedGroups;
}
diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/DeleteTopicsResult.java b/clients/src/main/java/org/apache/kafka/clients/admin/DeleteTopicsResult.java
index 725b82a78dee8..8ab7cdd801baf 100644
--- a/clients/src/main/java/org/apache/kafka/clients/admin/DeleteTopicsResult.java
+++ b/clients/src/main/java/org/apache/kafka/clients/admin/DeleteTopicsResult.java
@@ -70,16 +70,6 @@ public Map> topicNameValues() {
return nameFutures;
}
- /**
- * @return a map from topic names to futures which can be used to check the status of
- * individual deletions if the deleteTopics request used topic names. Otherwise return null.
- * @deprecated Since 3.0 use {@link #topicNameValues} instead
- */
- @Deprecated
- public Map> values() {
- return nameFutures;
- }
-
/**
* @return a future which succeeds only if all the topic deletions succeed.
*/
diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/DescribeClusterOptions.java b/clients/src/main/java/org/apache/kafka/clients/admin/DescribeClusterOptions.java
index 2eac1f055f6e0..a6ef6f7f0f34d 100644
--- a/clients/src/main/java/org/apache/kafka/clients/admin/DescribeClusterOptions.java
+++ b/clients/src/main/java/org/apache/kafka/clients/admin/DescribeClusterOptions.java
@@ -29,6 +29,8 @@ public class DescribeClusterOptions extends AbstractOptions>> values() {
- return descriptions().entrySet().stream()
- .collect(Collectors.toMap(
- Map.Entry::getKey,
- entry -> entry.getValue().thenApply(this::convertMapValues)));
- }
-
- @SuppressWarnings("deprecation")
- private Map convertMapValues(Map map) {
- Stream> stream = map.entrySet().stream();
- return stream.collect(Collectors.toMap(
- Map.Entry::getKey,
- infoEntry -> {
- LogDirDescription logDir = infoEntry.getValue();
- return new DescribeLogDirsResponse.LogDirInfo(logDir.error() == null ? Errors.NONE : Errors.forException(logDir.error()),
- logDir.replicaInfos().entrySet().stream().collect(Collectors.toMap(
- Map.Entry::getKey,
- replicaEntry -> new DescribeLogDirsResponse.ReplicaInfo(
- replicaEntry.getValue().size(),
- replicaEntry.getValue().offsetLag(),
- replicaEntry.getValue().isFuture())
- )));
- }));
- }
-
/**
* Return a map from brokerId to future which can be used to check the information of partitions on each individual broker.
* The result of the future is a map from broker log directory path to a description of that log directory.
@@ -81,18 +47,6 @@ public Map>> descriptions()
return futures;
}
- /**
- * Return a future which succeeds only if all the brokers have responded without error
- * @deprecated Deprecated Since Kafka 2.7. Use {@link #allDescriptions()}.
- */
- @Deprecated
- public KafkaFuture>> all() {
- return allDescriptions().thenApply(map -> map.entrySet().stream().collect(Collectors.toMap(
- Map.Entry::getKey,
- entry -> convertMapValues(entry.getValue())
- )));
- }
-
/**
* Return a future which succeeds only if all the brokers have responded without error.
* The result of the future is a map from brokerId to a map from broker log directory path
diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/DescribeTopicsResult.java b/clients/src/main/java/org/apache/kafka/clients/admin/DescribeTopicsResult.java
index 1c68d8180b4cb..6673918c7313a 100644
--- a/clients/src/main/java/org/apache/kafka/clients/admin/DescribeTopicsResult.java
+++ b/clients/src/main/java/org/apache/kafka/clients/admin/DescribeTopicsResult.java
@@ -37,11 +37,6 @@ public class DescribeTopicsResult {
private final Map> topicIdFutures;
private final Map> nameFutures;
- @Deprecated
- protected DescribeTopicsResult(Map> futures) {
- this(null, futures);
- }
-
// VisibleForTesting
protected DescribeTopicsResult(Map> topicIdFutures, Map> nameFutures) {
if (topicIdFutures != null && nameFutures != null)
@@ -80,30 +75,6 @@ public Map> topicNameValues() {
return nameFutures;
}
- /**
- * @return a map from topic names to futures which can be used to check the status of
- * individual topics if the request used topic names, otherwise return null.
- *
- * @deprecated Since 3.1.0 use {@link #topicNameValues} instead
- */
- @Deprecated
- public Map> values() {
- return nameFutures;
- }
-
- /**
- * @return A future map from topic names to descriptions which can be used to check
- * the status of individual description if the describe topic request used
- * topic names, otherwise return null, this request succeeds only if all the
- * topic descriptions succeed
- *
- * @deprecated Since 3.1.0 use {@link #allTopicNames()} instead
- */
- @Deprecated
- public KafkaFuture> all() {
- return all(nameFutures);
- }
-
/**
* @return A future map from topic names to descriptions which can be used to check
* the status of individual description if the describe topic request used
diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/DescribeUserScramCredentialsResult.java b/clients/src/main/java/org/apache/kafka/clients/admin/DescribeUserScramCredentialsResult.java
index 2eddd7ee28c0c..f13fd2a8e7d18 100644
--- a/clients/src/main/java/org/apache/kafka/clients/admin/DescribeUserScramCredentialsResult.java
+++ b/clients/src/main/java/org/apache/kafka/clients/admin/DescribeUserScramCredentialsResult.java
@@ -125,7 +125,7 @@ public KafkaFuture description(String userName)
// for users 1, 2, and 3 but this is looking for user 4), so explicitly take care of that case
Optional optionalUserResult =
data.results().stream().filter(result -> result.user().equals(userName)).findFirst();
- if (!optionalUserResult.isPresent()) {
+ if (optionalUserResult.isEmpty()) {
retval.completeExceptionally(new ResourceNotFoundException("No such user: " + userName));
} else {
DescribeUserScramCredentialsResponseData.DescribeUserScramCredentialsResult userResult = optionalUserResult.get();
diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/FeatureUpdate.java b/clients/src/main/java/org/apache/kafka/clients/admin/FeatureUpdate.java
index a2a71b528e402..0b7b78ff9a0b8 100644
--- a/clients/src/main/java/org/apache/kafka/clients/admin/FeatureUpdate.java
+++ b/clients/src/main/java/org/apache/kafka/clients/admin/FeatureUpdate.java
@@ -54,21 +54,6 @@ public static UpgradeType fromCode(int code) {
}
}
- /**
- * @param maxVersionLevel the new maximum version level for the finalized feature.
- * a value of zero is special and indicates that the update is intended to
- * delete the finalized feature, and should be accompanied by setting
- * the allowDowngrade flag to true.
- * @param allowDowngrade - true, if this feature update was meant to downgrade the existing
- * maximum version level of the finalized feature. Only "safe" downgrades are
- * enabled with this boolean. See {@link FeatureUpdate#FeatureUpdate(short, UpgradeType)}
- * - false, otherwise.
- */
- @Deprecated
- public FeatureUpdate(final short maxVersionLevel, final boolean allowDowngrade) {
- this(maxVersionLevel, allowDowngrade ? UpgradeType.SAFE_DOWNGRADE : UpgradeType.UPGRADE);
- }
-
/**
* @param maxVersionLevel The new maximum version level for the finalized feature.
* a value of zero is special and indicates that the update is intended to
@@ -96,11 +81,6 @@ public short maxVersionLevel() {
return maxVersionLevel;
}
- @Deprecated
- public boolean allowDowngrade() {
- return upgradeType != UpgradeType.UPGRADE;
- }
-
public UpgradeType upgradeType() {
return upgradeType;
}
diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/ForwardingAdmin.java b/clients/src/main/java/org/apache/kafka/clients/admin/ForwardingAdmin.java
index 285acaf695e30..45ed560dffc5f 100644
--- a/clients/src/main/java/org/apache/kafka/clients/admin/ForwardingAdmin.java
+++ b/clients/src/main/java/org/apache/kafka/clients/admin/ForwardingAdmin.java
@@ -103,12 +103,6 @@ public DescribeConfigsResult describeConfigs(Collection resource
return delegate.describeConfigs(resources, options);
}
- @Deprecated
- @Override
- public AlterConfigsResult alterConfigs(Map configs, AlterConfigsOptions options) {
- return delegate.alterConfigs(configs, options);
- }
-
@Override
public AlterConfigsResult incrementalAlterConfigs(Map> configs, AlterConfigsOptions options) {
return delegate.incrementalAlterConfigs(configs, options);
@@ -305,8 +299,8 @@ public DescribeShareGroupsResult describeShareGroups(Collection groupIds
}
@Override
- public ListShareGroupsResult listShareGroups(ListShareGroupsOptions options) {
- return delegate.listShareGroups(options);
+ public ListShareGroupOffsetsResult listShareGroupOffsets(Map groupSpecs, ListShareGroupOffsetsOptions options) {
+ return delegate.listShareGroupOffsets(groupSpecs, options);
}
@Override
diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/GroupListing.java b/clients/src/main/java/org/apache/kafka/clients/admin/GroupListing.java
index 0ee2a211e704b..8a0727c284fa6 100644
--- a/clients/src/main/java/org/apache/kafka/clients/admin/GroupListing.java
+++ b/clients/src/main/java/org/apache/kafka/clients/admin/GroupListing.java
@@ -17,7 +17,9 @@
package org.apache.kafka.clients.admin;
+import org.apache.kafka.common.GroupState;
import org.apache.kafka.common.GroupType;
+import org.apache.kafka.common.annotation.InterfaceStability;
import java.util.Objects;
import java.util.Optional;
@@ -25,22 +27,26 @@
/**
* A listing of a group in the cluster.
*/
+@InterfaceStability.Evolving
public class GroupListing {
private final String groupId;
private final Optional type;
private final String protocol;
+ private final Optional groupState;
/**
* Create an instance with the specified parameters.
*
- * @param groupId Group Id
- * @param type Group type
- * @param protocol Protocol
+ * @param groupId Group Id
+ * @param type Group type
+ * @param protocol Protocol
+ * @param groupState Group state
*/
- public GroupListing(String groupId, Optional type, String protocol) {
+ public GroupListing(String groupId, Optional type, String protocol, Optional groupState) {
this.groupId = groupId;
this.type = Objects.requireNonNull(type);
this.protocol = protocol;
+ this.groupState = groupState;
}
/**
@@ -75,6 +81,19 @@ public String protocol() {
return protocol;
}
+ /**
+ * The group state.
+ *
+ * If the broker returns a group state which is not recognised, as might
+ * happen when talking to a broker with a later version, the state will be
+ * Optional.of(GroupState.UNKNOWN).
+ *
+ * @return An Optional containing the state, if available.
+ */
+ public Optional groupState() {
+ return groupState;
+ }
+
/**
* If the group is a simple consumer group or not.
*/
@@ -88,12 +107,13 @@ public String toString() {
"groupId='" + groupId + '\'' +
", type=" + type.map(GroupType::toString).orElse("none") +
", protocol='" + protocol + '\'' +
+ ", groupState=" + groupState.map(GroupState::toString).orElse("none") +
')';
}
@Override
public int hashCode() {
- return Objects.hash(groupId, type, protocol);
+ return Objects.hash(groupId, type, protocol, groupState);
}
@Override
@@ -103,6 +123,7 @@ public boolean equals(Object o) {
GroupListing that = (GroupListing) o;
return Objects.equals(groupId, that.groupId) &&
Objects.equals(type, that.type) &&
- Objects.equals(protocol, that.protocol);
+ Objects.equals(protocol, that.protocol) &&
+ Objects.equals(groupState, that.groupState);
}
}
diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java b/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java
index ef46b49db6efc..3f88cd5aa8070 100644
--- a/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java
+++ b/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java
@@ -58,12 +58,13 @@
import org.apache.kafka.clients.admin.internals.ListConsumerGroupOffsetsHandler;
import org.apache.kafka.clients.admin.internals.ListOffsetsHandler;
import org.apache.kafka.clients.admin.internals.ListTransactionsHandler;
+import org.apache.kafka.clients.admin.internals.PartitionLeaderStrategy;
import org.apache.kafka.clients.admin.internals.RemoveMembersFromConsumerGroupHandler;
import org.apache.kafka.clients.consumer.OffsetAndMetadata;
import org.apache.kafka.clients.consumer.internals.ConsumerProtocol;
import org.apache.kafka.common.Cluster;
-import org.apache.kafka.common.ConsumerGroupState;
import org.apache.kafka.common.ElectionType;
+import org.apache.kafka.common.GroupState;
import org.apache.kafka.common.GroupType;
import org.apache.kafka.common.KafkaException;
import org.apache.kafka.common.KafkaFuture;
@@ -71,7 +72,6 @@
import org.apache.kafka.common.MetricName;
import org.apache.kafka.common.Node;
import org.apache.kafka.common.PartitionInfo;
-import org.apache.kafka.common.ShareGroupState;
import org.apache.kafka.common.TopicCollection;
import org.apache.kafka.common.TopicCollection.TopicIdCollection;
import org.apache.kafka.common.TopicCollection.TopicNameCollection;
@@ -182,8 +182,6 @@
import org.apache.kafka.common.requests.AddRaftVoterResponse;
import org.apache.kafka.common.requests.AlterClientQuotasRequest;
import org.apache.kafka.common.requests.AlterClientQuotasResponse;
-import org.apache.kafka.common.requests.AlterConfigsRequest;
-import org.apache.kafka.common.requests.AlterConfigsResponse;
import org.apache.kafka.common.requests.AlterPartitionReassignmentsRequest;
import org.apache.kafka.common.requests.AlterPartitionReassignmentsResponse;
import org.apache.kafka.common.requests.AlterReplicaLogDirsRequest;
@@ -407,6 +405,7 @@ public class KafkaAdminClient extends AdminClient {
private final long retryBackoffMaxMs;
private final ExponentialBackoff retryBackoff;
private final MetadataRecoveryStrategy metadataRecoveryStrategy;
+ private final Map partitionLeaderCache;
private final AdminFetchMetricsManager adminFetchMetricsManager;
private final Optional clientTelemetryReporter;
@@ -633,6 +632,7 @@ private KafkaAdminClient(AdminClientConfig config,
this.clientTelemetryReporter = clientTelemetryReporter;
this.clientTelemetryReporter.ifPresent(reporters::add);
this.metadataRecoveryStrategy = MetadataRecoveryStrategy.forName(config.getString(AdminClientConfig.METADATA_RECOVERY_STRATEGY_CONFIG));
+ this.partitionLeaderCache = new HashMap<>();
this.adminFetchMetricsManager = new AdminFetchMetricsManager(metrics);
config.logUnused();
AppInfoParser.registerAppInfo(JMX_PREFIX, clientId, metrics, time.milliseconds());
@@ -722,10 +722,11 @@ private interface NodeProvider {
private class MetadataUpdateNodeIdProvider implements NodeProvider {
@Override
public Node provide() {
- LeastLoadedNode leastLoadedNode = client.leastLoadedNode(time.milliseconds());
+ long now = time.milliseconds();
+ LeastLoadedNode leastLoadedNode = client.leastLoadedNode(now);
if (metadataRecoveryStrategy == MetadataRecoveryStrategy.REBOOTSTRAP
&& !leastLoadedNode.hasNodeAvailableOrConnectionReady()) {
- metadataManager.rebootstrap(time.milliseconds());
+ metadataManager.rebootstrap(now);
}
return leastLoadedNode.node();
@@ -1701,7 +1702,11 @@ public MetadataRequest.Builder createRequest(int timeoutMs) {
public void handleResponse(AbstractResponse abstractResponse) {
MetadataResponse response = (MetadataResponse) abstractResponse;
long now = time.milliseconds();
- metadataManager.update(response.buildCluster(), now);
+
+ if (response.topLevelError() == Errors.REBOOTSTRAP_REQUIRED)
+ metadataManager.initiateRebootstrap();
+ else
+ metadataManager.update(response.buildCluster(), now);
// Unassign all unsent requests after a metadata refresh to allow for a new
// destination to be selected from the new metadata
@@ -2499,10 +2504,14 @@ public DescribeClusterResult describeCluster(DescribeClusterOptions options) {
@Override
AbstractRequest.Builder createRequest(int timeoutMs) {
if (!useMetadataRequest) {
+ if (metadataManager.usingBootstrapControllers() && options.includeFencedBrokers()) {
+ throw new IllegalArgumentException("Cannot request fenced brokers from controller endpoint");
+ }
return new DescribeClusterRequest.Builder(new DescribeClusterRequestData()
.setIncludeClusterAuthorizedOperations(options.includeAuthorizedOperations())
.setEndpointType(metadataManager.usingBootstrapControllers() ?
- EndpointType.CONTROLLER.id() : EndpointType.BROKER.id()));
+ EndpointType.CONTROLLER.id() : EndpointType.BROKER.id())
+ .setIncludeFencedBrokers(options.includeFencedBrokers()));
} else {
// Since this only requests node information, it's safe to pass true for allowAutoTopicCreation (and it
// simplifies communication with older brokers)
@@ -2518,7 +2527,6 @@ AbstractRequest.Builder createRequest(int timeoutMs) {
void handleResponse(AbstractResponse abstractResponse) {
if (!useMetadataRequest) {
DescribeClusterResponse response = (DescribeClusterResponse) abstractResponse;
-
Errors error = Errors.forCode(response.data().errorCode());
if (error != Errors.NONE) {
ApiError apiError = new ApiError(error, response.data().errorMessage());
@@ -2566,6 +2574,12 @@ boolean handleUnsupportedVersionException(final UnsupportedVersionException exce
return false;
}
+ // If unsupportedVersion exception was caused by the option to include fenced brokers (only supported for version 2+)
+ // then we should not fall back to the metadataRequest.
+ if (options.includeFencedBrokers()) {
+ return false;
+ }
+
useMetadataRequest = true;
return true;
}
@@ -2861,72 +2875,6 @@ private ConfigEntry.ConfigSource configSource(DescribeConfigsResponse.ConfigSour
return configSource;
}
- @Override
- @Deprecated
- public AlterConfigsResult alterConfigs(Map configs, final AlterConfigsOptions options) {
- final Map> allFutures = new HashMap<>();
- // We must make a separate AlterConfigs request for every BROKER resource we want to alter
- // and send the request to that specific node. Other resources are grouped together into
- // a single request that may be sent to any node.
- final Collection unifiedRequestResources = new ArrayList<>();
-
- for (ConfigResource resource : configs.keySet()) {
- Integer node = nodeFor(resource);
- if (node != null) {
- NodeProvider nodeProvider = new ConstantBrokerOrActiveKController(node);
- allFutures.putAll(alterConfigs(configs, options, Collections.singleton(resource), nodeProvider));
- } else
- unifiedRequestResources.add(resource);
- }
- if (!unifiedRequestResources.isEmpty())
- allFutures.putAll(alterConfigs(configs, options, unifiedRequestResources, new LeastLoadedBrokerOrActiveKController()));
- return new AlterConfigsResult(new HashMap<>(allFutures));
- }
-
- private Map> alterConfigs(Map configs,
- final AlterConfigsOptions options,
- Collection resources,
- NodeProvider nodeProvider) {
- final Map> futures = new HashMap<>();
- final Map requestMap = new HashMap<>(resources.size());
- for (ConfigResource resource : resources) {
- List configEntries = new ArrayList<>();
- for (ConfigEntry configEntry: configs.get(resource).entries())
- configEntries.add(new AlterConfigsRequest.ConfigEntry(configEntry.name(), configEntry.value()));
- requestMap.put(resource, new AlterConfigsRequest.Config(configEntries));
- futures.put(resource, new KafkaFutureImpl<>());
- }
-
- final long now = time.milliseconds();
- runnable.call(new Call("alterConfigs", calcDeadlineMs(now, options.timeoutMs()), nodeProvider) {
-
- @Override
- public AlterConfigsRequest.Builder createRequest(int timeoutMs) {
- return new AlterConfigsRequest.Builder(requestMap, options.shouldValidateOnly());
- }
-
- @Override
- public void handleResponse(AbstractResponse abstractResponse) {
- AlterConfigsResponse response = (AlterConfigsResponse) abstractResponse;
- for (Map.Entry> entry : futures.entrySet()) {
- KafkaFutureImpl future = entry.getValue();
- ApiException exception = response.errors().get(entry.getKey()).exception();
- if (exception != null) {
- future.completeExceptionally(exception);
- } else {
- future.complete(null);
- }
- }
- }
-
- @Override
- void handleFailure(Throwable throwable) {
- completeAllExceptionally(futures.values(), throwable);
- }
- }, now);
- return futures;
- }
-
@Override
public AlterConfigsResult incrementalAlterConfigs(Map> configs,
final AlterConfigsOptions options) {
@@ -3343,7 +3291,8 @@ void handleFailure(Throwable throwable) {
@Override
public DeleteRecordsResult deleteRecords(final Map recordsToDelete,
final DeleteRecordsOptions options) {
- SimpleAdminApiFuture future = DeleteRecordsHandler.newFuture(recordsToDelete.keySet());
+ PartitionLeaderStrategy.PartitionLeaderFuture future =
+ DeleteRecordsHandler.newFuture(recordsToDelete.keySet(), partitionLeaderCache);
int timeoutMs = defaultApiTimeoutMs;
if (options.timeoutMs() != null) {
timeoutMs = options.timeoutMs();
@@ -3371,7 +3320,7 @@ public CreateDelegationTokenResult createDelegationToken(final CreateDelegationT
CreateDelegationTokenRequest.Builder createRequest(int timeoutMs) {
CreateDelegationTokenRequestData data = new CreateDelegationTokenRequestData()
.setRenewers(renewers)
- .setMaxLifetimeMs(options.maxlifeTimeMs());
+ .setMaxLifetimeMs(options.maxLifetimeMs());
if (options.owner().isPresent()) {
data.setOwnerPrincipalName(options.owner().get().getName());
data.setOwnerPrincipalType(options.owner().get().getPrincipalType());
@@ -3576,8 +3525,13 @@ ListGroupsRequest.Builder createRequest(int timeoutMs) {
.stream()
.map(GroupType::toString)
.collect(Collectors.toList());
+ List groupStates = options.groupStates()
+ .stream()
+ .map(GroupState::toString)
+ .collect(Collectors.toList());
return new ListGroupsRequest.Builder(new ListGroupsRequestData()
.setTypesFilter(groupTypes)
+ .setStatesFilter(groupStates)
);
}
@@ -3590,10 +3544,17 @@ private void maybeAddGroup(ListGroupsResponseData.ListedGroup group) {
type = Optional.of(GroupType.parse(group.groupType()));
}
final String protocolType = group.protocolType();
+ final Optional groupState;
+ if (group.groupState() == null || group.groupState().isEmpty()) {
+ groupState = Optional.empty();
+ } else {
+ groupState = Optional.of(GroupState.parse(group.groupState()));
+ }
final GroupListing groupListing = new GroupListing(
groupId,
type,
- protocolType
+ protocolType,
+ groupState
);
results.addListing(groupListing);
}
@@ -3718,9 +3679,9 @@ void handleResponse(AbstractResponse abstractResponse) {
runnable.call(new Call("listConsumerGroups", deadline, new ConstantNodeIdProvider(node.id())) {
@Override
ListGroupsRequest.Builder createRequest(int timeoutMs) {
- List states = options.states()
+ List states = options.groupStates()
.stream()
- .map(ConsumerGroupState::toString)
+ .map(GroupState::toString)
.collect(Collectors.toList());
List groupTypes = options.types()
.stream()
@@ -3736,17 +3697,17 @@ private void maybeAddConsumerGroup(ListGroupsResponseData.ListedGroup group) {
String protocolType = group.protocolType();
if (protocolType.equals(ConsumerProtocol.PROTOCOL_TYPE) || protocolType.isEmpty()) {
final String groupId = group.groupId();
- final Optional state = group.groupState().isEmpty()
+ final Optional groupState = group.groupState().isEmpty()
? Optional.empty()
- : Optional.of(ConsumerGroupState.parse(group.groupState()));
+ : Optional.of(GroupState.parse(group.groupState()));
final Optional type = group.groupType().isEmpty()
? Optional.empty()
: Optional.of(GroupType.parse(group.groupType()));
final ConsumerGroupListing groupListing = new ConsumerGroupListing(
groupId,
- protocolType.isEmpty(),
- state,
- type
+ groupState,
+ type,
+ protocolType.isEmpty()
);
results.addListing(groupListing);
}
@@ -3824,48 +3785,6 @@ public DeleteConsumerGroupOffsetsResult deleteConsumerGroupOffsets(
return new DeleteConsumerGroupOffsetsResult(future.get(CoordinatorKey.byGroupId(groupId)), partitions);
}
- private static final class ListShareGroupsResults {
- private final List errors;
- private final HashMap listings;
- private final HashSet remaining;
- private final KafkaFutureImpl> future;
-
- ListShareGroupsResults(Collection leaders,
- KafkaFutureImpl> future) {
- this.errors = new ArrayList<>();
- this.listings = new HashMap<>();
- this.remaining = new HashSet<>(leaders);
- this.future = future;
- tryComplete();
- }
-
- synchronized void addError(Throwable throwable, Node node) {
- ApiError error = ApiError.fromThrowable(throwable);
- if (error.message() == null || error.message().isEmpty()) {
- errors.add(error.error().exception("Error listing groups on " + node));
- } else {
- errors.add(error.error().exception("Error listing groups on " + node + ": " + error.message()));
- }
- }
-
- synchronized void addListing(ShareGroupListing listing) {
- listings.put(listing.groupId(), listing);
- }
-
- synchronized void tryComplete(Node leader) {
- remaining.remove(leader);
- tryComplete();
- }
-
- private synchronized void tryComplete() {
- if (remaining.isEmpty()) {
- ArrayList
+