diff --git a/.ci/snapshoty.yml b/.ci/snapshoty.yml deleted file mode 100644 index ef49bdf10..000000000 --- a/.ci/snapshoty.yml +++ /dev/null @@ -1,53 +0,0 @@ - ---- - -# Version of configuration to use -version: '1.0' - -# You can define a Google Cloud Account to use -account: - # Project id of the service account - project: '${GCS_PROJECT}' - # Private key id of the service account - private_key_id: '${GCS_PRIVATE_KEY_ID}' - # Private key of the service account - private_key: '${GCS_PRIVATE_KEY}' - # Email of the service account - client_email: '${GCS_CLIENT_EMAIL}' - # URI token - token_uri: 'https://oauth2.googleapis.com/token' - -x-metadata: &metadata - # Define static custom metadata - - name: 'custom' - data: - project: 'apm-agent-dotnet' - component: 'agent' - # Add git metadata - - name: 'git' - # Add github_actions metadata - - name: 'github_actions' - - -# List of artifacts -artifacts: - # Path to use for artifacts discovery - - path: './build/output' - # Files pattern to match - files_pattern: 'elastic_apm_profiler_(?P\d+\.\d+\.\d+)-(?P[\w\.]+)-(?P\w+)-(?P\w+)\.zip' - # File layout on GCS bucket - output_pattern: '{project}/{github_branch_name}/elastic-apm-dotnet-profiler-{app_version}-{app_version}-{os}-{arch}-{github_sha_short}.jar' - # List of metadata processors to use. - metadata: *metadata - - path: './build/output' - files_pattern: 'ElasticApmAgent_(?P\d+\.\d+\.\d+)-(?P[\w\.]+)\.zip' - output_pattern: '{project}/{github_branch_name}/elastic-apm-dotnet-agent-{app_version}-{revision}-{github_sha_short}.zip' - metadata: *metadata - - path: './build/output/_packages' - files_pattern: 'Elastic\.Apm\.(?P[\w\.]*)\.(?P\d+\.\d+\.\d+)-(?P[\w\.]+)(-(?P\w+)-(?P[\d-]+))?\.nupkg' - output_pattern: '{project}/{github_branch_name}/elastic-apm-dotnet-{component}-{app_version}-{revision}-{github_sha_short}.nupkg' - metadata: *metadata - - path: './build/output/_packages' - files_pattern: 'Elastic\.Apm\.(?P\d+\.\d+\.\d+)-(?P[\w\.]+)(-(?P\w+)-(?P[\d-]+))?\.nupkg' - output_pattern: '{project}/{github_branch_name}/elastic-apm-dotnet-{app_version}-{revision}-{github_sha_short}.nupkg' - metadata: *metadata diff --git a/.ci/updatecli/updatecli.d/update-gherkin-specs.yml b/.ci/updatecli/updatecli.d/update-gherkin-specs.yml deleted file mode 100644 index 13f0617a9..000000000 --- a/.ci/updatecli/updatecli.d/update-gherkin-specs.yml +++ /dev/null @@ -1,83 +0,0 @@ -name: update-gherkin-specs -pipelineid: update-gherkin-specs - -scms: - default: - kind: github - spec: - user: '{{ requiredEnv "GITHUB_ACTOR" }}' - owner: "{{ .github.owner }}" - repository: "{{ .github.repository }}" - token: '{{ requiredEnv "GITHUB_TOKEN" }}' - username: '{{ requiredEnv "GITHUB_ACTOR" }}' - branch: "{{ .github.branch }}" - commitusingapi: true - apm: - kind: github - spec: - user: '{{ requiredEnv "GITHUB_ACTOR" }}' - owner: "{{ .github.owner }}" - repository: "{{ .github.apm_repository }}" - token: '{{ requiredEnv "GITHUB_TOKEN" }}' - username: '{{ requiredEnv "GITHUB_ACTOR" }}' - branch: "{{ .github.branch }}" - -sources: - sha: - kind: file - spec: - file: 'https://github.com/{{ .github.owner }}/{{ .github.apm_repository }}/commit/{{ .github.branch }}.patch' - matchpattern: "^From\\s([0-9a-f]{40})\\s" - transformers: - - findsubmatch: - pattern: "[0-9a-f]{40}" - pull_request: - kind: shell - dependson: - - sha - spec: - command: gh api /repos/{{ .github.owner }}/{{ .github.apm_repository }}/commits/{{ source "sha" }}/pulls --jq '.[].html_url' - environments: - - name: GITHUB_TOKEN - - name: PATH - agents-gherkin-specs-tarball: - kind: shell - scmid: apm - dependson: - - sha - spec: - command: tar cvzf {{ requiredEnv "GITHUB_WORKSPACE" }}/gherkin-specs.tgz . - environments: - - name: PATH - workdir: "{{ .specs.apm_gherkin_path }}" - -actions: - pr: - kind: "github/pullrequest" - scmid: default - spec: - automerge: false - draft: false - labels: - - "automation" - description: |- - ### What - APM agent Gherkin specs automatic sync - - ### Why - *Changeset* - * {{ source "pull_request" }} - * https://github.com/elastic/apm/commit/{{ source "sha" }} - title: '[Automation] Update Gherkin specs' - -targets: - agent-gherkin-specs: - name: APM agent gherkin specs {{ source "sha" }} - scmid: default - disablesourceinput: true - kind: shell - spec: - # git diff helps to print what it changed, If it is empty, then updatecli report a success with no changes applied. - # See https://www.updatecli.io/docs/plugins/resource/shell/#_shell_target - command: 'tar -xzf {{ requiredEnv "GITHUB_WORKSPACE" }}/gherkin-specs.tgz && git --no-pager diff' - workdir: "{{ .apm_agent.gherkin_specs_path }}" diff --git a/.ci/updatecli/updatecli.d/update-json-specs.yml b/.ci/updatecli/updatecli.d/update-json-specs.yml deleted file mode 100644 index ae9816775..000000000 --- a/.ci/updatecli/updatecli.d/update-json-specs.yml +++ /dev/null @@ -1,83 +0,0 @@ -name: update-json-specs -pipelineid: update-json-specs - -scms: - default: - kind: github - spec: - user: '{{ requiredEnv "GITHUB_ACTOR" }}' - owner: "{{ .github.owner }}" - repository: "{{ .github.repository }}" - token: '{{ requiredEnv "GITHUB_TOKEN" }}' - username: '{{ requiredEnv "GITHUB_ACTOR" }}' - branch: "{{ .github.branch }}" - commitusingapi: true - apm: - kind: github - spec: - user: '{{ requiredEnv "GITHUB_ACTOR" }}' - owner: "{{ .github.owner }}" - repository: "{{ .github.apm_repository }}" - token: '{{ requiredEnv "GITHUB_TOKEN" }}' - username: '{{ requiredEnv "GITHUB_ACTOR" }}' - branch: "{{ .github.branch }}" - -sources: - sha: - kind: file - spec: - file: 'https://github.com/{{ .github.owner }}/{{ .github.apm_repository }}/commit/{{ .github.branch }}.patch' - matchpattern: "^From\\s([0-9a-f]{40})\\s" - transformers: - - findsubmatch: - pattern: "[0-9a-f]{40}" - pull_request: - kind: shell - dependson: - - sha - spec: - command: gh api /repos/{{ .github.owner }}/{{ .github.apm_repository }}/commits/{{ source "sha" }}/pulls --jq '.[].html_url' - environments: - - name: GITHUB_TOKEN - - name: PATH - agents-json-specs-tarball: - kind: shell - scmid: apm - dependson: - - sha - spec: - command: tar cvzf {{ requiredEnv "GITHUB_WORKSPACE" }}/json-specs.tgz . - environments: - - name: PATH - workdir: "{{ .specs.apm_json_path }}" - -actions: - pr: - kind: "github/pullrequest" - scmid: default - spec: - automerge: false - draft: false - labels: - - "automation" - description: |- - ### What - APM agent specs automatic sync - - ### Why - *Changeset* - * {{ source "pull_request" }} - * https://github.com/{{ .github.owner }}/{{ .github.apm_repository }}/commit/{{ source "sha" }} - title: '[Automation] Update JSON specs' - -targets: - agent-json-specs: - name: APM agent json specs {{ source "sha" }} - scmid: default - disablesourceinput: true - kind: shell - spec: - # git diff helps to print what it changed, If it is empty, then updatecli report a success with no changes applied. - # See https://www.updatecli.io/docs/plugins/resource/shell/#_shell_target - command: 'tar -xzf {{ requiredEnv "GITHUB_WORKSPACE" }}/json-specs.tgz && git --no-pager diff' - workdir: "{{ .apm_agent.json_specs_path }}" diff --git a/.ci/updatecli/updatecli.d/update-specs.yml b/.ci/updatecli/updatecli.d/update-specs.yml deleted file mode 100644 index e9bbaf945..000000000 --- a/.ci/updatecli/updatecli.d/update-specs.yml +++ /dev/null @@ -1,85 +0,0 @@ -name: update-specs -pipelineid: update-schema-specs - -scms: - default: - kind: github - spec: - user: '{{ requiredEnv "GITHUB_ACTOR" }}' - owner: "{{ .github.owner }}" - repository: "{{ .github.repository }}" - token: '{{ requiredEnv "GITHUB_TOKEN" }}' - username: '{{ requiredEnv "GITHUB_ACTOR" }}' - branch: "{{ .github.branch }}" - commitusingapi: true - - apm-data: - kind: github - spec: - user: '{{ requiredEnv "GITHUB_ACTOR" }}' - owner: "{{ .github.owner }}" - repository: "{{ .github.apm_data_repository }}" - token: '{{ requiredEnv "GITHUB_TOKEN" }}' - username: '{{ requiredEnv "GITHUB_ACTOR" }}' - branch: "{{ .github.branch }}" - -sources: - sha: - kind: file - spec: - file: 'https://github.com/{{ .github.owner }}/{{ .github.apm_data_repository }}/commit/{{ .github.branch }}.patch' - matchpattern: "^From\\s([0-9a-f]{40})\\s" - transformers: - - findsubmatch: - pattern: "[0-9a-f]{40}" - pull_request: - kind: shell - dependson: - - sha - spec: - command: gh api /repos/{{ .github.owner }}/{{ .github.apm_data_repository }}/commits/{{ source "sha" }}/pulls --jq '.[].html_url' - environments: - - name: GITHUB_TOKEN - - name: PATH - agent-specs-tarball: - kind: shell - scmid: apm-data - dependson: - - sha - spec: - command: tar cvzf {{ requiredEnv "GITHUB_WORKSPACE" }}/json-schema.tgz . - environments: - - name: PATH - workdir: "{{ .specs.apm_data_path }}" - -actions: - pr: - kind: "github/pullrequest" - scmid: default - sourceid: sha - spec: - automerge: false - draft: false - labels: - - "automation" - description: |- - ### What - APM agent json server schema automatic sync - - ### Why - *Changeset* - * {{ source "pull_request" }} - * https://github.com/{{ .github.owner }}/{{ .github.apm_data_repository }}/commit/{{ source "sha" }} - title: '[Automation] Update JSON server schema specs' - -targets: - agent-json-schema: - name: APM agent json server schema {{ source "sha" }} - scmid: default - disablesourceinput: true - kind: shell - spec: - # git diff helps to print what it changed, If it is empty, then updatecli report a success with no changes applied. - # See https://www.updatecli.io/docs/plugins/resource/shell/#_shell_target - command: 'tar -xzf {{ requiredEnv "GITHUB_WORKSPACE" }}/json-schema.tgz && git --no-pager diff' - workdir: "{{ .apm_agent.server_schema_specs_path }}" diff --git a/.ci/updatecli/values.d/apm-data-spec.yml b/.ci/updatecli/values.d/apm-data-spec.yml new file mode 100644 index 000000000..eba382561 --- /dev/null +++ b/.ci/updatecli/values.d/apm-data-spec.yml @@ -0,0 +1 @@ +apm_schema_specs_path: src/Elastic.Apm.Specification/specs diff --git a/.ci/updatecli/values.d/apm-gherkin.yml b/.ci/updatecli/values.d/apm-gherkin.yml new file mode 100644 index 000000000..86dace3dc --- /dev/null +++ b/.ci/updatecli/values.d/apm-gherkin.yml @@ -0,0 +1 @@ +apm_gherkin_specs_path: test/Elastic.Apm.Feature.Tests/Features diff --git a/.ci/updatecli/values.d/apm-json-specs.yml b/.ci/updatecli/values.d/apm-json-specs.yml new file mode 100644 index 000000000..27e728e1c --- /dev/null +++ b/.ci/updatecli/values.d/apm-json-specs.yml @@ -0,0 +1 @@ +apm_json_specs_path: test/Elastic.Apm.Tests.Utilities/TestResources/json-specs diff --git a/.ci/updatecli/values.d/scm.yml b/.ci/updatecli/values.d/scm.yml new file mode 100644 index 000000000..876325fb5 --- /dev/null +++ b/.ci/updatecli/values.d/scm.yml @@ -0,0 +1,7 @@ +scm: + enabled: true + owner: elastic + repository: apm-agent-dotnet + branch: main + +signedcommit: true \ No newline at end of file diff --git a/.ci/updatecli/values.yml b/.ci/updatecli/values.yml deleted file mode 100644 index 575ba4a45..000000000 --- a/.ci/updatecli/values.yml +++ /dev/null @@ -1,14 +0,0 @@ -github: - owner: "elastic" - repository: "apm-agent-dotnet" - apm_repository: "apm" - apm_data_repository: "apm-data" - branch: "main" -specs: - apm_data_path: "input/elasticapm/docs/spec/v2" - apm_json_path: "tests/agents/json-specs" - apm_gherkin_path: "tests/agents/gherkin-specs" -apm_agent: - gherkin_specs_path: "test/Elastic.Apm.Feature.Tests/Features" - json_specs_path: "test/Elastic.Apm.Tests.Utilities/TestResources/json-specs" - server_schema_specs_path: "src/Elastic.Apm.Specification/specs" \ No newline at end of file diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 2d8ddf3b0..9ec06ba2d 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -7,10 +7,6 @@ There are 4 main stages that run on GitHub actions: * Test * Release -There are some other stages that run for every push on the main branches: - -* [Snapshoty](./snapshoty.yml) - ### Scenarios * Tests should be triggered on branch, tag and PR basis. @@ -39,7 +35,7 @@ The tag release follows the naming convention: `v...`, wher ### OpenTelemetry -There is a GitHub workflow in charge to populate what the workflow run in terms of jobs and steps. Those details can be seen in [here](https://ela.st/oblt-ci-cd-stats) (**NOTE**: only available for Elasticians). +Every workflow and its logs are exported to OpenTelemetry traces/logs/metrics. Those details can be seen [here](https://ela.st/oblt-ci-cd-stats) (**NOTE**: only available for Elasticians). ## Bump automation diff --git a/.github/workflows/bootstrap/action.yml b/.github/workflows/bootstrap/action.yml index 4446c068d..88c6f40a0 100644 --- a/.github/workflows/bootstrap/action.yml +++ b/.github/workflows/bootstrap/action.yml @@ -11,6 +11,10 @@ inputs: description: 'Install azure functions tool chain ("true" or "false")' required: false default: "false" + tc-cloud: + description: 'Bootstrap TestContainers Cloud (TOKEN or "false")' + required: false + default: "false" outputs: agent-version: description: "The current agent version number" @@ -18,6 +22,7 @@ outputs: major-version: description: "The current major version number, semver" value: ${{ steps.dotnet.outputs.major-version }} + runs: using: "composite" @@ -27,7 +32,7 @@ runs: run: | git fetch --prune --unshallow --tags git tag --list - + - uses: actions/cache@v4 with: path: ~/.nuget/packages @@ -56,10 +61,11 @@ runs: # Setup git config - uses: elastic/apm-pipeline-library/.github/actions/setup-git@current - + # install common dependencies - name: Install common dependencies uses: ./.github/workflows/install-dependencies with: rust: '${{ inputs.rust }}' azure: '${{ inputs.azure }}' + tc-cloud: '${{ inputs.tc-cloud }}' diff --git a/.github/workflows/install-dependencies/action.yml b/.github/workflows/install-dependencies/action.yml index cc3e300fe..df05f3bde 100644 --- a/.github/workflows/install-dependencies/action.yml +++ b/.github/workflows/install-dependencies/action.yml @@ -1,57 +1,89 @@ --- -name: Install Dependencies -description: Ensures an action has the appropiate non .NET dependencies installed - -inputs: - rust: - description: 'Install rust toolchain ("true" or "false")' - required: false - default: "false" - azure: + name: Install Dependencies + description: Ensures an action has the appropiate non .NET dependencies installed + + inputs: + rust: + description: 'Install rust toolchain ("true" or "false")' + required: false + default: "false" + azure: description: 'Install azure functions tool chain ("true" or "false")' required: false default: "false" - -runs: - using: "composite" - steps: - # RUST - - name: Rustup - if: "${{ inputs.rust == 'true' }}" - shell: bash - run: rustup default 1.69.0 - - # - name: Cargo make - # if: "${{ inputs.rust == 'true' }}" - # shell: bash - # run: cargo install --force cargo-make - - - name: Install cargo-make using cache - if: "${{ inputs.rust == 'true' }}" - uses: baptiste0928/cargo-install@v3 - with: - crate: cargo-make - version: "^0.36.8" - - - uses: Swatinem/rust-cache@v2 - if: "${{ inputs.rust == 'true' }}" - with: - cache-targets: "false" - cache-all-crates: "true" - + tc-cloud: + description: 'Bootstrap TestContainers Cloud (TOKEN or "false")' + required: false + default: "false" - # AZURE - - name: 'Linux: Azure functions core tools' - if: "${{ inputs.azure == 'true' && runner.os == 'Linux' }}" - shell: bash - run: | - wget -q https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - sudo dpkg -i packages-microsoft-prod.deb - sudo apt-get update - sudo apt-get install azure-functions-core-tools-4 + runs: + using: "composite" + steps: + # ZIG + - name: Zig + if: "${{ inputs.rust == 'true' && runner.os == 'Linux' }}" + shell: bash + run: sudo snap install zig --beta --classic + + # RUST + - name: Rustup + if: "${{ inputs.rust == 'true' }}" + shell: bash + run: rustup default 1.79.0 + + # - name: Cargo make + # if: "${{ inputs.rust == 'true' }}" + # shell: bash + # run: cargo install --force cargo-make + + - name: Install cargo-make using cache + if: "${{ inputs.rust == 'true' }}" + uses: baptiste0928/cargo-install@v3 + with: + crate: cargo-make + version: "^0.36.8" + + - name: Install cargo zigbuild + if: "${{ inputs.rust == 'true' && runner.os == 'Linux' }}" + shell: bash + run: cargo install --force cargo-zigbuild + + - uses: Swatinem/rust-cache@v2 + if: "${{ inputs.rust == 'true' }}" + with: + cache-targets: "false" + cache-all-crates: "true" + + # AZURE + - name: 'Linux: Azure functions core tools' + if: "${{ inputs.azure == 'true' && runner.os == 'Linux' }}" + shell: bash + run: | + wget -q https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + sudo apt-get update + sudo apt-get install azure-functions-core-tools-4 + + - name: 'Windows: Azure functions core tools' + if: "${{ inputs.azure == 'true' && runner.os == 'Windows' }}" + shell: cmd + run: choco install azure-functions-core-tools -y --no-progress -r --version 4.0.4829 - - name: 'Windows: Azure functions core tools' - if: "${{ inputs.azure == 'true' && runner.os == 'Windows' }}" - shell: cmd - run: choco install azure-functions-core-tools -y --no-progress -r --version 4.0.4829 - + # TEST CONTAINERS CLOUD + # If no PR event or if a PR event that's caused by a non-fork and non dependabot actor + - name: Setup TestContainers Cloud Client + if: | + inputs.tc-cloud != 'false' + && (github.event_name != 'pull_request' + || (github.event_name == 'pull_request' + && github.event.pull_request.head.repo.fork == false + && github.actor != 'dependabot[bot]' + ) + ) + uses: atomicjar/testcontainers-cloud-setup-action@c335bdbb570ec7c48f72c7d450c077f0a002293e # v1.3.0 + with: + token: ${{ inputs.tc-cloud }} + + + + \ No newline at end of file diff --git a/.github/workflows/release-main.yml b/.github/workflows/release-main.yml index f65c1fda6..bd5b886b5 100644 --- a/.github/workflows/release-main.yml +++ b/.github/workflows/release-main.yml @@ -5,9 +5,11 @@ on: branches: [ "main" ] permissions: - contents: write - issues: write - packages: write + attestations: write + contents: write + id-token: write + issues: write + packages: write env: NUGET_PACKAGES: ${{ github.workspace }}/.nuget/packages @@ -15,7 +17,12 @@ env: jobs: release: runs-on: ubuntu-latest - + env: + DOCKER_IMAGE_NAME: "docker.elastic.co/observability/apm-agent-dotnet" + PREFIX_APM_AGENT: "build/output/ElasticApmAgent_" + PREFIX_APM_PROFILER: "build/output/elastic_apm_profiler_" + SUFFIX_APM_AGENT: ".zip" + SUFFIX_APM_PROFILER: "-linux-x64.zip" steps: - uses: actions/checkout@v4 - name: Bootstrap Action Workspace @@ -27,31 +34,79 @@ jobs: - name: Package with canary suffix run: ./build.sh pack - - name: Prepare feedz.io - uses: hashicorp/vault-action@v3.0.0 - with: - url: ${{ secrets.VAULT_ADDR }} - method: approle - roleId: ${{ secrets.VAULT_ROLE_ID }} - secretId: ${{ secrets.VAULT_SECRET_ID }} - secrets: | - secret/apm-team/ci/elastic-observability-feedz.io apiKey | REPO_API_KEY ; - secret/apm-team/ci/elastic-observability-feedz.io url | REPO_API_URL - + - name: generate build provenance + uses: actions/attest-build-provenance@bdd51370e0416ac948727f861e03c2f05d32d78e # v1.3.2 + with: + subject-path: "${{ github.workspace }}/build/output/_packages/*.nupkg" + # Github packages requires authentication, this is likely going away in the future so for now we publish to feedz.io - name: publish canary packages to feedz.io - run: dotnet nuget push 'build/output/_packages/*.nupkg' -k ${REPO_API_KEY} -s ${REPO_API_URL} --skip-duplicate --no-symbols + run: dotnet nuget push 'build/output/_packages/*.nupkg' -k ${{ secrets.FEEDZ_IO_API_KEY }} -s ${{ secrets.FEEDZ_IO_API_URL }} --skip-duplicate --no-symbols - name: publish canary packages github package repository run: dotnet nuget push 'build/output/_packages/*.nupkg' -k ${{secrets.GITHUB_TOKEN}} -s https://nuget.pkg.github.com/elastic/index.json --skip-duplicate --no-symbols - + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v3.3.0 + + - name: Log in to the Elastic Container registry + uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3.2.0 + with: + registry: ${{ secrets.ELASTIC_DOCKER_REGISTRY }} + username: ${{ secrets.ELASTIC_DOCKER_USERNAME }} + password: ${{ secrets.ELASTIC_DOCKER_PASSWORD }} + + - name: Extract metadata (tags, labels) + id: docker-meta + uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1 + with: + images: ${{ env.DOCKER_IMAGE_NAME }} + flavor: | + latest=auto + tags: | + # "1.2.3" and "latest" Docker tags on push of git tag "v1.2.3" + type=raw,value=${{ steps.bootstrap.outputs.agent-version }} + # "edge" Docker tag on git push to default branch + type=edge + + - name: Build and Push Profiler Docker Image + id: docker-push + continue-on-error: true # continue for now until we see it working in action + uses: docker/build-push-action@31159d49c0d4756269a0940a750801a1ea5d7003 # v6.1.0 + with: + cache-from: type=gha + cache-to: type=gha,mode=max + context: . + platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ steps.docker-meta.outputs.tags }} + labels: ${{ steps.docker-meta.outputs.labels }} + build-args: | + AGENT_ZIP_FILE=${{ env.PREFIX_APM_PROFILER }}${{ steps.bootstrap.outputs.agent-version }}${{ env.SUFFIX_APM_PROFILER }} + + - name: Attest image + uses: actions/attest-build-provenance@bdd51370e0416ac948727f861e03c2f05d32d78e # v1.3.2 + continue-on-error: true # continue for now until we see it working in action + with: + subject-name: ${{ env.DOCKER_IMAGE_NAME }} + subject-digest: ${{ steps.docker-push.outputs.digest }} + push-to-registry: true + + - name: generate build provenance (APM Agent) + uses: actions/attest-build-provenance@bdd51370e0416ac948727f861e03c2f05d32d78e # v1.3.2 + with: + subject-path: "${{ github.workspace }}/${{ env.PREFIX_APM_AGENT }}${{ steps.bootstrap.outputs.agent-version }}${{ env.SUFFIX_APM_AGENT }}" + + - name: generate build provenance (APM Profiler) + uses: actions/attest-build-provenance@bdd51370e0416ac948727f861e03c2f05d32d78e # v1.3.2 + with: + subject-path: "${{ github.workspace }}/${{ env.PREFIX_APM_PROFILER }}${{ steps.bootstrap.outputs.agent-version }}${{ env.SUFFIX_APM_PROFILER }}" + - if: ${{ failure() }} - uses: elastic/apm-pipeline-library/.github/actions/slack-message@current + uses: elastic/oblt-actions/slack/send@v1 with: - url: ${{ secrets.VAULT_ADDR }} - roleId: ${{ secrets.VAULT_ROLE_ID }} - secretId: ${{ secrets.VAULT_SECRET_ID }} - channel: "#apm-agent-dotnet" + bot-token: ${{ secrets.SLACK_BOT_TOKEN }} + channel-id: "#apm-agent-dotnet" message: | :large_yellow_circle: [${{ github.repository }}] Snapshot could not be published to feedz.io. Build: (<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|here>) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ceca035a9..a8ad00f38 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -17,77 +17,109 @@ env: jobs: release: runs-on: ubuntu-latest - + env: + PREFIX_APM_AGENT: "build/output/ElasticApmAgent_" + PREFIX_APM_PROFILER: "build/output/elastic_apm_profiler_" + SUFFIX_APM_AGENT: ".zip" + SUFFIX_APM_PROFILER: "-linux-x64.zip" + DOCKER_IMAGE_NAME: "docker.elastic.co/observability/apm-agent-dotnet" + permissions: + attestations: write + contents: write + id-token: write steps: - uses: actions/checkout@v4 - name: Bootstrap Action Workspace id: bootstrap uses: ./.github/workflows/bootstrap with: - rust: 'true' + rust: 'true' - name: Package run: ./build.sh pack - - name: Prepare Nuget - uses: hashicorp/vault-action@v3.0.0 + - name: Release to Nuget + run: .ci/linux/deploy.sh ${{ secrets.NUGET_API_KEY }} ${{ secrets.NUGET_API_URL }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v3.3.0 + + - name: Log in to the Elastic Container registry + uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3.2.0 with: - url: ${{ secrets.VAULT_ADDR }} - method: approle - roleId: ${{ secrets.VAULT_ROLE_ID }} - secretId: ${{ secrets.VAULT_SECRET_ID }} - secrets: | - secret/apm-team/ci/elastic-observability-nuget apiKey | REPO_API_KEY ; - secret/apm-team/ci/elastic-observability-nuget url | REPO_API_URL + registry: ${{ secrets.ELASTIC_DOCKER_REGISTRY }} + username: ${{ secrets.ELASTIC_DOCKER_USERNAME }} + password: ${{ secrets.ELASTIC_DOCKER_PASSWORD }} - - name: Release to Nuget - run: .ci/linux/deploy.sh ${REPO_API_KEY} ${REPO_API_URL} + - name: Extract metadata (tags, labels) + id: docker-meta + uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1 + with: + images: ${{ env.DOCKER_IMAGE_NAME }} + flavor: | + latest=auto + tags: | + # "1.2.3" and "latest" Docker tags on push of git tag "v1.2.3" + type=semver,pattern={{version}},value=${{ steps.bootstrap.outputs.agent-version }} + # "edge" Docker tag on git push to default branch + type=edge + + - name: Build and Push Profiler Docker Image + id: docker-push + continue-on-error: true # continue for now until we see it working in action + uses: docker/build-push-action@31159d49c0d4756269a0940a750801a1ea5d7003 # v6.1.0 + with: + cache-from: type=gha + cache-to: type=gha,mode=max + context: . + platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ steps.docker-meta.outputs.tags }} + labels: ${{ steps.docker-meta.outputs.labels }} + build-args: | + AGENT_ZIP_FILE=${{ env.PREFIX_APM_PROFILER }}${{ steps.bootstrap.outputs.agent-version }}${{ env.SUFFIX_APM_PROFILER }} + + - name: Attest image + uses: actions/attest-build-provenance@bdd51370e0416ac948727f861e03c2f05d32d78e # v1.3.2 + continue-on-error: true # continue for now until we see it working in action + with: + subject-name: ${{ env.DOCKER_IMAGE_NAME }} + subject-digest: ${{ steps.docker-push.outputs.digest }} + push-to-registry: true - - uses: elastic/apm-pipeline-library/.github/actions/docker-login@current + - name: generate build provenance (APM Agent) + uses: actions/attest-build-provenance@bdd51370e0416ac948727f861e03c2f05d32d78e # v1.3.2 with: - registry: docker.elastic.co - secret: secret/observability-team/ci/docker-registry/prod - url: ${{ secrets.VAULT_ADDR }} - roleId: ${{ secrets.VAULT_ROLE_ID }} - secretId: ${{ secrets.VAULT_SECRET_ID }} + subject-path: "${{ github.workspace }}/${{ env.PREFIX_APM_AGENT }}${{ steps.bootstrap.outputs.agent-version }}${{ env.SUFFIX_APM_AGENT }}" + + - name: generate build provenance (APM Profiler) + uses: actions/attest-build-provenance@bdd51370e0416ac948727f861e03c2f05d32d78e # v1.3.2 + with: + subject-path: "${{ github.workspace }}/${{ env.PREFIX_APM_PROFILER }}${{ steps.bootstrap.outputs.agent-version }}${{ env.SUFFIX_APM_PROFILER }}" - - name: Build Profiler Docker Image - continue-on-error: true #continue for now until we see it working in action - run: | - docker build . -t docker.elastic.co/observability/apm-agent-dotnet:${{ steps.bootstrap.outputs.agent-version }} \ - --build-arg AGENT_ZIP_FILE=build/output/elastic_apm_profiler_${{ steps.bootstrap.outputs.agent-version }}-linux-x64.zip - - - name: Push Profiler Docker Image - continue-on-error: true #continue for now until we see it working in action - run: | - for i in $(seq 1 3); do [ $i -gt 1 ] && sleep 15; docker push docker.elastic.co/observability/apm-agent-dotnet:${{ steps.bootstrap.outputs.agent-version }} && s=0 && break || s=$?; done; (exit $s) - - name: Attach Profiler And Startup Hooks - continue-on-error: true #continue for now until we see it working in action env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - gh release upload ${{ github.ref_name }} "build/output/ElasticApmAgent_${{ steps.bootstrap.outputs.agent-version }}.zip" "build/output/elastic_apm_profiler_${{ steps.bootstrap.outputs.agent-version }}-linux-x64.zip" + gh release upload ${{ github.ref_name }} "${{ env.PREFIX_APM_AGENT }}${{ steps.bootstrap.outputs.agent-version }}${{ env.SUFFIX_APM_AGENT }}" "${{ env.PREFIX_APM_PROFILER }}${{ steps.bootstrap.outputs.agent-version }}${{ env.SUFFIX_APM_PROFILER }}" - if: ${{ success() }} - uses: elastic/apm-pipeline-library/.github/actions/slack-message@current + uses: elastic/oblt-actions/slack/send@v1 + continue-on-error: true #continue for now until we see it working in action with: - url: ${{ secrets.VAULT_ADDR }} - roleId: ${{ secrets.VAULT_ROLE_ID }} - secretId: ${{ secrets.VAULT_SECRET_ID }} - channel: ${{ env.SLACK_CHANNEL }} + bot-token: ${{ secrets.SLACK_BOT_TOKEN }} + channel-id: ${{ env.SLACK_CHANNEL }} message: | :large_green_circle: [${{ github.repository }}] Release *${{ github.ref_name }}* published. Build: (<${{ env.JOB_URL }}|here>) Release URL: () - if: ${{ failure() }} - uses: elastic/apm-pipeline-library/.github/actions/slack-message@current + uses: elastic/oblt-actions/slack/send@v1 + continue-on-error: true #continue for now until we see it working in action with: - url: ${{ secrets.VAULT_ADDR }} - roleId: ${{ secrets.VAULT_ROLE_ID }} - secretId: ${{ secrets.VAULT_SECRET_ID }} - channel: ${{ env.SLACK_CHANNEL }} + bot-token: ${{ secrets.SLACK_BOT_TOKEN }} + channel-id: ${{ env.SLACK_CHANNEL }} message: | :large_yellow_circle: [${{ github.repository }}] Release *${{ github.ref_name }}* could not be published. Build: (<${{ env.JOB_URL }}|here>) @@ -98,7 +130,13 @@ jobs: outputs: agent-version: ${{ steps.bootstrap.outputs.agent-version }} major-version: ${{ steps.bootstrap.outputs.major-version }} - + env: + PREFIX_ZIP_FILE: "build/output/elastic_apm_profiler_" + SUFFIX_ZIP_FILE: "-win-x64.zip" + permissions: + attestations: write + contents: write + id-token: write steps: - uses: actions/checkout@v4 - name: Bootstrap Action Workspace @@ -110,12 +148,16 @@ jobs: - name: Build profiler run: ./build.bat profiler-zip + - name: generate build provenance (APM Profiler) + uses: actions/attest-build-provenance@bdd51370e0416ac948727f861e03c2f05d32d78e # v1.3.2 + with: + subject-path: "${{ github.workspace }}/${{ env.PREFIX_ZIP_FILE }}${{ steps.bootstrap.outputs.agent-version }}${{ env.SUFFIX_ZIP_FILE }}" + - name: Attach Profiler env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - continue-on-error: true #continue for now until we see it working in action run: | - gh release upload ${{ github.ref_name }} "build/output/elastic_apm_profiler_${{ steps.bootstrap.outputs.agent-version }}-win-x64.zip" + gh release upload ${{ github.ref_name }} "${{ env.PREFIX_ZIP_FILE }}${{ steps.bootstrap.outputs.agent-version }}${{ env.SUFFIX_ZIP_FILE }}" post-release: needs: [ 'release-windows'] diff --git a/.github/workflows/snapshoty.yml b/.github/workflows/snapshoty.yml deleted file mode 100644 index d16216ba3..000000000 --- a/.github/workflows/snapshoty.yml +++ /dev/null @@ -1,43 +0,0 @@ ---- -# Publish a snapshot. A "snapshot" is a packaging of the latest *unreleased* APM agent, -# published to a known GCS bucket for use in edge demo/test environments. -name: snapshoty - -on: - workflow_run: - workflows: ['test-linux'] - types: - - completed - branches: - - main - -permissions: - contents: read - -jobs: - upload: - runs-on: ubuntu-latest - if: ${{ github.event.workflow_run.conclusion == 'success' }} - steps: - - uses: actions/checkout@v4 - - # used by opbeans .NET to reference the on commit nuget packages - # TODO: update opbeans to use our feedz.io packages - - name: Retrieve the artifact uploaded from `test-linux.yml` workflow - uses: actions/download-artifact@v4 - with: - name: snapshoty-linux - path: build/output - github-token: ${{ secrets.GITHUB_TOKEN }} - run-id: ${{ github.event.workflow_run.id }} - - - name: Display structure of downloaded files - run: find build -type f - - - name: Publish snaphosts - uses: elastic/apm-pipeline-library/.github/actions/snapshoty-simple@current - with: - config: '.ci/snapshoty.yml' - vaultUrl: ${{ secrets.VAULT_ADDR }} - vaultRoleId: ${{ secrets.VAULT_ROLE_ID }} - vaultSecretId: ${{ secrets.VAULT_SECRET_ID }} diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index 837f7607f..4cde21905 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -18,16 +18,38 @@ permissions: contents: read concurrency: - group: ${{ github.workflow }}-${{ github.ref }} + group: '${{ github.workflow }}-${{ github.ref }}' cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +defaults: + run: + shell: bash env: NUGET_PACKAGES: ${{ github.workspace }}/.nuget/packages + +# 'pack' & 'tests' are required checks in this workflow. + +# To not burn unneeded CI cycles: +# - Our required checks will always succeed if doc only changes are detected. +# - all jobs depend on 'format' to not waste cycles on quickly fixable errors. + jobs: + + format: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Bootstrap Action Workspace + uses: ./.github/workflows/bootstrap + - name: Format + run: ./build.sh format + + #required step pack: runs-on: ubuntu-latest - + needs: [ 'format' ] steps: - uses: actions/checkout@v4 - name: Bootstrap Action Workspace @@ -35,42 +57,38 @@ jobs: with: rust: 'true' - - name: Format - run: ./build.sh format - - name: Package run: ./build.sh pack - - - uses: actions/upload-artifact@v4 - if: github.event_name == 'push' && startswith(github.ref, 'refs/heads') - with: - name: snapshoty-linux - path: build/output/* - retention-days: 1 + #required step tests: runs-on: ubuntu-latest + needs: [ 'format' ] timeout-minutes: 30 steps: - uses: actions/checkout@v4 - name: Bootstrap Action Workspace uses: ./.github/workflows/bootstrap - + - name: 'Tests: Unit' - shell: bash run: ./build.sh test --test-suite unit - - name: 'Tests: Integrations' - shell: bash - run: ./build.sh test --test-suite integrations - azure-tests: runs-on: ubuntu-latest - if: ${{ false }} - #if: | - # github.event_name != 'pull_request' - # || github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false + needs: [ 'format', 'tests' ] + if: | + github.event_name != 'pull_request' + || github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false + env: + ARM_CLIENT_ID: ${{ secrets.ARM_CLIENT_ID }} + ARM_SUBSCRIPTION_ID: ${{ secrets.ARM_SUBSCRIPTION_ID }} + ARM_TENANT_ID: ${{ secrets.ARM_TENANT_ID }} + ARM_USE_OIDC: true + AZURE_RESOURCE_GROUP_PREFIX: ci-dotnet-${{ github.run_id }} + permissions: + contents: read + id-token: write steps: - uses: actions/checkout@v4 - name: Bootstrap Action Workspace @@ -78,51 +96,48 @@ jobs: with: azure: 'true' - - name: 'Read credentials' - uses: hashicorp/vault-action@v3.0.0 + - name: 'Az CLI login' + uses: azure/login@6c251865b4e6290e7b78be643ea2d005bc51f69a # v2.1.1 with: - url: ${{ secrets.VAULT_ADDR }} - roleId: ${{ secrets.VAULT_ROLE_ID }} - secretId: ${{ secrets.VAULT_SECRET_ID }} - method: approle - secrets: | - secret/apm-team/ci/apm-agent-dotnet-azure client_id | ARM_CLIENT_ID ; - secret/apm-team/ci/apm-agent-dotnet-azure client_secret | ARM_CLIENT_SECRET ; - secret/apm-team/ci/apm-agent-dotnet-azure subscription_id | ARM_SUBSCRIPTION_ID ; - secret/apm-team/ci/apm-agent-dotnet-azure tenant_id | ARM_TENANT_ID - - - name: 'Login to Azure' - run: | - az login --service-principal --username ${{ env.ARM_CLIENT_ID }} --password ${{ env.ARM_CLIENT_SECRET }} --tenant ${{ env.ARM_TENANT_ID }} - az account set --subscription ${{ env.ARM_SUBSCRIPTION_ID }} - echo "AZURE_RESOURCE_GROUP_PREFIX=ci-dotnet-${GITHUB_RUN_ID}" >> ${GITHUB_ENV} + client-id: ${{ secrets.ARM_CLIENT_ID }} + tenant-id: ${{ secrets.ARM_TENANT_ID }} + subscription-id: ${{ secrets.ARM_SUBSCRIPTION_ID }} - name: 'Tests: Azure' - shell: bash run: ./build.sh test --test-suite azure - + - name: 'Teardown tests infra' if: ${{ always() }} run: | for group in $(az group list --query "[?name | starts_with(@,'${{ env.AZURE_RESOURCE_GROUP_PREFIX }}')]" --out json | jq .[].name --raw-output); do az group delete --name "${group}" --no-wait --yes done + + integration-tests: + runs-on: ubuntu-latest + needs: [ 'format', 'tests' ] + steps: + - uses: actions/checkout@v4 + - name: Bootstrap Action Workspace + uses: ./.github/workflows/bootstrap + - name: 'Tests: Integrations' + run: ./build.sh test --test-suite integrations + startup-hook-tests: runs-on: ubuntu-latest - + needs: [ 'format', 'tests' ] steps: - uses: actions/checkout@v4 - name: Bootstrap Action Workspace uses: ./.github/workflows/bootstrap - name: 'Tests: StartupHooks' - shell: bash run: ./build.sh test --test-suite startuphooks profiler-tests: runs-on: ubuntu-latest - + needs: [ 'format', 'tests' ] steps: - uses: actions/checkout@v4 - name: Bootstrap Action Workspace @@ -132,7 +147,6 @@ jobs: rust: 'true' - name: 'Tests: Profiler' - shell: bash run: ./build.sh test --test-suite profiler - name: Build Profiler Docker Image diff --git a/.github/workflows/test-windows-iis.yml b/.github/workflows/test-windows-iis.yml deleted file mode 100644 index cc799068e..000000000 --- a/.github/workflows/test-windows-iis.yml +++ /dev/null @@ -1,110 +0,0 @@ -name: test-windows-iis - -on: - push: - branches: - - main - - 1.* - paths-ignore: - - '*.md' - - '*.asciidoc' - - 'docs/**' - pull_request: - paths-ignore: - - '*.md' - - '*.asciidoc' - - 'docs/**' - -permissions: - contents: read - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} - -env: - ELASTIC_STACK_VERSION: '8.4.0' - NUGET_PACKAGES: ${{ github.workspace }}/.nuget/packages - -jobs: - - test-iis: - runs-on: windows-latest - - steps: - - uses: actions/checkout@v4 - - name: Bootstrap Action Workspace - uses: ./.github/workflows/bootstrap - - - uses: actions/cache@v4 - with: - path: ~/.nuget/packages - key: ${{ runner.os }}-nuget-${{ hashFiles('**/*.[cf]sproj*') }} - - - name: Add msbuild to PATH - uses: microsoft/setup-msbuild@v2 - - - - name: Clean the application - shell: cmd - run: msbuild /t:Clean /p:Configuration=Release - - - name: Restore the application - shell: cmd - run: msbuild /t:Restore /p:Configuration=Release - - - name: Build the application - shell: cmd - run: | - set INCLUDE_CSHARP_TARGETS=true - msbuild ^ - /p:EnforceCodeStyleInBuild=false /p:_SkipUpgradeNetAnalyzersNuGetWarning=true /p:EnableNETAnalyzers=false ^ - -clp:ForceConsoleColor -clp:Summary -verbosity:minimal ^ - /t:Build /p:Configuration=Release /restore - - #- name: Discover Windows Features - # shell: cmd - # run: | - # DISM /online /get-features /format:table - - # TODO See if this really needed - - name: Enable Windows Features - shell: cmd - run: | - DISM /online /enable-feature /featurename:IIS-HttpErrors - DISM /online /enable-feature /featurename:IIS-HttpRedirect - - - name: Ensure AppPool Permissions - shell: cmd - run: | - REM enable permissions for the Application Pool Identity group - icacls C:\Windows\Temp /grant "IIS_IUSRS:(OI)(CI)F" /T - icacls "c:\Windows\Microsoft.NET\Framework64\v4.0.30319\Temporary ASP.NET Files" /grant "IIS_IUSRS:(OI)(CI)F" /T - icacls %cd% /t /q /grant "IIS_IUSRS:(OI)(CI)(IO)(RX)" - REM enable permissions for the anonymous access group - icacls %cd% /t /q /grant "IUSR:(OI)(CI)(IO)(RX)" - icacls C:\Windows\Temp /grant "IUSR:(OI)(CI)F" /T - icacls "c:\Windows\Microsoft.NET\Framework64\v4.0.30319\Temporary ASP.NET Files" /grant "IUSR:(OI)(CI)F" /T - - - name: Run tests - shell: cmd - run: | - set ELASTIC_APM_TESTS_FULL_FRAMEWORK_ENABLED=true - set sample_app_log_dir=C:\Elastic_APM_TEMP - if not exist "%sample_app_log_dir%" mkdir "%sample_app_log_dir%" - icacls %sample_app_log_dir% /t /q /grant Everyone:F - set ELASTIC_APM_ASP_NET_FULL_FRAMEWORK_SAMPLE_APP_LOG_FILE=%sample_app_log_dir%\Elastic.Apm.AspNetFullFramework.Tests.SampleApp.log - - dotnet test -c Release test\iis\Elastic.Apm.AspNetFullFramework.Tests --no-build ^ - --verbosity normal ^ - --results-directory build/output ^ - --diag build/output/diag-iis.log ^ - --filter "FullyQualifiedName=Elastic.Apm.AspNetFullFramework.Tests.CustomServiceNodeNameSetViaSettings.Test" ^ - --logger:"junit;LogFilePath=%cd%\build\output\junit-{framework}-{assembly}.xml;MethodFormat=Class;FailureBodyFormat=Verbose" - - - name: Store test results - if: success() || failure() - uses: actions/upload-artifact@v3 - with: - name: test-results-iis - path: build/output/junit-*.xml \ No newline at end of file diff --git a/.github/workflows/test-windows.yml b/.github/workflows/test-windows.yml index b7cadf4f6..64ed6fbe5 100644 --- a/.github/workflows/test-windows.yml +++ b/.github/workflows/test-windows.yml @@ -18,7 +18,7 @@ permissions: contents: read concurrency: - group: ${{ github.workflow }}-${{ github.ref }} + group: '${{ github.workflow }}-${{ github.ref }}' cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} defaults: @@ -28,32 +28,56 @@ defaults: env: NUGET_PACKAGES: ${{ github.workspace }}/.nuget/packages +# 'tests' is a required check in this workflow. + +# To not burn unneeded CI cycles: +# - Our required checks will always succeed if doc only changes are detected. +# - all jobs depend on 'format' to not waste cycles on quickly fixable errors. + jobs: + + format: + runs-on: windows-2022 + steps: + - uses: actions/checkout@v4 + - name: Bootstrap Action Workspace + uses: ./.github/workflows/bootstrap + - name: Format + run: ./build.bat format + + #required step tests: runs-on: windows-2022 + needs: [ 'format' ] timeout-minutes: 30 steps: - uses: actions/checkout@v4 - name: Bootstrap Action Workspace uses: ./.github/workflows/bootstrap - - - name: Setup Testcontainers Cloud Client - if: github.event_name != 'pull_request' || github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false - uses: atomicjar/testcontainers-cloud-setup-action@v1 - with: - token: ${{ secrets.TC_CLOUD_TOKEN }} + with: + tc-cloud: ${{ secrets.TC_CLOUD_TOKEN }} - name: 'Tests: Unit' run: ./build.bat test --test-suite unit - name: 'Tests: Integrations' - shell: bash run: ./build.bat test --test-suite integrations + integrations-tests: + runs-on: windows-2022 + needs: [ 'format', 'tests' ] + steps: + - uses: actions/checkout@v4 + - name: Bootstrap Action Workspace + uses: ./.github/workflows/bootstrap + + - name: 'Tests: Integrations' + run: ./build.bat test --test-suite integrations + startup-hook-tests: runs-on: windows-2022 - + needs: [ 'format', 'tests' ] steps: - uses: actions/checkout@v4 - name: Bootstrap Action Workspace @@ -64,20 +88,89 @@ jobs: profiler-tests: runs-on: windows-2022 - # Disable profiler tests for now - if: ${{ false }} + needs: [ 'format', 'tests' ] steps: - uses: actions/checkout@v4 - name: Bootstrap Action Workspace uses: ./.github/workflows/bootstrap with: rust: 'true' + tc-cloud: ${{ secrets.TC_CLOUD_TOKEN }} + + - name: 'Tests: Profiler' + run: ./build.bat test --test-suite profiler + + test-iis: + runs-on: windows-latest + needs: [ 'format', 'tests' ] + + steps: + - uses: actions/checkout@v4 + - name: Bootstrap Action Workspace + uses: ./.github/workflows/bootstrap - - name: Setup Testcontainers Cloud Client - if: github.event_name != 'pull_request' || github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false - uses: atomicjar/testcontainers-cloud-setup-action@v1 + - uses: actions/cache@v4 with: - token: ${{ secrets.TC_CLOUD_TOKEN }} + path: ~/.nuget/packages + key: "${{ runner.os }}-nuget-${{ hashFiles('**/*.[cf]sproj*') }}" - - name: 'Tests: Profiler' - run: ./build.bat test --test-suite profiler \ No newline at end of file + - name: Add msbuild to PATH + uses: microsoft/setup-msbuild@6fb02220983dee41ce7ae257b6f4d8f9bf5ed4ce # v2 + + + - name: Clean the application + run: msbuild /t:Clean /p:Configuration=Release + + - name: Restore the application + run: msbuild /t:Restore /p:Configuration=Release + + - name: Build the application + run: | + set INCLUDE_CSHARP_TARGETS=true + msbuild ^ + /p:EnforceCodeStyleInBuild=false /p:_SkipUpgradeNetAnalyzersNuGetWarning=true /p:EnableNETAnalyzers=false ^ + -clp:ForceConsoleColor -clp:Summary -verbosity:minimal ^ + /t:Build /p:Configuration=Release /restore + + #- name: Discover Windows Features + # run: | + # DISM /online /get-features /format:table + + # TODO See if this really needed + - name: Enable Windows Features + run: | + DISM /online /enable-feature /featurename:IIS-HttpErrors + DISM /online /enable-feature /featurename:IIS-HttpRedirect + + - name: Ensure AppPool Permissions + run: | + REM enable permissions for the Application Pool Identity group + icacls C:\Windows\Temp /grant "IIS_IUSRS:(OI)(CI)F" /T + icacls "c:\Windows\Microsoft.NET\Framework64\v4.0.30319\Temporary ASP.NET Files" /grant "IIS_IUSRS:(OI)(CI)F" /T + icacls %cd% /t /q /grant "IIS_IUSRS:(OI)(CI)(IO)(RX)" + REM enable permissions for the anonymous access group + icacls %cd% /t /q /grant "IUSR:(OI)(CI)(IO)(RX)" + icacls C:\Windows\Temp /grant "IUSR:(OI)(CI)F" /T + icacls "c:\Windows\Microsoft.NET\Framework64\v4.0.30319\Temporary ASP.NET Files" /grant "IUSR:(OI)(CI)F" /T + + - name: Run tests + run: | + set ELASTIC_APM_TESTS_FULL_FRAMEWORK_ENABLED=true + set sample_app_log_dir=C:\Elastic_APM_TEMP + if not exist "%sample_app_log_dir%" mkdir "%sample_app_log_dir%" + icacls %sample_app_log_dir% /t /q /grant Everyone:F + set ELASTIC_APM_ASP_NET_FULL_FRAMEWORK_SAMPLE_APP_LOG_FILE=%sample_app_log_dir%\Elastic.Apm.AspNetFullFramework.Tests.SampleApp.log + + dotnet test -c Release test\iis\Elastic.Apm.AspNetFullFramework.Tests --no-build ^ + --verbosity normal ^ + --results-directory build/output ^ + --diag build/output/diag-iis.log ^ + --filter "FullyQualifiedName=Elastic.Apm.AspNetFullFramework.Tests.CustomServiceNodeNameSetViaSettings.Test" ^ + --logger:"junit;LogFilePath=%cd%\build\output\junit-{framework}-{assembly}.xml;MethodFormat=Class;FailureBodyFormat=Verbose" + + - name: Store test results + if: success() || failure() + uses: actions/upload-artifact@v3 + with: + name: test-results-iis + path: build/output/junit-*.xml diff --git a/.github/workflows/updatecli.yml b/.github/workflows/updatecli.yml index 3c667994c..7103377de 100644 --- a/.github/workflows/updatecli.yml +++ b/.github/workflows/updatecli.yml @@ -9,14 +9,29 @@ permissions: contents: read jobs: - bump: + compose: runs-on: ubuntu-latest + permissions: + contents: read + packages: read steps: - uses: actions/checkout@v4 + - uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3.2.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - uses: elastic/oblt-actions/updatecli/run@v1 + with: + command: --experimental compose diff + env: + GITHUB_TOKEN: ${{ secrets.UPDATECLI_GH_TOKEN }} + - uses: elastic/oblt-actions/updatecli/run@v1 with: - command: "--experimental apply --config .ci/updatecli/updatecli.d --values .ci/updatecli/values.yml" + command: --experimental compose apply env: GITHUB_TOKEN: ${{ secrets.UPDATECLI_GH_TOKEN }} diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 52f284723..ad9e0e69e 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -23,12 +23,38 @@ endif::[] [[release-notes-1.x]] === .NET Agent version 1.x +[[release-notes-1.28.0]] +==== 1.28.0 - 2024/07/03 + +===== Bug fixes + +{pull}2389[#2389] Fix Linux build dependency of glibc + +[[release-notes-1.27.3]] +==== 1.27.3 - 2024/06/18 + +===== Bug fixes + +{pull}2380[#2380] Release Automation fix + +[[release-notes-1.27.2]] +==== 1.27.2 - 2024/06/18 + +===== Bug fixes + +{pull}2308[#2308] Clean up dependency graph for .NET core installations +{pull}2356[#2356] Open Telemetry Bridge should only log when enabled +{pull}2166[#2166] Bump Microsoft.AspNetCore.Http dep to 2.1.22 +{pull}2350[#2350] Fix message format for logging in managed profiler +{pull}2225[#2225] Only mark bodies as redacted if explicitly configured to do so. +{pull}2377[#2377] Do not read claims from SqlRoleProvider under classic ASP.NET + [[release-notes-1.27.1]] ==== 1.27.1 - 2024/05/16 ===== Bug fixes -* Remove invalid profiler method integrations by @stevejgordon in https://github.com/elastic/apm-agent-dotnet/pull/2349 +{pull}2349[#2349] Remove invalid profiler method integrations [[release-notes-1.27.0]] ==== 1.27.0 - 2024/04/30 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b3a24839d..ce9e9c06c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -15,18 +15,22 @@ feedback and ideas are always welcome. ### .NET source In order to build the .NET source code, you'll need - * [.NET 6.0 or later](https://dotnet.microsoft.com/download/dotnet/6.0) + * [.NET 8.0 or later](https://dotnet.microsoft.com/download/dotnet/8.0) * **If** you're running on Windows **and** also wish to build projects that target .NET Framework, -you'll need a minimum of .NET Framework 4.6.1 installed. +you'll need a minimum of .NET Framework 4.6.2 installed. You can use any IDE that supports .NET development, and you can use any OS that is supported by .NET. ### Rust source -In order to build the CLR profiler source code, you'll need - * [Rust 1.54 or later](https://www.rust-lang.org/tools/install) +In order to build the CLR profiler source code, you'll need: + * [Rust 1.79 or later](https://www.rust-lang.org/tools/install) * [Cargo make](https://github.com/sagiegurari/cargo-make#installation) + On Linux, you will also require: + * [Cargo zigbuild](https://github.com/rust-cross/cargo-zigbuild) + * [Zig](https://github.com/ziglang/zig) + You can use any IDE that supports Rust development; we typically use [CLion](https://www.jetbrains.com/clion/) with the [Rust plugin](https://plugins.jetbrains.com/plugin/8182-rust/docs), or [VS Code](https://code.visualstudio.com/) diff --git a/Makefile.toml b/Makefile.toml index 20bcfe1ae..539cac4c1 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -35,6 +35,15 @@ description = "Builds CLR Profiler for release" # loader assembly is embedded in the profiler dependencies = ["build-loader"] +[tasks.build-release-linux] +description = "Builds CLR Profiler for release against a known version of glibc" +install_crate = false +toolchain = "${CARGO_MAKE_RUST_DEFAULT_TOOLCHAIN}" +command = "cargo" +args = ["zigbuild", "--release", "--target", "x86_64-unknown-linux-gnu.2.14", "@@split(CARGO_MAKE_CARGO_BUILD_TEST_FLAGS, )"] +# loader assembly is embedded in the profiler +dependencies = ["build-loader"] + [tasks.build-integrations] description = "Builds Managed Profiler .NET integrations" command = "dotnet" @@ -83,7 +92,7 @@ cargo expand --manifest-path src/profiler/elastic_apm_profiler/Cargo.toml --colo [tasks.set-profiler-env] private = true -env = { "CORECLR_PROFILER_PATH" = "${CARGO_MAKE_WORKING_DIRECTORY}/target/release/libelastic_apm_profiler.so" } +env = { "CORECLR_PROFILER_PATH" = "${CARGO_MAKE_WORKING_DIRECTORY}/target/x86_64-unknown-linux-gnu/release/libelastic_apm_profiler.so" } [tasks.set-profiler-env.mac] env = { "CORECLR_PROFILER_PATH" = "${CARGO_MAKE_WORKING_DIRECTORY}/target/release/libelastic_apm_profiler.dylib" } diff --git a/build/scripts/Build.fs b/build/scripts/Build.fs index 9a5982294..e6f9954fb 100644 --- a/build/scripts/Build.fs +++ b/build/scripts/Build.fs @@ -164,8 +164,8 @@ module Build = let logger = match BuildServer.isGitHubActionsBuild with - | true -> Some "--logger:\"GitHubActions;summary.includePassedTests=false\"" - | fase -> None + | true -> Some "--logger:\"GitHubActions;summary.includePassedTests=false;summary.includeNotFoundTests=false\"" + | _ -> None let filter = match suite with @@ -183,22 +183,23 @@ module Build = @ (match filter with None -> [] | Some f -> ["--filter"; f]) @ (match framework with None -> [] | Some f -> ["-f"; f]) @ (match logger with None -> [] | Some l -> [l]) + @ ["--"; "RunConfiguration.CollectSourceInformation=true"] DotNet.ExecWithTimeout command (TimeSpan.FromMinutes 30) - - + /// Builds the CLR profiler and supporting .NET managed assemblies let BuildProfiler () = dotnet "build" (Paths.ProfilerProjFile "Elastic.Apm.Profiler.Managed") - Cargo.Exec [ "make"; "build-release"; "--disable-check-for-update"] - + if isWindows then Cargo.Exec [ "make"; "build-release"; "--disable-check-for-update"] + else Cargo.Exec [ "make"; "build-release-linux"; "--disable-check-for-update"] + /// Publishes all projects with framework versions - let Publish targets = + let Publish targets = let projs = match targets with | Some t -> t | None -> allSrcProjects - + projs |> Seq.map getAllTargetFrameworks |> Seq.iter (fun (proj, frameworks) -> @@ -289,7 +290,6 @@ module Build = // include version in the zip file name ZipFile.CreateFromDirectory(agentDir.FullName, Paths.BuildOutput versionedName + ".zip") - let ProfilerIntegrations () = DotNet.Exec ["run"; "--project"; Paths.ProfilerProjFile "Elastic.Apm.Profiler.IntegrationsGenerator"; "--" "-i"; Paths.SrcProfiler "Elastic.Apm.Profiler.Managed/bin/Release/netstandard2.0/Elastic.Apm.Profiler.Managed.dll" @@ -298,20 +298,25 @@ module Build = /// Creates versioned elastic_apm_profiler.zip file containing all components needed for profiler auto-instrumentation let ProfilerZip () = let name = "elastic_apm_profiler" + let directory = Paths.BuildOutput name + + if Directory.Exists(directory) then + Directory.Delete(directory, true) + let currentAssemblyVersion = Versioning.CurrentVersion.FileVersion let versionedName = let os = if RuntimeInformation.IsOSPlatform(OSPlatform.Windows) then "win-x64" - else "linux-x64" + else "linux-x64" sprintf "%s_%s-%s" name (currentAssemblyVersion.ToString()) os - let profilerDir = Paths.BuildOutput name |> DirectoryInfo + let profilerDir = directory |> DirectoryInfo profilerDir.Create() - + seq { Paths.SrcProfiler "Elastic.Apm.Profiler.Managed/integrations.yml" "target/release/elastic_apm_profiler.dll" - "target/release/libelastic_apm_profiler.so" + "target/x86_64-unknown-linux-gnu/release/libelastic_apm_profiler.so" Paths.SrcProfiler "elastic_apm_profiler/NOTICE" Paths.SrcProfiler "elastic_apm_profiler/README" "LICENSE" @@ -320,12 +325,13 @@ module Build = |> Seq.filter (fun file -> file.Exists) |> Seq.iter (fun file -> let destination = Path.combine profilerDir.FullName file.Name - let newFile = file.CopyTo(destination, true) + let newFile = file.CopyTo(destination, true) if newFile.Name = "README" then File.applyReplace (fun s -> s.Replace("${VERSION}", sprintf "%i.%i" currentAssemblyVersion.Major currentAssemblyVersion.Minor)) newFile.FullName ) - + Directory.GetDirectories((Paths.BuildOutput "Elastic.Apm.Profiler.Managed"), "*", SearchOption.TopDirectoryOnly) + |> Array.filter (fun dir -> isWindows || not (dir.EndsWith("net462"))) |> Seq.map DirectoryInfo |> Seq.iter (fun sourceDir -> copyDllsAndPdbs (profilerDir.CreateSubdirectory(sourceDir.Name)) sourceDir) @@ -334,8 +340,4 @@ module Build = if File.exists zip then printf $"%s{zip} already exists on disk" File.delete zip - ZipFile.CreateFromDirectory(profilerDir.FullName, zip) - - - - \ No newline at end of file + ZipFile.CreateFromDirectory(profilerDir.FullName, zip) \ No newline at end of file diff --git a/docs/setup-auto-instrumentation.asciidoc b/docs/setup-auto-instrumentation.asciidoc index 413acfbbb..9bff72f74 100644 --- a/docs/setup-auto-instrumentation.asciidoc +++ b/docs/setup-auto-instrumentation.asciidoc @@ -16,7 +16,7 @@ This approach works with the following |=== | 2.+^|**Operating system** -|**Architecture** |**Windows** |**Linux** +|**Architecture** |**Windows** |**Linux** ** |x64 @@ -33,7 +33,11 @@ This approach works with the following _* Due to binding issues introduced by Microsoft, we recommend at least .NET Framework 4.7.2 for best compatibility._ -NOTE: The Profiler based agent only supports 64-bit applications. 32-bit applications aren't supported. +_** Minimum GLIBC version 2.14._ + +NOTE: The profiler-based agent only supports 64-bit applications. 32-bit applications aren't supported. + +NOTE: The profiler-based agent does not currently support ARM. It instruments the following assemblies: diff --git a/src/Elastic.Apm/AgentComponents.cs b/src/Elastic.Apm/AgentComponents.cs index 0f4af390e..ded9d166f 100644 --- a/src/Elastic.Apm/AgentComponents.cs +++ b/src/Elastic.Apm/AgentComponents.cs @@ -43,12 +43,14 @@ internal AgentComponents( ICurrentExecutionSegmentsContainer currentExecutionSegmentsContainer, ICentralConfigurationFetcher centralConfigurationFetcher, IApmServerInfo apmServerInfo, - BreakdownMetricsProvider breakdownMetricsProvider = null + BreakdownMetricsProvider breakdownMetricsProvider = null, + IHostNameDetector hostNameDetector = null ) { try { var config = CreateConfiguration(logger, configurationReader); + hostNameDetector ??= new HostNameDetector(); Logger = logger ?? GetGlobalLogger(DefaultLogger(null, configurationReader), config.LogLevel); ConfigurationStore = new ConfigurationStore(new RuntimeConfigurationSnapshot(config), Logger); @@ -64,7 +66,7 @@ internal AgentComponents( ElasticActivityListener = new ElasticActivityListener(this, HttpTraceConfiguration); #endif var systemInfoHelper = new SystemInfoHelper(Logger); - var system = systemInfoHelper.GetSystemInfo(Configuration.HostName); + var system = systemInfoHelper.GetSystemInfo(Configuration.HostName, hostNameDetector); PayloadSender = payloadSender ?? new PayloadSenderV2(Logger, ConfigurationStore.CurrentSnapshot, Service, system, diff --git a/src/Elastic.Apm/Extensions/TransactionExtensions.cs b/src/Elastic.Apm/Extensions/TransactionExtensions.cs index f40038a8d..ef2127eba 100644 --- a/src/Elastic.Apm/Extensions/TransactionExtensions.cs +++ b/src/Elastic.Apm/Extensions/TransactionExtensions.cs @@ -33,17 +33,20 @@ internal static void CollectRequestBody(this ITransaction transaction, bool isFo // Is request body already captured? // We check transaction.IsContextCreated to avoid creating empty Context (that accessing transaction.Context directly would have done). - var hasContext = (transaction is Transaction t && t.IsContextCreated) || transaction.Context != null; + var hasContext = transaction is Transaction { IsContextCreated: true } || transaction.Context != null; if (hasContext && transaction.Context.Request.Body != null && !ReferenceEquals(transaction.Context.Request.Body, Consts.Redacted)) return; + if (transaction.Configuration.CaptureBody.Equals(ConfigConsts.SupportedValues.CaptureBodyOff)) + body = Consts.Redacted; + if (transaction.IsCaptureRequestBodyEnabled(isForError) && IsCaptureRequestBodyEnabledForContentType(transaction, httpRequest?.ContentType, logger)) body = httpRequest.ExtractBody(logger, transaction.Configuration); - // According to the documentation - the default value of 'body' is '[Redacted]' - transaction.Context.Request.Body = body ?? Consts.Redacted; + if (transaction.Context != null) + transaction.Context.Request.Body = body; } internal static bool IsCaptureRequestBodyEnabled(this ITransaction transaction, bool isForError) => diff --git a/src/Elastic.Apm/Helpers/HostNameDetector.cs b/src/Elastic.Apm/Helpers/HostNameDetector.cs new file mode 100644 index 000000000..a7aeb9ed6 --- /dev/null +++ b/src/Elastic.Apm/Helpers/HostNameDetector.cs @@ -0,0 +1,91 @@ +// Licensed to Elasticsearch B.V under +// one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System; +using System.Net; +using System.Net.NetworkInformation; +using Elastic.Apm.Logging; + +namespace Elastic.Apm.Helpers; + +internal interface IHostNameDetector +{ + string GetDetectedHostName(IApmLogger logger); +} +public class HostNameDetector : IHostNameDetector +{ + public string GetDetectedHostName(IApmLogger logger) + { + var fqdn = string.Empty; + + try + { + fqdn = Dns.GetHostEntry(string.Empty).HostName; + } + catch (Exception e) + { + logger.Warning()?.LogException(e, "Failed to get hostname via Dns.GetHostEntry(string.Empty).HostName."); + } + + if (!string.IsNullOrEmpty(fqdn)) + return NormalizeHostName(fqdn); + + try + { + var hostName = IPGlobalProperties.GetIPGlobalProperties().HostName; + var domainName = IPGlobalProperties.GetIPGlobalProperties().DomainName; + + if (!string.IsNullOrEmpty(domainName)) + { + hostName = $"{hostName}.{domainName}"; + } + + fqdn = hostName; + + } + catch (Exception e) + { + logger.Warning()?.LogException(e, "Failed to get hostname via IPGlobalProperties.GetIPGlobalProperties()."); + } + + if (!string.IsNullOrEmpty(fqdn)) + return NormalizeHostName(fqdn); + + try + { + fqdn = Environment.MachineName; + } + catch (Exception e) + { + logger.Warning()?.LogException(e, "Failed to get hostname via Environment.MachineName."); + } + + if (!string.IsNullOrEmpty(fqdn)) + return NormalizeHostName(fqdn); + + logger.Debug()?.Log("Falling back to environment variables to get hostname."); + + try + { + fqdn = (Environment.GetEnvironmentVariable("COMPUTERNAME") + ?? Environment.GetEnvironmentVariable("HOSTNAME")) + ?? Environment.GetEnvironmentVariable("HOST"); + + if (string.IsNullOrEmpty(fqdn)) + logger.Error()?.Log("Failed to get hostname via environment variables."); + + return NormalizeHostName(fqdn); + } + catch (Exception e) + { + logger.Error()?.LogException(e, "Failed to get hostname."); + } + + return null; + + static string NormalizeHostName(string hostName) => + string.IsNullOrEmpty(hostName) ? null : hostName.Trim().ToLower(); + } +} diff --git a/src/Elastic.Apm/Helpers/SystemInfoHelper.cs b/src/Elastic.Apm/Helpers/SystemInfoHelper.cs index b5647da9b..c737dbb11 100644 --- a/src/Elastic.Apm/Helpers/SystemInfoHelper.cs +++ b/src/Elastic.Apm/Helpers/SystemInfoHelper.cs @@ -5,8 +5,7 @@ using System; using System.Data.Common; using System.IO; -using System.Net; -using System.Net.NetworkInformation; +using System.Linq; using System.Text.RegularExpressions; using Elastic.Apm.Api; using Elastic.Apm.Api.Kubernetes; @@ -28,6 +27,28 @@ internal class SystemInfoHelper public SystemInfoHelper(IApmLogger logger) => _logger = logger.Scoped(nameof(SystemInfoHelper)); + + //3997 3984 253:1 /var/lib/docker/containers/6548c6863fb748e72d1e2a4f824fde92f720952d062dede1318c2d6219a672d6/hostname /etc/hostname rw,relatime shared:1877 - ext4 /dev/mapper/vgubuntu-root rw,errors=remount-ro + internal void ParseMountInfo(Api.System system, string reportedHostName, string line) + { + + var fields = line.Split(' '); + if (fields.Length <= 3) + return; + + var path = fields[3]; + foreach (var folder in path.Split('/')) + { + //naive implementation to check for guid. + if (folder.Length != 64) + continue; + system.Container = new Container { Id = folder }; + } + + } + + // "1:name=systemd:/ecs/03752a671e744971a862edcee6195646/03752a671e744971a862edcee6195646-4015103728" + // "0::/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod121157b5_c67d_4c3e_9052_cb27bbb711fb.slice/cri-containerd-1cd3449e930b8a28c7595240fa32ba20c84f36d059e5fbe63104ad40057992d1.scope" internal void ParseContainerId(Api.System system, string reportedHostName, string line) { var fields = line.Split(':'); @@ -43,11 +64,13 @@ internal void ParseContainerId(Api.System system, string reportedHostName, strin if (string.IsNullOrWhiteSpace(idPart)) return; - // Legacy, e.g.: /system.slice/docker-.scope + // Legacy, e.g.: /system.slice/docker-.scope or cri-containerd-.scope if (idPart.EndsWith(".scope")) { - idPart = idPart.Substring(0, idPart.Length - ".scope".Length) - .Substring(idPart.IndexOf("-", StringComparison.Ordinal) + 1); + var idParts = idPart.Split(new[] { '-' }, StringSplitOptions.RemoveEmptyEntries); + var containerIdWithScope = idParts.Last(); + + idPart = containerIdWithScope.Substring(0, containerIdWithScope.Length - ".scope".Length); } // Looking for kubernetes info @@ -84,9 +107,9 @@ internal void ParseContainerId(Api.System system, string reportedHostName, strin _logger.Info()?.Log("Could not parse container ID from '/proc/self/cgroup' line: {line}", line); } - internal Api.System GetSystemInfo(string hostName) + internal Api.System GetSystemInfo(string hostName, IHostNameDetector detector) { - var detectedHostName = GetHostName(); + var detectedHostName = detector.GetDetectedHostName(_logger); var system = new Api.System { DetectedHostName = detectedHostName, ConfiguredHostName = hostName }; if (AgentFeaturesProvider.Get(_logger).Check(AgentFeature.ContainerInfo)) @@ -98,83 +121,13 @@ internal Api.System GetSystemInfo(string hostName) return system; } - internal string GetHostName() - { - var fqdn = string.Empty; - - try - { - fqdn = Dns.GetHostEntry(string.Empty).HostName; - } - catch (Exception e) - { - _logger.Warning()?.LogException(e, "Failed to get hostname via Dns.GetHostEntry(string.Empty).HostName."); - } - - if (!string.IsNullOrEmpty(fqdn)) - return NormalizeHostName(fqdn); - - try - { - var hostName = IPGlobalProperties.GetIPGlobalProperties().HostName; - var domainName = IPGlobalProperties.GetIPGlobalProperties().DomainName; - - if (!string.IsNullOrEmpty(domainName)) - { - hostName = $"{hostName}.{domainName}"; - } - - fqdn = hostName; - - } - catch (Exception e) - { - _logger.Warning()?.LogException(e, "Failed to get hostname via IPGlobalProperties.GetIPGlobalProperties()."); - } - - if (!string.IsNullOrEmpty(fqdn)) - return NormalizeHostName(fqdn); - - try - { - fqdn = Environment.MachineName; - } - catch (Exception e) - { - _logger.Warning()?.LogException(e, "Failed to get hostname via Environment.MachineName."); - } - - if (!string.IsNullOrEmpty(fqdn)) - return NormalizeHostName(fqdn); - - _logger.Debug()?.Log("Falling back to environment variables to get hostname."); - - try - { - fqdn = (Environment.GetEnvironmentVariable("COMPUTERNAME") - ?? Environment.GetEnvironmentVariable("HOSTNAME")) - ?? Environment.GetEnvironmentVariable("HOST"); - - if (string.IsNullOrEmpty(fqdn)) - _logger.Error()?.Log("Failed to get hostname via environment variables."); - - return NormalizeHostName(fqdn); - } - catch (Exception e) - { - _logger.Error()?.LogException(e, "Failed to get hostname."); - } - - return null; - - static string NormalizeHostName(string hostName) => - string.IsNullOrEmpty(hostName) ? null : hostName.Trim().ToLower(); - } private void ParseContainerInfo(Api.System system, string reportedHostName) { + //0::/ try { + var fallBackToMountInfo = false; using var sr = GetCGroupAsStream(); if (sr is null) { @@ -183,12 +136,34 @@ private void ParseContainerInfo(Api.System system, string reportedHostName) return; } + var i = 0; string line; while ((line = sr.ReadLine()) != null) { + if (line == "0::/" && i == 0) + fallBackToMountInfo = true; ParseContainerId(system, reportedHostName, line); if (system.Container != null) return; + i++; + } + if (!fallBackToMountInfo) + return; + + using var mi = GetMountInfoAsStream(); + if (mi is null) + { + _logger.Debug()?.Log("No /proc/self/mountinfo found - no information to fallback to"); + return; + } + + while ((line = mi.ReadLine()) != null) + { + if (!line.Contains("/etc/hostname")) + continue; + ParseMountInfo(system, reportedHostName, line); + if (system.Container != null) + return; } } catch (Exception e) @@ -201,8 +176,12 @@ private void ParseContainerInfo(Api.System system, string reportedHostName) "Failed parsing container id - the agent will not report container id. Likely the application is not running within a container"); } - protected virtual StreamReader GetCGroupAsStream() - => File.Exists("/proc/self/cgroup") ? new StreamReader("/proc/self/cgroup") : null; + protected virtual StreamReader GetCGroupAsStream() => + File.Exists("/proc/self/cgroup") ? new StreamReader("/proc/self/cgroup") : null; + + protected virtual StreamReader GetMountInfoAsStream() => + File.Exists("/proc/self/mountinfo") ? new StreamReader("/proc/self/mountinfo") : null; + internal const string Namespace = "KUBERNETES_NAMESPACE"; internal const string PodName = "KUBERNETES_POD_NAME"; diff --git a/src/integrations/Elastic.Apm.AspNetFullFramework/Elastic.Apm.AspNetFullFramework.csproj b/src/integrations/Elastic.Apm.AspNetFullFramework/Elastic.Apm.AspNetFullFramework.csproj index b25a0dce4..e05324f00 100644 --- a/src/integrations/Elastic.Apm.AspNetFullFramework/Elastic.Apm.AspNetFullFramework.csproj +++ b/src/integrations/Elastic.Apm.AspNetFullFramework/Elastic.Apm.AspNetFullFramework.csproj @@ -18,6 +18,7 @@ + diff --git a/src/integrations/Elastic.Apm.AspNetFullFramework/ElasticApmModule.cs b/src/integrations/Elastic.Apm.AspNetFullFramework/ElasticApmModule.cs index ef767aa47..26febdbf6 100644 --- a/src/integrations/Elastic.Apm.AspNetFullFramework/ElasticApmModule.cs +++ b/src/integrations/Elastic.Apm.AspNetFullFramework/ElasticApmModule.cs @@ -631,8 +631,8 @@ private void FillSampledTransactionContextUser(HttpContext context, ITransaction return; var user = new User { UserName = userIdentity.Name }; - - if (context.User is ClaimsPrincipal claimsPrincipal) + var sqlRoleProvider = System.Web.Security.Roles.Providers.Cast().Any(provider => provider.GetType().Name == "SqlRoleProvider"); + if (!sqlRoleProvider && context.User is ClaimsPrincipal claimsPrincipal) { try { diff --git a/src/profiler/elastic_apm_profiler/Cargo.toml b/src/profiler/elastic_apm_profiler/Cargo.toml index 61c594288..b5e1e8551 100644 --- a/src/profiler/elastic_apm_profiler/Cargo.toml +++ b/src/profiler/elastic_apm_profiler/Cargo.toml @@ -14,7 +14,7 @@ c_vec = "2.0.0" com = { version = "0.6.0", features = ["production"] } hex = "0.4.3" log = "0.4.14" -log4rs = { version = "1.0.0", default_features = false, features = ["console_appender", "rolling_file_appender", "compound_policy", "size_trigger", "fixed_window_roller"] } +log4rs = { version = "1.0.0", default-features = false, features = ["console_appender", "rolling_file_appender", "compound_policy", "size_trigger", "fixed_window_roller"] } num-derive = "0.3" num-traits = "0.2" once_cell = "1.8.0" diff --git a/test/Directory.Build.props b/test/Directory.Build.props index 1e0539a9b..03a58c389 100644 --- a/test/Directory.Build.props +++ b/test/Directory.Build.props @@ -22,7 +22,7 @@ - + diff --git a/test/Elastic.Apm.Tests.Utilities/CGroupTestCasesAttribute.cs b/test/Elastic.Apm.Tests.Utilities/CGroupTestCasesAttribute.cs new file mode 100644 index 000000000..e73eaa5f7 --- /dev/null +++ b/test/Elastic.Apm.Tests.Utilities/CGroupTestCasesAttribute.cs @@ -0,0 +1,78 @@ +// Licensed to Elasticsearch B.V under +// one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Reflection; +using Elastic.Apm.Libraries.Newtonsoft.Json.Linq; +using Xunit.Sdk; + +namespace Elastic.Apm.Tests.Utilities; + +public struct CgroupFiles +{ + public string ProcSelfCgroup; + public string[] MountInfo; +} + +public struct CGroupTestData +{ + public CgroupFiles Files; + public string ContainerId; + public string PodId; +} + + +public class CGroupTestCasesAttribute : DataAttribute +{ + private readonly string _fileName = "./TestResources/json-specs//container_metadata_discovery.json"; + + public override IEnumerable GetData(MethodInfo testMethod) + { + if (!File.Exists(_fileName)) + throw new ArgumentException($"JSON input file {_fileName} does not exist"); + + var jToken = JToken.Parse(File.ReadAllText(_fileName), new JsonLoadSettings + { + CommentHandling = CommentHandling.Ignore + }); + + foreach (var kvp in (JObject)jToken) + { + var name = kvp.Key; + var data = ParseTestData(kvp.Value as JObject); + yield return [name, data]; + } + } + + private static CGroupTestData ParseTestData(JObject jToken) + { + var testData = new CGroupTestData { Files = new CgroupFiles() }; + + foreach (var kvp in jToken) + { + switch (kvp.Key) + { + case "containerId": + testData.ContainerId = kvp.Value?.Value(); + break; + case "podId": + testData.PodId = kvp.Value?.Value(); + break; + case "files": + var o = (JObject)kvp.Value; + var cgroupA = o.Property("/proc/self/cgroup")?.Value as JArray; + testData.Files.ProcSelfCgroup = cgroupA?.Values().FirstOrDefault(); + + var mountInfoA = o.Property("/proc/self/mountinfo")?.Value as JArray; + testData.Files.MountInfo = mountInfoA?.Values().ToArray(); + break; + } + } + return testData; + } +} diff --git a/test/Elastic.Apm.Tests.Utilities/Terraform/TerraformResources.cs b/test/Elastic.Apm.Tests.Utilities/Terraform/TerraformResources.cs index 3e21b4d91..91c671141 100644 --- a/test/Elastic.Apm.Tests.Utilities/Terraform/TerraformResources.cs +++ b/test/Elastic.Apm.Tests.Utilities/Terraform/TerraformResources.cs @@ -21,7 +21,7 @@ namespace Elastic.Apm.Tests.Utilities.Terraform /// public class TerraformResources { - private static readonly TimeSpan _defaultTimeout = TimeSpan.FromMinutes(15); + private static readonly TimeSpan _defaultTimeout = TimeSpan.FromMinutes(180); private readonly string _resourceDirectory; private readonly IMessageSink _messageSink; diff --git a/test/Elastic.Apm.Tests.Utilities/TestAgentComponents.cs b/test/Elastic.Apm.Tests.Utilities/TestAgentComponents.cs index e1a8e1a7a..692d99caf 100644 --- a/test/Elastic.Apm.Tests.Utilities/TestAgentComponents.cs +++ b/test/Elastic.Apm.Tests.Utilities/TestAgentComponents.cs @@ -27,7 +27,10 @@ public TestAgentComponents( new FakeMetricsCollector(), currentExecutionSegmentsContainer, centralConfigurationFetcher ?? new NoopCentralConfigurationFetcher(), - apmServerInfo ?? MockApmServerInfo.Version710 + apmServerInfo ?? MockApmServerInfo.Version710, + null, + new TestHostNameDetector(configuration) + ) { } } diff --git a/test/Elastic.Apm.Tests.Utilities/TestHostNameDetector.cs b/test/Elastic.Apm.Tests.Utilities/TestHostNameDetector.cs new file mode 100644 index 000000000..0aff9bbb6 --- /dev/null +++ b/test/Elastic.Apm.Tests.Utilities/TestHostNameDetector.cs @@ -0,0 +1,23 @@ +// Licensed to Elasticsearch B.V under +// one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using Elastic.Apm.Config; +using Elastic.Apm.Helpers; +using Elastic.Apm.Logging; + +namespace Elastic.Apm.Tests.Utilities; + +internal class TestHostNameDetector : IHostNameDetector +{ + private readonly string _hostName; + + public TestHostNameDetector(IConfiguration configuration) => + _hostName = configuration?.HostName ?? "MY_COMPUTER"; + + public TestHostNameDetector(string detectedHostName) => + _hostName = detectedHostName; + + public string GetDetectedHostName(IApmLogger logger) => _hostName; +} diff --git a/test/Elastic.Apm.Tests.Utilities/TestResources/json-specs/cgroup_parsing.json b/test/Elastic.Apm.Tests.Utilities/TestResources/json-specs/cgroup_parsing.json deleted file mode 100644 index f28d87d4c..000000000 --- a/test/Elastic.Apm.Tests.Utilities/TestResources/json-specs/cgroup_parsing.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "testUnderscores": { - "groupLine": "1:name=systemd:/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod90d81341_92de_11e7_8cf2_507b9d4141fa.slice/crio-2227daf62df6694645fee5df53c1f91271546a9560e8600a525690ae252b7f63.scope", - "containerId": "2227daf62df6694645fee5df53c1f91271546a9560e8600a525690ae252b7f63", - "podId": "90d81341-92de-11e7-8cf2-507b9d4141fa" - }, - "testOpenshiftForm": { - "groupLine": "9:freezer:/kubepods.slice/kubepods-pod22949dce_fd8b_11ea_8ede_98f2b32c645c.slice/docker-b15a5bdedd2e7645c3be271364324321b908314e4c77857bbfd32a041148c07f.scope", - "containerId": "b15a5bdedd2e7645c3be271364324321b908314e4c77857bbfd32a041148c07f", - "podId": "22949dce-fd8b-11ea-8ede-98f2b32c645c" - }, - "testUbuntuCGroup": { - "groupLine": "1:name=systemd:/user.slice/user-1000.slice/user@1000.service/apps.slice/apps-org.gnome.Terminal.slice/vte-spawn-75bc72bd-6642-4cf5-b62c-0674e11bfc84.scope", - "containerId": null, - "podId": null - }, - "testAwsEcsCGroup": { - "groupLine": "1:name=systemd:/ecs/03752a671e744971a862edcee6195646/03752a671e744971a862edcee6195646-4015103728", - "containerId": "03752a671e744971a862edcee6195646-4015103728", - "podId": null - } -} diff --git a/test/Elastic.Apm.Tests/BackendCommTests/CentralConfig/CentralConfigFetcherTests.cs b/test/Elastic.Apm.Tests/BackendCommTests/CentralConfig/CentralConfigFetcherTests.cs index abb037297..af12f82d8 100644 --- a/test/Elastic.Apm.Tests/BackendCommTests/CentralConfig/CentralConfigFetcherTests.cs +++ b/test/Elastic.Apm.Tests/BackendCommTests/CentralConfig/CentralConfigFetcherTests.cs @@ -238,7 +238,7 @@ public void Dispose_stops_the_thread() using (var agent = new ApmAgent(new TestAgentComponents(LoggerBase, centralConfigurationFetcher: new CentralConfigurationFetcher(LoggerBase, configStore, service, handler), payloadSender: new PayloadSenderV2(LoggerBase, snapshot, service, - new SystemInfoHelper(LoggerBase).GetSystemInfo(null), MockApmServerInfo.Version710)))) + new SystemInfoHelper(LoggerBase).GetSystemInfo(null, new TestHostNameDetector("detected_hostname")), MockApmServerInfo.Version710)))) { lastCentralConfigurationFetcher = (CentralConfigurationFetcher)agent.CentralConfigurationFetcher; lastCentralConfigurationFetcher.IsRunning.Should().BeTrue(); @@ -283,7 +283,7 @@ public void Create_many_concurrent_instances(int numberOfAgentInstances) LoggerBase, snapshot, service, - new SystemInfoHelper(LoggerBase).GetSystemInfo(null), + new SystemInfoHelper(LoggerBase).GetSystemInfo(null, new TestHostNameDetector("detected_hostname")), MockApmServerInfo.Version710); var components = new TestAgentComponents(LoggerBase, centralConfigurationFetcher: centralConfigFetcher, payloadSender: payloadSender); diff --git a/test/Elastic.Apm.Tests/SerializationTests.cs b/test/Elastic.Apm.Tests/SerializationTests.cs index 9a76672c8..7ae39c0e1 100644 --- a/test/Elastic.Apm.Tests/SerializationTests.cs +++ b/test/Elastic.Apm.Tests/SerializationTests.cs @@ -451,7 +451,7 @@ public void System_Should_Serialize_ConfiguredHostName_And_DetectedHostName() var systemHelper = new SystemInfoHelper(new NoopLogger()); var hostName = "this_is_my_hostname"; - var system = systemHelper.GetSystemInfo(hostName); + var system = systemHelper.GetSystemInfo(hostName, new TestHostNameDetector("detected_hostname")); var json = _payloadItemSerializer.Serialize(system); var jObject = JObject.Parse(json); diff --git a/test/Elastic.Apm.Tests/SystemInfoHelperTests.cs b/test/Elastic.Apm.Tests/SystemInfoHelperTests.cs index 204edfdb2..04e7cf1c6 100644 --- a/test/Elastic.Apm.Tests/SystemInfoHelperTests.cs +++ b/test/Elastic.Apm.Tests/SystemInfoHelperTests.cs @@ -4,12 +4,14 @@ using System; using System.Collections.Generic; +using System.Linq; using Elastic.Apm.Api.Kubernetes; using Elastic.Apm.Features; using Elastic.Apm.Helpers; using Elastic.Apm.Logging; using Elastic.Apm.Tests.Utilities; using FluentAssertions; +using Newtonsoft.Json; using Xunit; namespace Elastic.Apm.Tests; @@ -25,7 +27,7 @@ public class SystemInfoHelperTests : IDisposable public void ParseSystemInfo_Should_Use_HostName_For_ConfiguredHostName() { var hostName = "This_is_my_host"; - var system = _systemInfoHelper.GetSystemInfo(hostName); + var system = _systemInfoHelper.GetSystemInfo(hostName, new TestHostNameDetector("detected_host_name")); #pragma warning disable 618 system.HostName.Should().Be(hostName); @@ -41,7 +43,7 @@ public void Feature_ContainerInfo_ShouldBeDisabled_OnAzure() var logger = new TestLogger(LogLevel.Trace); using (new AgentFeaturesProviderScope(new AzureFunctionsAgentFeatures(logger))) { - new SystemInfoHelper(logger).GetSystemInfo("bert"); + new SystemInfoHelper(logger).GetSystemInfo("bert", new TestHostNameDetector("detected_host_name")); // // The actual parsing (not happening) is hard to test currently. // Let's assert the log output that tells us that this part gets skipped. @@ -61,30 +63,33 @@ public void ParseKubernetesInfo_ShouldReturnNull_WhenNoEnvironmentVariablesAreSe system.Kubernetes.Should().BeNull(); } - public struct CGroupTestData - { - public string GroupLine; - public string ContainerId; - public string PodId; - } - // Remove warning about unused test parameter "name" #pragma warning disable xUnit1026 [Theory] - [JsonFileData("./TestResources/json-specs/cgroup_parsing.json", typeof(CGroupTestData))] + [CGroupTestCases] public void ParseKubernetesInfo_FromCGroupLine(string name, CGroupTestData data) { - var line = data.GroupLine; + data.Files.ProcSelfCgroup.Should().NotBeNull(); + var line = data.Files.ProcSelfCgroup; var containerId = data.ContainerId; var podId = data.PodId; var system = new Api.System(); - _systemInfoHelper.ParseContainerId(system, "hostname", line); + if (line == "0::/") + { + line = data.Files.MountInfo.FirstOrDefault(l => l.Contains("/etc/hostname")); + _systemInfoHelper.ParseMountInfo(system, "hostname", line); + } + else + _systemInfoHelper.ParseContainerId(system, "hostname", line); if (containerId is null) system.Container.Should().BeNull(); else + { + system.Container.Should().NotBeNull("{0}", line); system.Container.Id.Should().Be(containerId); + } if (podId is null) system.Kubernetes.Should().BeNull(); diff --git a/test/iis/Elastic.Apm.AspNetFullFramework.Tests/TestsBase.cs b/test/iis/Elastic.Apm.AspNetFullFramework.Tests/TestsBase.cs index 26096cdb5..3612af656 100644 --- a/test/iis/Elastic.Apm.AspNetFullFramework.Tests/TestsBase.cs +++ b/test/iis/Elastic.Apm.AspNetFullFramework.Tests/TestsBase.cs @@ -629,7 +629,7 @@ private void FullFwAssertValid(Api.System system) { system.Should().NotBeNull(); - system.DetectedHostName.Should().Be(new SystemInfoHelper(LoggerBase).GetHostName()); + system.DetectedHostName.Should().Be(new HostNameDetector().GetDetectedHostName(LoggerBase)); #pragma warning disable 618 system.HostName.Should().Be(AgentConfig.HostName ?? system.DetectedHostName); #pragma warning restore 618 diff --git a/test/instrumentations/Elastic.Apm.Docker.Tests/ContainerIdCalculationTests.cs b/test/instrumentations/Elastic.Apm.Docker.Tests/ContainerIdCalculationTests.cs index 43e673b76..b58eed0b0 100644 --- a/test/instrumentations/Elastic.Apm.Docker.Tests/ContainerIdCalculationTests.cs +++ b/test/instrumentations/Elastic.Apm.Docker.Tests/ContainerIdCalculationTests.cs @@ -43,7 +43,7 @@ public void TestCGroupContent(string cGroupContent, string expectedContainerId) var noopLogger = new NoopLogger(); var systemInfoHelper = new TestSystemInfoHelper(noopLogger, cGroupContent); - var systemInfo = systemInfoHelper.GetSystemInfo(null); + var systemInfo = systemInfoHelper.GetSystemInfo(null, new TestHostNameDetector("detected_hostname")); systemInfo.Should().NotBeNull(); systemInfo.Container.Should().NotBeNull(); systemInfo.Container.Id.Should().Be(expectedContainerId); @@ -58,7 +58,7 @@ public void TestCGroupContentWithInvalidData() var noopLogger = new NoopLogger(); var systemInfoHelper = new TestSystemInfoHelper(noopLogger, "asdf:invalid-dockerid:243543"); - var systemInfo = systemInfoHelper.GetSystemInfo(null); + var systemInfo = systemInfoHelper.GetSystemInfo(null, new TestHostNameDetector("detected_hostname")); systemInfo.Container.Should().BeNull(); } diff --git a/test/profiler/Elastic.Apm.Profiler.Managed.Tests/ProfiledApplication.cs b/test/profiler/Elastic.Apm.Profiler.Managed.Tests/ProfiledApplication.cs index d4b98a126..ab9a523b6 100644 --- a/test/profiler/Elastic.Apm.Profiler.Managed.Tests/ProfiledApplication.cs +++ b/test/profiler/Elastic.Apm.Profiler.Managed.Tests/ProfiledApplication.cs @@ -56,7 +56,9 @@ protected ProfiledApplication(string projectName, params string[] folders) else profilerFile = "libelastic_apm_profiler.dylib"; - _profilerPath = Path.Combine(SolutionPaths.Root, "target", "release", profilerFile); + _profilerPath = TestEnvironment.IsLinux ? + Path.Combine(SolutionPaths.Root, "target", "x86_64-unknown-linux-gnu", "release", profilerFile) : + Path.Combine(SolutionPaths.Root, "target", "release", profilerFile); if (!File.Exists(_profilerPath)) { diff --git a/update-compose.yaml b/update-compose.yaml new file mode 100644 index 000000000..d40020933 --- /dev/null +++ b/update-compose.yaml @@ -0,0 +1,18 @@ +policies: + - name: Handle apm-data server specs + policy: ghcr.io/elastic/oblt-updatecli-policies/apm/apm-data-spec:0.2.0@sha256:7069c0773d44a74c4c8103b4d9957b468f66081ee9d677238072fe11c4d2197c + values: + - .ci/updatecli/values.d/scm.yml + - .ci/updatecli/values.d/apm-data-spec.yml + + - name: Handle apm gherkin specs + policy: ghcr.io/elastic/oblt-updatecli-policies/apm/apm-gherkin:0.2.0@sha256:26a30ad2b98a6e4cb17fb88a28fa3277ced8ca862d6388943afaafbf8ee96e7d + values: + - .ci/updatecli/values.d/scm.yml + - .ci/updatecli/values.d/apm-gherkin.yml + + - name: Handle apm json specs + policy: ghcr.io/elastic/oblt-updatecli-policies/apm/apm-json-specs:0.2.0@sha256:969a6d21eabd6ebea66cb29b35294a273d6dbc0f7da78589c416aedf08728e78 + values: + - .ci/updatecli/values.d/scm.yml + - .ci/updatecli/values.d/apm-json-specs.yml