diff --git a/.bingo/Variables.mk b/.bingo/Variables.mk index 132e432f87..5318abf9aa 100644 --- a/.bingo/Variables.mk +++ b/.bingo/Variables.mk @@ -89,11 +89,11 @@ $(JSONNETFMT): $(BINGO_DIR)/jsonnetfmt.mod @echo "(re)installing $(GOBIN)/jsonnetfmt-v0.17.0" @cd $(BINGO_DIR) && $(GO) build -mod=mod -modfile=jsonnetfmt.mod -o=$(GOBIN)/jsonnetfmt-v0.17.0 "github.com/google/go-jsonnet/cmd/jsonnetfmt" -MDOX := $(GOBIN)/mdox-v0.2.2-0.20210617084122-22b44c491197 +MDOX := $(GOBIN)/mdox-v0.2.2-0.20210810104227-dd2d81061a27 $(MDOX): $(BINGO_DIR)/mdox.mod @# Install binary/ries using Go 1.14+ build command. This is using bwplotka/bingo-controlled, separate go module with pinned dependencies. - @echo "(re)installing $(GOBIN)/mdox-v0.2.2-0.20210617084122-22b44c491197" - @cd $(BINGO_DIR) && $(GO) build -mod=mod -modfile=mdox.mod -o=$(GOBIN)/mdox-v0.2.2-0.20210617084122-22b44c491197 "github.com/bwplotka/mdox" + @echo "(re)installing $(GOBIN)/mdox-v0.2.2-0.20210810104227-dd2d81061a27" + @cd $(BINGO_DIR) && $(GO) build -mod=mod -modfile=mdox.mod -o=$(GOBIN)/mdox-v0.2.2-0.20210810104227-dd2d81061a27 "github.com/bwplotka/mdox" MINIO := $(GOBIN)/minio-v0.0.0-20200527010300-cccf2de129da $(MINIO): $(BINGO_DIR)/minio.mod diff --git a/.bingo/mdox.mod b/.bingo/mdox.mod index 540dbc6126..f42fa1e2b6 100644 --- a/.bingo/mdox.mod +++ b/.bingo/mdox.mod @@ -4,4 +4,4 @@ go 1.16 replace github.com/Kunde21/markdownfmt/v2 => github.com/bwplotka/markdownfmt/v2 v2.0.0-20210616121647-559e77044d46 -require github.com/bwplotka/mdox v0.2.2-0.20210617084122-22b44c491197 +require github.com/bwplotka/mdox v0.2.2-0.20210810104227-dd2d81061a27 diff --git a/.bingo/variables.env b/.bingo/variables.env index f49d01a057..99fd25ee65 100644 --- a/.bingo/variables.env +++ b/.bingo/variables.env @@ -32,7 +32,7 @@ JSONNET="${GOBIN}/jsonnet-v0.17.0" JSONNETFMT="${GOBIN}/jsonnetfmt-v0.17.0" -MDOX="${GOBIN}/mdox-v0.2.2-0.20210617084122-22b44c491197" +MDOX="${GOBIN}/mdox-v0.2.2-0.20210810104227-dd2d81061a27" MINIO="${GOBIN}/minio-v0.0.0-20200527010300-cccf2de129da" diff --git a/.mdox.prev-release.yaml b/.mdox.prev-release.yaml new file mode 100644 index 0000000000..57b3f07a18 --- /dev/null +++ b/.mdox.prev-release.yaml @@ -0,0 +1,65 @@ +version: 1 + +inputDir: "$(OUTPUT_CONTENT_DIR)/$(tags)-git-docs" +outputDir: "$(OUTPUT_CONTENT_DIR)/$(tags)" +extraInputGlobs: + - "CHANGELOG.md" + - "SECURITY.md" + - "MAINTAINERS.md" + - "CONTRIBUTING.md" + - "CODE_OF_CONDUCT.md" + +gitIgnored: true +localLinksStyle: + hugo: + indexFileName: "_index.md" + +transformations: + + - glob: "../../../CHANGELOG.md" + path: /thanos/CHANGELOG.md + frontMatter: + template: | + title: Changelog + type: docs + menu: thanos + lastmod: "{{ .Origin.LastMod }}" + + - glob: "../../../MAINTAINERS.md" + path: /thanos/MAINTAINERS.md + frontMatter: + template: | + title: Maintainers + type: docs + menu: thanos + lastmod: "{{ .Origin.LastMod }}" + + - glob: "../../../SECURITY.md" + path: /thanos/SECURITY.md + frontMatter: + template: | + title: Security + type: docs + menu: thanos + lastmod: "{{ .Origin.LastMod }}" + + - glob: "../../../CODE_OF_CONDUCT.md" + path: /contributing/CODE_OF_CONDUCT.md + frontMatter: + template: | + title: Code of Conduct + type: docs + menu: contributing + lastmod: "{{ .Origin.LastMod }}" + + - glob: "../../../CONTRIBUTING.md" + path: /contributing/CONTRIBUTING.md + frontMatter: + template: | + title: Contributing + type: docs + menu: contributing + lastmod: "{{ .Origin.LastMod }}" + + - glob: "*.md" + path: /thanos/* \ No newline at end of file diff --git a/.mdox.validate.yaml b/.mdox.validate.yaml new file mode 100644 index 0000000000..2b9bcd28bd --- /dev/null +++ b/.mdox.validate.yaml @@ -0,0 +1,19 @@ +version: 1 + +validators: + # Validators to skip checking PR/issue links of Thanos, Prometheus and Cortex. + - regex: '(^http[s]?:\/\/)(www\.)?(github\.com\/)thanos-io\/thanos(\/pull\/|\/issues\/)' + type: 'githubPullsIssues' + - regex: '(^http[s]?:\/\/)(www\.)?(github\.com\/)prometheus\/prometheus(\/pull\/|\/issues\/)' + type: 'githubPullsIssues' + - regex: '(^http[s]?:\/\/)(www\.)?(github\.com\/)cortexproject\/cortex(\/pull\/|\/issues\/)' + type: 'githubPullsIssues' + # Ignore Thanos release links. + - regex: '(^http[s]?:\/\/)(www\.)?(github\.com\/)thanos-io\/thanos(\/releases\/)' + type: 'ignore' + # Causes http stream errors with statuscode 0 sometimes. But is safe to skip. + - regex: 'slack\.cncf\.io' + type: 'ignore' + # 301 errors even when curl-ed. + - regex: 'envoyproxy\.io' + type: 'ignore' \ No newline at end of file diff --git a/.mdox.yaml b/.mdox.yaml new file mode 100644 index 0000000000..cca9c7caf5 --- /dev/null +++ b/.mdox.yaml @@ -0,0 +1,146 @@ +version: 1 + +inputDir: "docs" +outputDir: "website/docs-pre-processed/tip" +extraInputGlobs: + - "CHANGELOG.md" + - "SECURITY.md" + - "MAINTAINERS.md" + - "CONTRIBUTING.md" + - "CODE_OF_CONDUCT.md" + +gitIgnored: true +localLinksStyle: + hugo: + indexFileName: "_index.md" + +transformations: + + - glob: "../CHANGELOG.md" + path: /thanos/CHANGELOG.md + frontMatter: + template: | + title: Changelog + type: docs + menu: thanos + lastmod: "{{ .Origin.LastMod }}" + backMatter: &docBackMatter + template: | + Found a typo, inconsistency or missing information in our docs? + Help us to improve [Thanos](https://thanos.io) documentation by proposing a fix [on GitHub here](https://github.com/thanos-io/thanos/edit/main/{{ .Origin.Path }}) :heart: + + - glob: "../MAINTAINERS.md" + path: /thanos/MAINTAINERS.md + frontMatter: + template: | + title: Maintainers + type: docs + menu: thanos + lastmod: "{{ .Origin.LastMod }}" + backMatter: *docBackMatter + + - glob: "../SECURITY.md" + path: /thanos/SECURITY.md + frontMatter: + template: | + title: Security + type: docs + menu: thanos + lastmod: "{{ .Origin.LastMod }}" + backMatter: *docBackMatter + + - glob: "../CODE_OF_CONDUCT.md" + path: /contributing/CODE_OF_CONDUCT.md + frontMatter: + template: | + title: Code of Conduct + type: docs + menu: contributing + lastmod: "{{ .Origin.LastMod }}" + backMatter: *docBackMatter + + - glob: "../CONTRIBUTING.md" + path: /contributing/CONTRIBUTING.md + frontMatter: + template: | + title: Contributing + type: docs + menu: contributing + lastmod: "{{ .Origin.LastMod }}" + backMatter: *docBackMatter + + - glob: "**/README.md" + path: _index.md + frontMatter: &justTitleFrontMatter + template: | + title: "{{ .Origin.FirstHeader }}" + backMatter: *docBackMatter + + - glob: "getting-started.md" + path: /thanos/getting-started.md + popHeader: false + frontMatter: &weightOneFrontMatter + template: | + type: docs + title: "{{ .Origin.FirstHeader }}" + menu: thanos + weight: 1 + lastmod: "{{ .Origin.LastMod }}" + backMatter: *docBackMatter + + - glob: "quick-tutorial.md" + path: /thanos/quick-tutorial.md + popHeader: false + frontMatter: *weightOneFrontMatter + backMatter: *docBackMatter + + - glob: "proposals-*/README.md" + path: /proposals-*/_index.md + frontMatter: *justTitleFrontMatter + backMatter: *docBackMatter + + - glob: "README.md" + path: /thanos/_index.md + frontMatter: *justTitleFrontMatter + backMatter: *docBackMatter + + - glob: "*.md" + path: /thanos/* + popHeader: false + frontMatter: + template: | + type: docs + title: "{{ .Origin.FirstHeader }}" + menu: thanos + lastmod: "{{ .Origin.LastMod }}" + backMatter: *docBackMatter + + - glob: "operating/*.md" + path: /operating/* + frontMatter: + template: | + type: docs + title: "{{ .Origin.FirstHeader }}" + menu: operating + lastmod: "{{ .Origin.LastMod }}" + backMatter: *docBackMatter + + - glob: "contributing/*.md" + path: /contributing/* + frontMatter: + template: | + type: docs + title: "{{ .Origin.FirstHeader }}" + menu: contributing + lastmod: "{{ .Origin.LastMod }}" + backMatter: *docBackMatter + + - glob: "components/*.md" + path: /components/* + frontMatter: + template: | + type: docs + title: "{{ .Origin.FirstHeader }}" + menu: components + lastmod: "{{ .Origin.LastMod }}" + backMatter: *docBackMatter diff --git a/CHANGELOG.md b/CHANGELOG.md index 9544df63b3..f6718244b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -263,7 +263,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re Highlights: -- New Thanos component, [Query Frontend](https://thanos.io/tip/components/query-frontend.md/) has more options and supports shared cache (currently: Memcached). +- New Thanos component, [Query Frontend](docs/components/query-frontend.md) has more options and supports shared cache (currently: Memcached). - Added debug mode in Thanos UI that allows to filter Stores to query from by their IPs from Store page (!). This helps enormously in e.g debugging the slowest store etc. All raw Thanos API allows passing `storeMatch[]` arguments with `__address__` matchers. - Improved debuggability on all Thanos components by exposing [off-CPU profiles thanks to fgprof endpoint](https://github.com/felixge/fgprof). - Significantly improved sidecar latency and CPU usage for metrics fetches. @@ -302,7 +302,7 @@ Highlights: Highlights: -- Added new Thanos component: [Query Frontend](https://thanos.io/v0.15/components/query-frontend/) responsible for response caching, query scheduling and parallelization (based on Cortex Query Frontend). +- Added new Thanos component: [Query Frontend](https://thanos.io/v0.15/components/query-frontend.md/) responsible for response caching, query scheduling and parallelization (based on Cortex Query Frontend). - Added various new, improved UIs to Thanos based on React: Querier BuildInfo & Flags, Ruler UI, BlockViewer. - Optimized Sidecar, Store, Receive, Ruler data retrieval with new TSDB ChunkIterator (capping chunks to 120 samples), which fixed various leaks. - Fixed sample limit on Store Gateway. @@ -344,7 +344,7 @@ Highlights: - [#2973](https://github.com/thanos-io/thanos/pull/2973) Add Thanos Query Frontend component. - [#2980](https://github.com/thanos-io/thanos/pull/2980) Bucket Viewer: Migrate block viewer to React. - [#2725](https://github.com/thanos-io/thanos/pull/2725) Add bucket index operation durations: `thanos_bucket_store_cached_series_fetch_duration_seconds` and `thanos_bucket_store_cached_postings_fetch_duration_seconds`. -- [#2931](https://github.com/thanos-io/thanos/pull/2931) Query: Allow passing a `storeMatch[]` to select matching stores when debugging the querier. See [documentation](https://thanos.io/tip/components/query.md/#store-filtering) +- [#2931](https://github.com/thanos-io/thanos/pull/2931) Query: Allow passing a `storeMatch[]` to select matching stores when debugging the querier. See [documentation](docs/components/query.md#store-filtering) ### Changed @@ -422,7 +422,7 @@ sse_config: ### Changed - [#2194](https://github.com/thanos-io/thanos/pull/2194) Updated to golang v1.14.2. -- [#2505](https://github.com/thanos.io/thanos/pull/2505) Store: Removed obsolete `thanos_store_node_info` metric. +- [#2505](https://github.com/thanos-io/thanos/pull/2505) Store: Removed obsolete `thanos_store_node_info` metric. - [#2513](https://github.com/thanos-io/thanos/pull/2513) Tools: Moved `thanos bucket` commands to `thanos tools bucket`, also moved `thanos check rules` to `thanos tools rules-check`. `thanos tools rules-check` also takes rules by `--rules` repeated flag not argument anymore. - [#2548](https://github.com/thanos-io/thanos/pull/2548/commits/53e69bd89b2b08c18df298eed7d90cb7179cc0ec) Store, Querier: remove duplicated chunks on StoreAPI. - [#2596](https://github.com/thanos-io/thanos/pull/2596) Updated Prometheus dependency to [@cd73b3d33e064bbd846fc7a26dc8c313d46af382](https://github.com/prometheus/prometheus/commit/cd73b3d33e064bbd846fc7a26dc8c313d46af382) which falls in between v2.17.0 and v2.18.0. @@ -468,7 +468,7 @@ sse_config: ### Added -- [#2252](https://github.com/thanos-io/thanos/pull/2252) Query: add new `--store-strict` flag. More information available [here](/docs/proposals/202001_thanos_query_health_handling.md). +- [#2252](https://github.com/thanos-io/thanos/pull/2252) Query: add new `--store-strict` flag. More information available [here](docs/proposals-done/202001-thanos-query-health-handling.md). - [#2265](https://github.com/thanos-io/thanos/pull/2265) Compact: add `--wait-interval` to specify compaction wait interval between consecutive compact runs when `--wait` is enabled. - [#2250](https://github.com/thanos-io/thanos/pull/2250) Compact: enable vertical compaction for offline deduplication (experimental). Uses `--deduplication.replica-label` flag to specify the replica label on which to deduplicate (hidden). Please note that this uses a NAIVE algorithm for merging (no smart replica deduplication, just chaining samples together). This works well for deduplication of blocks with **precisely the same samples** like those produced by Receiver replication. We plan to add a smarter algorithm in the following weeks. - [#1714](https://github.com/thanos-io/thanos/pull/1714) Compact: the compact component now exposes the bucket web UI when it is run as a long-lived process. @@ -507,7 +507,7 @@ sse_config: ### Added - [#2003](https://github.com/thanos-io/thanos/pull/2003) Query: Support downsampling for /series. -- [#1952](https://github.com/thanos-io/thanos/pull/1952) Store Gateway: Implemented [binary index header](https://thanos.io/tip/proposals/201912_thanos_binary_index_header.md/). This significantly reduces resource consumption (memory, CPU, net bandwidth) for startup and data loading processes as well as baseline memory. This means that adding more blocks into object storage, without querying them will use almost no resources. This, however, **still means that querying large amounts of data** will result in high spikes of memory and CPU use as before, due to simply fetching large amounts of metrics data. Since we fixed baseline, we are now focusing on query performance optimizations in separate initiatives. To enable experimental `index-header` mode run store with hidden `experimental.enable-index-header` flag. +- [#1952](https://github.com/thanos-io/thanos/pull/1952) Store Gateway: Implemented [binary index header](docs/proposals-done/201912-thanos-binary-index-header.md). This significantly reduces resource consumption (memory, CPU, net bandwidth) for startup and data loading processes as well as baseline memory. This means that adding more blocks into object storage, without querying them will use almost no resources. This, however, **still means that querying large amounts of data** will result in high spikes of memory and CPU use as before, due to simply fetching large amounts of metrics data. Since we fixed baseline, we are now focusing on query performance optimizations in separate initiatives. To enable experimental `index-header` mode run store with hidden `experimental.enable-index-header` flag. - [#2009](https://github.com/thanos-io/thanos/pull/2009) Store Gateway: Minimum age of all blocks before they are being read. Set it to a safe value (e.g 30m) if your object storage is eventually consistent. GCS and S3 are (roughly) strongly consistent. - [#1963](https://github.com/thanos-io/thanos/pull/1963) Mixin: Add Thanos Ruler alerts. - [#1984](https://github.com/thanos-io/thanos/pull/1984) Query: Add cache-control header to not cache on error. @@ -856,7 +856,7 @@ This version moved tarballs to Golang 1.12.5 from 1.11 as well, so same warning :warning: **IMPORTANT** :warning: This is the last release that supports gossip. From Thanos v0.5.0, gossip will be completely removed. -This release also disables gossip mode by default for all components. See [this](docs/proposals/201809_gossip-removal.md) for more details. +This release also disables gossip mode by default for all components. See [this](docs/proposals-done/201809-gossip-removal.md) for more details. :warning: This release moves Thanos docker images (NOT artifacts by accident) to Golang 1.12. This release includes change in GC's memory release which gives following effect: @@ -1009,7 +1009,7 @@ New Store tracing span: \* `store_query_gate_ismyturn` shows how long it took fo - Support for gzip compressed configuration files before envvar substitution for reloader package. - `bucket inspect` command for better insights on blocks in object storage. -- Support for [Tencent COS](docs/storage.md#tencent-cos-configuration) object storage. +- Support for [Tencent COS](docs/storage.md#tencent-cos) object storage. - Partial Response disable option for StoreAPI and QueryAPI. - Partial Response disable button on Thanos UI - We have initial docs for goDoc documentation! @@ -1130,7 +1130,7 @@ Note lots of necessary breaking changes in flags that relates to bucket configur ## [v0.1.0](https://github.com/thanos-io/thanos/releases/tag/v0.1.0) - 2018.09.14 -Initial version to have a stable reference before [gossip protocol removal](/docs/proposals/201809_gossip-removal.md). +Initial version to have a stable reference before [gossip protocol removal](docs/proposals-done/201809-gossip-removal.md). ### Added diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 01b6c770d3..2d1007bdb4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -29,7 +29,7 @@ If you encounter a security vulnerability, please refer to [Reporting a Vulnerab When contributing a complex change to Thanos repository, please discuss the change you wish to make within a Github issue, in Slack, or by another method with the owners of this repository before making the change. -Adding a large new feature or/and component to Thanos should be done by first creating a [proposal](docs/contributing/proposal-process.md) document outlining the design decisions of the change, motivations for the change, and any alternatives that might have been considered. +Adding a large new feature or/and component to Thanos should be done by first creating a [proposal](docs/proposals-done) document outlining the design decisions of the change, motivations for the change, and any alternatives that might have been considered. ## General Naming diff --git a/MAINTAINERS.md b/MAINTAINERS.md index c5289a12fc..ba81d0a804 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -86,7 +86,7 @@ Self explanatory ones: ## Storage plugins maintainers -Maintainers of bucket storage clients are available [here](/docs/storage.md#implementations) +Maintainers of bucket storage clients are available [here](docs/storage.md#supported-clients) ## How to be maintainer? diff --git a/Makefile b/Makefile index ee0083a5b8..04ce3a8eca 100644 --- a/Makefile +++ b/Makefile @@ -27,6 +27,7 @@ endif GOPATH ?= $(shell go env GOPATH) TMP_GOPATH ?= /tmp/thanos-go GOBIN ?= $(firstword $(subst :, ,${GOPATH}))/bin +export GOBIN # Promu is using this exact variable name, do not rename. PREFIX ?= $(GOBIN) @@ -55,6 +56,9 @@ JSONNET_VENDOR_DIR ?= mixin/vendor WEB_DIR ?= website WEBSITE_BASE_URL ?= https://thanos.io +MDOX_VALIDATE_CONFIG ?= .mdox.validate.yaml +# for website pre process +export MDOX PUBLIC_DIR ?= $(WEB_DIR)/public ME ?= $(shell whoami) @@ -171,13 +175,13 @@ docker-push: docs: ## Regenerates flags in docs for all thanos commands localise links, ensure GitHub format. docs: $(MDOX) build @echo ">> generating docs" - PATH=${PATH}:$(GOBIN) $(MDOX) fmt --links.localize.address-regex="https://thanos.io/.*" $(MD_FILES_TO_FORMAT) + PATH=${PATH}:$(GOBIN) $(MDOX) fmt -l --links.localize.address-regex="https://thanos.io/.*" --links.validate.config-file=$(MDOX_VALIDATE_CONFIG) $(MD_FILES_TO_FORMAT) .PHONY: check-docs check-docs: ## checks docs against discrepancy with flags, links, white noise. check-docs: $(MDOX) build - @echo ">> checking local links" - PATH=${PATH}:$(GOBIN) $(MDOX) fmt --check --links.localize.address-regex="https://thanos.io/.*" $(MD_FILES_TO_FORMAT) + @echo ">> checking formatting and local/remote links" + PATH=${PATH}:$(GOBIN) $(MDOX) fmt --check -l --links.localize.address-regex="https://thanos.io/.*" --links.validate.config-file=$(MDOX_VALIDATE_CONFIG) $(MD_FILES_TO_FORMAT) $(call require_clean_work_tree,'run make docs and commit changes') .PHONY: shell-format @@ -265,7 +269,7 @@ else endif .PHONY: web-pre-process -web-pre-process: +web-pre-process: $(MDOX) @echo ">> running documentation website pre processing" scripts/website/websitepreprocess.sh diff --git a/README.md b/README.md index 06cb22b591..0724bd30fe 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ Concretely the aims of the project are: * [Design](https://thanos.io/tip/thanos/design.md/) * [Blog posts](docs/getting-started.md#blog-posts) * [Talks](docs/getting-started.md#talks) -* [Proposals](docs/proposals) +* [Proposals](docs/proposals-done) * [Integrations](docs/integrations.md) ## Features diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000000..e75d613777 --- /dev/null +++ b/docs/README.md @@ -0,0 +1 @@ +# Thanos General Documents: diff --git a/docs/components/README.md b/docs/components/README.md new file mode 100644 index 0000000000..d4d1e6e5e4 --- /dev/null +++ b/docs/components/README.md @@ -0,0 +1 @@ +# Components: diff --git a/docs/components/_index.md b/docs/components/_index.md deleted file mode 100644 index bc9d4437b5..0000000000 --- a/docs/components/_index.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -title: 'Components:' ---- - - diff --git a/docs/components/compact.md b/docs/components/compact.md index 176ea4950c..50090c8c26 100644 --- a/docs/components/compact.md +++ b/docs/components/compact.md @@ -1,9 +1,3 @@ ---- -type: docs -title: Compactor -menu: components ---- - # Compactor The `thanos compact` command applies the compaction procedure of the Prometheus 2.0 storage engine to block data stored in object storage. It is generally not semantically concurrency safe and must be deployed as a singleton against a bucket. diff --git a/docs/components/query-frontend.md b/docs/components/query-frontend.md index 6e1a6a7f74..3f32247248 100644 --- a/docs/components/query-frontend.md +++ b/docs/components/query-frontend.md @@ -1,9 +1,3 @@ ---- -type: docs -title: Query Frontend -menu: components ---- - # Query Frontend The `thanos query-frontend` command implements a service that can be put in front of Thanos Queriers to improve the read path. It is based on the [Cortex Query Frontend](https://cortexmetrics.io/docs/architecture/#query-frontend) component so you can find some common features like `Splitting` and `Results Caching`. @@ -20,7 +14,7 @@ thanos query-frontend \ _**NOTE:** Currently only range queries (`/api/v1/query_range` API call) are actually processed through Query Frontend. All other API calls just directly go to the downstream Querier, which means only range queries are split and cached. But we are planning to support instant queries as well. -For more information please check out [initial design proposal](https://thanos.io/tip/proposals/202004_embedd_cortex_frontend.md/). +For more information please check out [initial design proposal](../proposals-done/202004-embedd-cortex-frontend.md). ## Features @@ -82,7 +76,7 @@ config: If a `set` operation is skipped because of the item size is larger than `max_item_size`, this event is tracked by a counter metric `cortex_memcache_client_set_skip_total`. -Other cache configuration parameters, you can refer to [memcached-index-cache](https://thanos.io/tip/components/store.md/#memcached-index-cache). +Other cache configuration parameters, you can refer to [memcached-index-cache](store.md#memcached-index-cache). The default memcached config is: diff --git a/docs/components/query.md b/docs/components/query.md index 348de99799..21aa3604a2 100644 --- a/docs/components/query.md +++ b/docs/components/query.md @@ -1,9 +1,3 @@ ---- -type: docs -title: Query -menu: components ---- - # Querier/Query The `thanos query` command (also known as "Querier") implements the [Prometheus HTTP v1 API](https://prometheus.io/docs/prometheus/latest/querying/api/) to query data in a Thanos cluster via PromQL. @@ -35,7 +29,7 @@ Since for Querier "a backend" is anything that implements gRPC StoreAPI we can a * Metrics received from Prometheus remote write streams (see [Receiver](receive.md)) * Another Querier (you can stack Queriers on top of each other) * Non-Prometheus systems! - * e.g [OpenTSDB](../integrations.md#opentsdb) + * e.g [OpenTSDB](../integrations.md#opentsdb-as-storeapi) Thanks to that, you can run queries (manually, from Grafana or via Alerting rule) that aggregate metrics from mix of those sources. @@ -57,7 +51,7 @@ Thanos Querier instead pulls the data from both replicas, and deduplicate those Overall QueryAPI exposed by Thanos is guaranteed to be compatible with [Prometheus 2.x. API](https://prometheus.io/docs/prometheus/latest/querying/api/). The above diagram shows what Querier does for each Prometheus query request. -See [here](https://thanos.io/tip/thanos/service-discovery.md/) on how to connect Querier with desired StoreAPIs. +See [here](../service-discovery.md) on how to connect Querier with desired StoreAPIs. ### Deduplication diff --git a/docs/components/receive.md b/docs/components/receive.md index 8e60de1d30..318e2860eb 100644 --- a/docs/components/receive.md +++ b/docs/components/receive.md @@ -1,9 +1,3 @@ ---- -type: docs -title: Receiver -menu: components ---- - # Receiver The `thanos receive` command implements the [Prometheus Remote Write API](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write). It builds on top of existing Prometheus TSDB and retains its usefulness while extending its functionality with long-term-storage, horizontal scalability, and downsampling. Prometheus instances are configured to continuously write metrics to it, and then Thanos Receive uploads TSDB blocks to an object storage bucket every 2 hours by default. Thanos Receive exposes the StoreAPI so that [Thanos Queriers](query.md) can query received metrics in real-time. @@ -14,7 +8,7 @@ Thanos Receive supports multi-tenancy by using labels. See [Multitenancy documen Thanos Receive supports ingesting [exemplars](https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#exemplars) via remote-write. By default, the exemplars are silently discarded as `--tsdb.max-exemplars` is set to `0`. To enable exemplars storage, set the `--tsdb.max-exemplars` flag to a non-zero value. It exposes the ExemplarsAPI so that the [Thanos Queriers](query.md) can query the stored exemplars. Take a look at the documentation for [exemplars storage in Prometheus](https://prometheus.io/docs/prometheus/latest/disabled_features/#exemplars-storage) to know more about it. -For more information please check out [initial design proposal](../proposals/201812_thanos-remote-receive.md). For further information on tuning Prometheus Remote Write [see remote write tuning document](https://prometheus.io/docs/practices/remote_write/). +For more information please check out [initial design proposal](../proposals-done/201812-thanos-remote-receive.md). For further information on tuning Prometheus Remote Write [see remote write tuning document](https://prometheus.io/docs/practices/remote_write/). > NOTE: As the block producer it's important to set correct "external labels" that will identify data block across Thanos clusters. See [external labels](../storage.md#external-labels) docs for details. diff --git a/docs/components/rule.md b/docs/components/rule.md index 2445121263..446f894004 100644 --- a/docs/components/rule.md +++ b/docs/components/rule.md @@ -1,9 +1,3 @@ ---- -type: docs -title: Rule -menu: components ---- - # Rule (aka Ruler) ***NOTE:** It is recommended to keep deploying rules inside the relevant Prometheus servers locally. Use ruler only on specific cases. Read details [below](#risk) why.* @@ -168,7 +162,7 @@ Those metrics are important for vanilla Prometheus as well, but even more import // TODO(bwplotka): Rereview them after recent changes in metrics. -See [alerts](/examples/alerts/alerts.md#Ruler) for more example alerts for ruler. +See [alerts](https://github.com/thanos-io/thanos/blob/e3b0baf7de9dde1887253b1bb19d78ae71a01bf8/examples/alerts/alerts.md#ruler) for more example alerts for ruler. NOTE: It is also recommended to set a mocked Alert on Ruler that checks if Query is up. This might be something simple like `vector(1)` query, just to check if Querier is live. diff --git a/docs/components/sidecar.md b/docs/components/sidecar.md index 2fc1489e90..0c6d6865f6 100644 --- a/docs/components/sidecar.md +++ b/docs/components/sidecar.md @@ -1,9 +1,3 @@ ---- -type: docs -title: Sidecar -menu: components ---- - # Sidecar The `thanos sidecar` command runs a component that gets deployed along with a Prometheus instance. This allows sidecar to optionally upload metrics to object storage and allow [Queriers](query.md) to query Prometheus data with common, efficient StoreAPI. @@ -27,7 +21,7 @@ Prometheus servers connected to the Thanos cluster via the sidecar are subject t If you choose to use the sidecar to also upload data to object storage: * Must specify object storage (`--objstore.*` flags) -* It only uploads uncompacted Prometheus blocks. For compacted blocks, see [Upload compacted blocks](./sidecar.md/#upload-compacted-blocks). +* It only uploads uncompacted Prometheus blocks. For compacted blocks, see [Upload compacted blocks](#upload-compacted-blocks). * The `--storage.tsdb.min-block-duration` and `--storage.tsdb.max-block-duration` must be set to equal values to disable local compaction on order to use Thanos sidecar upload, otherwise leave local compaction on if sidecar just exposes StoreAPI and your retention is normal. The default of `2h` is recommended. Mentioned parameters set to equal values disable the internal Prometheus compaction, which is needed to avoid the uploaded data corruption when Thanos compactor does its job, this is critical for data consistency and should not be ignored if you plan to use Thanos compactor. Even though you set mentioned parameters equal, you might observe Prometheus internal metric `prometheus_tsdb_compactions_total` being incremented, don't be confused by that: Prometheus writes initial head block to filesystem via its internal compaction mechanism, but if you have followed recommendations - data won't be modified by Prometheus before the sidecar uploads it. Thanos sidecar will also check sanity of the flags set to Prometheus on the startup and log errors or warning if they have been configured improperly (#838). * The retention of Prometheus is recommended to not be lower than three times of the min block duration, so 6 hours. This achieves resilience in the face of connectivity issues to the object storage since all local data will remain available within the Thanos cluster. If connectivity gets restored the backlog of blocks gets uploaded to the object storage. diff --git a/docs/components/store.md b/docs/components/store.md index 0b955a4b3d..3ae037beb9 100644 --- a/docs/components/store.md +++ b/docs/components/store.md @@ -1,9 +1,3 @@ ---- -type: docs -title: Store -menu: components ---- - # Store The `thanos store` command (also known as Store Gateway) implements the Store API on top of historical data in an object storage bucket. It acts primarily as an API gateway and therefore does not need significant amounts of local disk space. It joins a Thanos cluster on startup and advertises the data it can access. It keeps a small amount of information about all remote blocks on local disk and keeps it in sync with the bucket. This data is generally safe to delete across restarts at the cost of increased startup times. @@ -235,11 +229,11 @@ We recommend having overlapping time ranges with Thanos Sidecar and other Thanos Thanos Querier deals with overlapping time series by merging them together. -Filtering is done on a [Chunk](../design.md/#chunk) level, so Thanos Store might still return Samples which are outside of `--min-time` & `--max-time`. +Filtering is done on a [Chunk](../design.md#chunk) level, so Thanos Store might still return Samples which are outside of `--min-time` & `--max-time`. ### External Label Partitioning (Sharding) -Check more [here](https://thanos.io/tip/thanos/sharding.md/). +Check more [here](../sharding.md). ## Probes @@ -295,7 +289,7 @@ config: The **required** settings are: -- `addresses`: list of memcached addresses, that will get resolved with the [DNS service discovery](../service-discovery.md/#dns-service-discovery) provider. If your cluster supports auto-discovery, you should use the flag `auto_discovery` instead and only point to *one of* the memcached servers. This typically means that there should be only one address specified that resolves to any of the alive memcached servers. Use this for Amazon ElastiCache and other similar services. +- `addresses`: list of memcached addresses, that will get resolved with the [DNS service discovery](../service-discovery.md#dns-service-discovery) provider. If your cluster supports auto-discovery, you should use the flag `auto_discovery` instead and only point to *one of* the memcached servers. This typically means that there should be only one address specified that resolves to any of the alive memcached servers. Use this for Amazon ElastiCache and other similar services. While the remaining settings are **optional**: @@ -311,7 +305,7 @@ While the remaining settings are **optional**: ## Caching Bucket -Thanos Store Gateway supports a "caching bucket" with [chunks](../design.md/#chunk) and metadata caching to speed up loading of [chunks](../design.md/#chunk) from TSDB blocks. To configure caching, one needs to use `--store.caching-bucket.config=` or `--store.caching-bucket.config-file=`. +Thanos Store Gateway supports a "caching bucket" with [chunks](../design.md#chunk) and metadata caching to speed up loading of [chunks](../design.md#chunk) from TSDB blocks. To configure caching, one needs to use `--store.caching-bucket.config=` or `--store.caching-bucket.config-file=`. Both memcached and in-memory cache "backend"s are supported: @@ -340,11 +334,11 @@ metafile_max_size: 1MiB `config` field for memcached supports all the same configuration as memcached for [index cache](#memcached-index-cache). `addresses` in the config field is a **required** setting -Additional options to configure various aspects of [chunks](../design.md/#chunk) cache are available: +Additional options to configure various aspects of [chunks](../design.md#chunk) cache are available: -- `chunk_subrange_size`: size of segment of [chunks](../design.md/#chunk) object that is stored to the cache. This is the smallest unit that chunks cache is working with. +- `chunk_subrange_size`: size of segment of [chunks](../design.md#chunk) object that is stored to the cache. This is the smallest unit that chunks cache is working with. - `max_chunks_get_range_requests`: how many "get range" sub-requests may cache perform to fetch missing subranges. -- `chunk_object_attrs_ttl`: how long to keep information about [chunk file](../design.md/#chunk-file) attributes (e.g. size) in the cache. +- `chunk_object_attrs_ttl`: how long to keep information about [chunk file](../design.md#chunk-file) attributes (e.g. size) in the cache. - `chunk_subrange_ttl`: how long to keep individual subranges in the cache. Following options are used for metadata caching (meta.json files, deletion mark files, iteration result): @@ -355,7 +349,7 @@ Following options are used for metadata caching (meta.json files, deletion mark - `metafile_content_ttl`: how long to cache content of meta.json and deletion mark files. - `metafile_max_size`: maximum size of cached meta.json and deletion mark file. Larger files are not cached. -The yml structure for setting the in memory cache configs for caching bucket is the same as the [in-memory index cache](https://thanos.io/tip/components/store.md/#in-memory-index-cache) and all the options to configure Caching Buket mentioned above can be used. +The yml structure for setting the in memory cache configs for caching bucket is the same as the [in-memory index cache](#in-memory-index-cache) and all the options to configure Caching Buket mentioned above can be used. Note that chunks and metadata cache is an experimental feature, and these fields may be renamed or removed completely in the future. diff --git a/docs/components/tools.md b/docs/components/tools.md index 3557c9f286..b000316a54 100644 --- a/docs/components/tools.md +++ b/docs/components/tools.md @@ -1,9 +1,3 @@ ---- -type: docs -title: Tools -menu: components ---- - # Tools The `thanos tools` subcommand of Thanos is a set of additional CLI, short-living tools that are meant to be ran for development or debugging purposes. diff --git a/docs/contributing/README.md b/docs/contributing/README.md new file mode 100644 index 0000000000..a218eaa0b8 --- /dev/null +++ b/docs/contributing/README.md @@ -0,0 +1 @@ +# Contributing: diff --git a/docs/contributing/_index.md b/docs/contributing/_index.md deleted file mode 100644 index 3ec7a0a4a2..0000000000 --- a/docs/contributing/_index.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -title: 'Contributing:' ---- - - diff --git a/docs/contributing/coding-style-guide.md b/docs/contributing/coding-style-guide.md index 8eb238ad26..3bd3cad9d2 100644 --- a/docs/contributing/coding-style-guide.md +++ b/docs/contributing/coding-style-guide.md @@ -1,9 +1,3 @@ ---- -type: docs -title: Coding Style Guide -menu: contributing ---- - # Thanos Coding Style Guide This document details the official style guides for the various languages we use in the Thanos project. Feel free to familiarize yourself with and refer to this document during code reviews. If something in our codebase does not match the style, it means it was missed or it was written before this document. Help wanted to fix it! (: diff --git a/docs/contributing/community.md b/docs/contributing/community.md index a4f5f5eae6..4d471efcf0 100644 --- a/docs/contributing/community.md +++ b/docs/contributing/community.md @@ -1,9 +1,3 @@ ---- -type: docs -title: Community -menu: contributing ---- - # Community Thanos is an open source project from the very first commit. We always value and welcome new contributors and members of the community. Here are ways to get in touch with the community: @@ -48,8 +42,8 @@ Meeting details: [https://bit.ly/prometheus-community-agenda](https://bit.ly/pro ### Mentorships -We participate in periodic mentorship programs. Read more [here](https://thanos.io/tip/contributing/mentorship.md/). +We participate in periodic mentorship programs. Read more [here](mentorship.md). ### Further Questions -Feel free to contact any of the [Maintainers](https://thanos.io/tip/thanos/maintainers.md/) +Feel free to contact any of the [Maintainers](../../MAINTAINERS.md) diff --git a/docs/contributing/how-to-change-go-version.md b/docs/contributing/how-to-change-go-version.md index f7d79d0a2f..9fc3be5163 100644 --- a/docs/contributing/how-to-change-go-version.md +++ b/docs/contributing/how-to-change-go-version.md @@ -1,8 +1,4 @@ ---- -type: docs -title: Changing Golang version -menu: contributing ---- +# Changing Golang version Thanos build system is pinned to certain Golang version. This is to ensure that Golang version changes is done by us in controlled, traceable way. diff --git a/docs/contributing/how-to-contribute-to-docs.md b/docs/contributing/how-to-contribute-to-docs.md index 659c10338e..857c802343 100644 --- a/docs/contributing/how-to-contribute-to-docs.md +++ b/docs/contributing/how-to-contribute-to-docs.md @@ -1,9 +1,3 @@ ---- -type: docs -title: Contribute to docs -menu: contributing ---- - # How to contribute to Docs/Website `./docs` directory is used as markdown source files using [blackfriday](https://github.com/russross/blackfriday) to render Thanos website resources. diff --git a/docs/contributing/mentorship.md b/docs/contributing/mentorship.md index db2e7c4db0..6706fa7893 100644 --- a/docs/contributing/mentorship.md +++ b/docs/contributing/mentorship.md @@ -1,14 +1,8 @@ ---- -type: docs -title: Mentorship -menu: contributing ---- - # Thanos / Prometheus Open Source Mentorship Both Thanos and Prometheus projects participate in various, periodic mentoring programs. -[Follow us on Twitter](https://thanos.io/tip/contributing/community.md/#twitter) to be up-to-date with application process and announcements. +[Follow us on Twitter](https://twitter.com/ThanosMetrics) to be up-to-date with application process and announcements. Programs we participated / are participating: @@ -22,14 +16,14 @@ You have been selected for mentorship with Prometheus or Thanos? Congratulations ### Onboarding Check List -- [X] Please see and acknowledge our [Code of Conduct](https://thanos.io/tip/contributing/code_of_conduct.md/). A similar one is for the Prometheus project. TL;DR: Be nice and friendly! +- [X] Please see and acknowledge our [Code of Conduct](../../CODE_OF_CONDUCT.md). A similar one is for the Prometheus project. TL;DR: Be nice and friendly! - [X] Create an account on the [CNCF Slack](https://slack.cncf.io/). Please add any profile photo to make it unique. (: This is Thanos's main chat provider. You can reach any of the mentors there. - [X] If you are a Prometheus mentee OR if your task will involve collaborating with the Prometheus project, it's also recommended to set up an IRC account e.g by [joining Element](https://prometheus.io/community/). This the main channel Prometheus Team is using currently. - [X] Send your mentors your slack account handle, so we can join you to private channels. We have one with just you and your main mentors, and second with all mentees, ex-mentees and mentors. -- [X] Join main [Thanos channels](https://thanos.io/tip/contributing/community.md/#slack) and write a few sentences to about yourself, say hello! 💜 +- [X] Join main [Thanos channels](community.md#slack) and write a few sentences to about yourself, say hello! 💜 - [X] You are welcome to join Twitter, start following others and gather your own followers! Who knows maybe you will get addicted? (: Feel free to post any related content around Thanos while mentioning `@ThanosMetrics`. - [X] Ask maintainers to set up some weekly 2:1 meeting (highly recommended). -- [X] Read our [contributing guides](https://thanos.io/tip/contributing/contributing.md/) +- [X] Read our [contributing guides](../../CONTRIBUTING.md) - [X] Read carefully your main GitHub issue and start thinking about it, but don't stress out! It's better to start slowly with a smaller task to get things going. 🚀 ### Suggestions diff --git a/docs/design.md b/docs/design.md index 9fe65197d2..e7fc9e088b 100644 --- a/docs/design.md +++ b/docs/design.md @@ -1,9 +1,3 @@ ---- -type: docs -title: Design -menu: thanos ---- - # Design Thanos is a set of components that can be composed into a highly available Prometheus setup with long term storage capabilities. Its main goals are operation simplicity and retaining of Prometheus's reliability properties. @@ -42,8 +36,8 @@ Data sources that persist their data for long-term storage do so via the Prometh A blocks top-level directory is a ULID (like UUID but lexicographically sortable and encoding the creation time). -- [Chunk files](design.md/#chunk-file) hold a few hundred MB worth of chunks each. Chunks for the same series are sequentially aligned. Series in return are aligned by their metric name. This becomes relevant further down. -- The index file holds all information needed to look up specific series by their labels and the positions of their [chunks](design.md/#chunk). +- [Chunk files](#chunk-file) hold a few hundred MB worth of chunks each. Chunks for the same series are sequentially aligned. Series in return are aligned by their metric name. This becomes relevant further down. +- The index file holds all information needed to look up specific series by their labels and the positions of their [chunks](#chunk). - `meta.json` holds meta-information about a block like stats, time range, and compaction level. Those block files can be backed up to object storage and later be queried by another component (see below). All data is uploaded as it is created by the Prometheus server/storage engine. The `meta.json` file may be extended by a `thanos` section, to which Thanos-specific metadata can be added. Currently, this includes the "external labels" the producer of the block has assigned. This later helps in filtering blocks for querying without accessing their data files. The meta.json is updated during upload time on sidecars. @@ -75,15 +69,15 @@ A store node acts as a gateway to block data that is stored in an object storage It continuously synchronizes which blocks exist in the bucket and translates requests for metric data into object storage requests. It implements various strategies to minimize the number of requests to the object storage such as filtering relevant blocks by their metadata (e.g. time range and labels) and caching frequent index lookups. -The Prometheus 2.0 storage layout is optimized for minimal read amplification. For example, sample data for the same time series is sequentially aligned in a [chunk file](design.md/#chunk-file). Similarly, series for the same metric name is sequentially aligned as well. The store node is aware of the files' layout and translates data requests into a plan of a minimum amount of object storage request. Each request may fetch up to hundreds of thousands of [chunks](design.md/#chunk) at once. This is essential to satisfy even big queries with a limited amount of requests to the object storage. +The Prometheus 2.0 storage layout is optimized for minimal read amplification. For example, sample data for the same time series is sequentially aligned in a [chunk file](#chunk-file). Similarly, series for the same metric name is sequentially aligned as well. The store node is aware of the files' layout and translates data requests into a plan of a minimum amount of object storage request. Each request may fetch up to hundreds of thousands of [chunks](#chunk) at once. This is essential to satisfy even big queries with a limited amount of requests to the object storage. -Currently, only index data is cached. [Chunk](design.md/#chunk) data could be cached but is orders of magnitude larger in size. In the current state, fetching chunk data from the object storage already only accounts for a small fraction of end-to-end latency. Thus, there's currently no incentive to increase the store nodes resource requirements/limit its scalability by adding chunk caching. +Currently, only index data is cached. [Chunk](#chunk) data could be cached but is orders of magnitude larger in size. In the current state, fetching chunk data from the object storage already only accounts for a small fraction of end-to-end latency. Thus, there's currently no incentive to increase the store nodes resource requirements/limit its scalability by adding chunk caching. ### Stores & Data Sources - It's all the same Since store nodes and data sources expose the same gRPC Store API, clients can largely treat them as equivalent and don't have to be concerned with which specific component they are querying. Each implementer of the Store API advertises meta-information about the data they provide. This allows clients to minimize the set of nodes they have to fan out, to satisfy a particular data query. -In its essence, the Store API allows to look up data by a set of label matches (as known from PromQL), and a time range. It returns compressed [chunks](design.md/#chunk) of samples as they are found in the block data. It is purely a data retrieval API and does *not* provide complex query execution. +In its essence, the Store API allows to look up data by a set of label matches (as known from PromQL), and a time range. It returns compressed [chunks](#chunk) of samples as they are found in the block data. It is purely a data retrieval API and does *not* provide complex query execution. ``` ┌──────────────────────┐ ┌────────────┬─────────┐ ┌────────────┐ diff --git a/docs/getting-started.md b/docs/getting-started.md index 40129a0d0a..2875a18faa 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -1,10 +1,3 @@ ---- -weight: 1 -type: docs -title: Getting Started -menu: thanos ---- - # Getting started Thanos provides a global query view, high availability, data backup with historical, cheap data access as its core features in a single binary. diff --git a/docs/governance.md b/docs/governance.md index 558577149e..3731f24d6c 100644 --- a/docs/governance.md +++ b/docs/governance.md @@ -1,18 +1,12 @@ ---- -type: docs -title: Governance -menu: thanos ---- - # Governance This document describes the rules and governance of the project. It is a slightly modified version of the [Prometheus Governance](https://prometheus.io/governance/#governance-changes). It is meant to be followed by all the developers of the Thanos project and the Thanos community. Common terminology used in this governance document are listed below: -* **Maintainers Team**: A core Thanos team that have owner access to http://github.com/thanos-io organization and all projects within it. Current list is available [here](https://thanos.io/tip/thanos/maintainers.md/). +* **Maintainers Team**: A core Thanos team that have owner access to http://github.com/thanos-io organization and all projects within it. Current list is available [here](../MAINTAINERS.md). -* **Triage Team**: Contributors who does not belong to Maintainer's team, but has `Triage` GitHub role on [Thanos](https://github.com/thanos-io/thanos) repository allowing to change GitHub issues and PRs statuses and labels. They are listed [here](https://thanos.io/tip/thanos/maintainers.md/#triage). +* **Triage Team**: Contributors who does not belong to Maintainer's team, but has `Triage` GitHub role on [Thanos](https://github.com/thanos-io/thanos) repository allowing to change GitHub issues and PRs statuses and labels. They are listed [here](../MAINTAINERS.md#triage). * **The Thanos project**: The sum of all activities performed under the [thanos-io organization on GitHub](https://github.com/thanos-io), concerning one or more repositories or the community. @@ -36,7 +30,7 @@ If they choose to accept, the following steps are taken: * Maintainer is added to the [GitHub organization](https://github.com/thanos-io) as *Owner*. * Maintainer is added to the [thanos-io](https://groups.google.com/forum/#!forum/thanos-io). -* Maintainer is added to the list of team members [here](https://thanos.io/tip/thanos/maintainers.md/) +* Maintainer is added to the list of team members [here](../MAINTAINERS.md) * New maintainer is announced on the [Thanos Twitter](https://twitter.com/ThanosMetrics) by an existing team member. Team members may retire at any time by emailing [thanos-io@googlegroups.com](https://groups.google.com/forum/#!forum/thanos-io). @@ -57,7 +51,7 @@ If they choose to accept, the following steps are taken: * Triage member is added to the [Thanos project](http://github.com/thanos-io/thanos) with `Triage` access. * Triage member is added to the [thanos-io](https://groups.google.com/forum/#!forum/thanos-io). -* Triage member is added to the list of Triage members [here](https://thanos.io/tip/thanos/maintainers.md/). +* Triage member is added to the list of Triage members [here](../MAINTAINERS.md#triage). * New team Triage member are announced on the [Thanos Twitter](https://twitter.com/ThanosMetrics) by an existing team member. Triage member may retire at any time by emailing [thanos-io@googlegroups.com](https://groups.google.com/forum/#!forum/thanos-io). It can be proposed to step up as Maintainer in any time as well. diff --git a/docs/integrations.md b/docs/integrations.md index d0764d3e1a..e19e518daf 100644 --- a/docs/integrations.md +++ b/docs/integrations.md @@ -1,14 +1,8 @@ ---- -type: docs -title: Integrations -menu: thanos ---- - # Integrations ## StoreAPI -[StoreAPI](https://github.com/thanos-io/thanos/blob/main/pkg/store/storepb/rpc.proto) is a common proto interface for gRPC component that can connect to [Querier](../components/query.md) in order to fetch the metric series. Natively Thanos implements [Sidecar](../components/sidecar.md) (local Prometheus data), [Ruler](../components/rule.md) and [Store gateway](../components/store.md). This solves fetching series from Prometheus or Prometheus TSDB format, however same interface can be used to fetch metrics from other storages. +[StoreAPI](https://github.com/thanos-io/thanos/blob/main/pkg/store/storepb/rpc.proto) is a common proto interface for gRPC component that can connect to [Querier](components/query.md) in order to fetch the metric series. Natively Thanos implements [Sidecar](components/sidecar.md) (local Prometheus data), [Ruler](components/rule.md) and [Store gateway](components/store.md). This solves fetching series from Prometheus or Prometheus TSDB format, however same interface can be used to fetch metrics from other storages. Below you can find some public integrations with other systems through StoreAPI: @@ -22,4 +16,4 @@ Although OpenTSDB is able to aggregate the data, it's not supported by Geras at [thanos-remote-read](https://github.com/G-Research/thanos-remote-read) is another StoreAPI integration from our friends at G-Research. -It's a proxy, that allows exposing any Thanos service (or anything that exposes gRPC StoreAPI e.g. [Querier](../components/query.md)) via Prometheus [remote read](https://github.com/prometheus/prometheus/blob/38d32e06862f6b72700f67043ce574508b5697f0/prompb/remote.proto#L27) protocol. This means that you can connect Thanos and expose its data to any remote-read compatible applications including [Prometheus itself.](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_read) +It's a proxy, that allows exposing any Thanos service (or anything that exposes gRPC StoreAPI e.g. [Querier](components/query.md)) via Prometheus [remote read](https://github.com/prometheus/prometheus/blob/38d32e06862f6b72700f67043ce574508b5697f0/prompb/remote.proto#L27) protocol. This means that you can connect Thanos and expose its data to any remote-read compatible applications including [Prometheus itself.](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_read) diff --git a/docs/operating/README.md b/docs/operating/README.md new file mode 100644 index 0000000000..c2e2bb5812 --- /dev/null +++ b/docs/operating/README.md @@ -0,0 +1 @@ +# Operating Guides: diff --git a/docs/operating/_index.md b/docs/operating/_index.md deleted file mode 100644 index 931dbfffec..0000000000 --- a/docs/operating/_index.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -title: 'Operating Guides:' ---- - - diff --git a/docs/operating/binary-index-header.md b/docs/operating/binary-index-header.md index 4524b9e126..1ab548073e 100644 --- a/docs/operating/binary-index-header.md +++ b/docs/operating/binary-index-header.md @@ -1,9 +1,3 @@ ---- -type: docs -title: Binary index-header -menu: operating ---- - # Binary index-header In order to query series inside blocks from object storage, [Store Gateway](../components/store.md) has to know certain initial info about each block such as: diff --git a/docs/operating/cross-cluster-tls-communication.md b/docs/operating/cross-cluster-tls-communication.md index a606a5a842..81f22cf752 100644 --- a/docs/operating/cross-cluster-tls-communication.md +++ b/docs/operating/cross-cluster-tls-communication.md @@ -1,9 +1,3 @@ ---- -type: docs -title: Configuring Thanos Secure TLS Cross-Cluster Communication -menu: operating ---- - # Configuring Thanos Secure TLS Cross-Cluster Communication ###### *This guide was contributed by the community thanks to [gmintoco](https://github.com/gmintoco)* diff --git a/docs/operating/https.md b/docs/operating/https.md index 22e17d1af9..93fa5b398e 100644 --- a/docs/operating/https.md +++ b/docs/operating/https.md @@ -1,10 +1,4 @@ ---- -type: docs -title: Running Thanos with HTTPS and basic auth -menu: operating ---- - -# HTTPS and authentication +# Running Thanos with HTTPS and basic authentication Thanos supports basic authentication and TLS. This is **experimental** and might change in the future. diff --git a/docs/operating/multi-tenancy.md b/docs/operating/multi-tenancy.md index 671f3ad697..a52741505d 100644 --- a/docs/operating/multi-tenancy.md +++ b/docs/operating/multi-tenancy.md @@ -1,9 +1,3 @@ ---- -type: docs -title: Multi-Tenancy -menu: operating ---- - # Multi-Tenancy Thanos supports multi-tenancy by using external labels. For such use cases, the [Thanos Sidecar](../components/sidecar.md) based approach with layered [Thanos Queriers](../components/query.md) is recommended. diff --git a/docs/operating/reverse-proxy.md b/docs/operating/reverse-proxy.md index 6e5833518c..70b1dfed3a 100644 --- a/docs/operating/reverse-proxy.md +++ b/docs/operating/reverse-proxy.md @@ -1,9 +1,3 @@ ---- -type: docs -title: Running Thanos behind a reverse proxy -menu: operating ---- - # Running Thanos behind a reverse proxy There are many reasons to use a [reverse proxy](https://www.nginx.com/resources/glossary/reverse-proxy-server/) in front of Thanos, for example, TLS termination (serve Thanos over HTTPS) and basic authentication. This small guide will tell you how to do that. diff --git a/docs/operating/troubleshooting.md b/docs/operating/troubleshooting.md index 49234fc5b0..5c8060f592 100644 --- a/docs/operating/troubleshooting.md +++ b/docs/operating/troubleshooting.md @@ -1,9 +1,3 @@ ---- -type: docs -title: Troubleshooting -menu: operating ---- - # Troubleshooting; Common cases ## Overlaps @@ -23,7 +17,7 @@ In this halted example, we can read that compactor detected 2 overlapped blocks. * Duplicated upload with different ULID (non-persistent storage for Prometheus can cause this) * 2 Prometheus instances are misconfigured and they are uploading the data with exactly the same external labels. This is wrong, they should be unique. -Checking producers log for such ULID, and checking meta.json (e.g if sample stats are the same or not) helps. Checksum the index and [chunks files](../design.md/#chunk-file) as well to reveal if data is exactly the same, thus ok to be removed manually. You may find `scripts/thanos-block.jq` script useful when inspecting `meta.json` files, as it translates timestamps to human-readable form. +Checking producers log for such ULID, and checking meta.json (e.g if sample stats are the same or not) helps. Checksum the index and [chunks files](../design.md#chunk-file) as well to reveal if data is exactly the same, thus ok to be removed manually. You may find `scripts/thanos-block.jq` script useful when inspecting `meta.json` files, as it translates timestamps to human-readable form. ### Reasons @@ -31,7 +25,7 @@ Checking producers log for such ULID, and checking meta.json (e.g if sample stat - Misconfiguraiton of sidecar/ruler: Same external labels or no external labels across many block producers. - Running multiple compactors for single block "stream", even for short duration. - Manually uploading blocks to the bucket. -- Eventually consistent block storage until we fully implement [RW for bucket](https://thanos.io/tip/proposals/201901-read-write-operations-bucket.md) +- Eventually consistent block storage until we fully implement [RW for bucket](../proposals-done/201901-read-write-operations-bucket.md) ### Solutions diff --git a/docs/operating/use-cases.md b/docs/operating/use-cases.md index 2b533e0d3d..7057891d8d 100644 --- a/docs/operating/use-cases.md +++ b/docs/operating/use-cases.md @@ -1,10 +1,4 @@ ---- -type: docs -title: Use cases -menu: operating ---- - -# Overview +# Use Cases There are many great use cases of [Thanos](https://thanos.io/), we collect them here for others to refer to. diff --git a/docs/proposals-accepted/202005-scalable-rule-storage.md b/docs/proposals-accepted/202005-scalable-rule-storage.md index 368ad4dfcc..5fa0a86cea 100644 --- a/docs/proposals-accepted/202005-scalable-rule-storage.md +++ b/docs/proposals-accepted/202005-scalable-rule-storage.md @@ -30,7 +30,7 @@ Allow scaling storage and execution of rule evaluations. Allow specifying one of the following flags: * `--remote-write` -* `--remote-write.config` or `--remote-write.config-file` flag following the same scheme as [`--query.config`, and `--query.config-file`](https://thanos.io/tip/components/rule.md/#query-api) +* `--remote-write.config` or `--remote-write.config-file` flag following the same scheme as [`--query.config`, and `--query.config-file`](../components/rule.md#query-api) * `--remote-write.tenant-label-name` which label-value to use to set the tenant to be communicated to the receive component If any of these are specified the ruler would run a stateless mode, without local storage, and instead writing samples to the configured remote server, which must implement the `storepb.WritableStore` gRPC service. diff --git a/docs/proposals-accepted/202012-receive-split.md b/docs/proposals-accepted/202012-receive-split.md index 48ef27de9b..82d6687247 100644 --- a/docs/proposals-accepted/202012-receive-split.md +++ b/docs/proposals-accepted/202012-receive-split.md @@ -75,6 +75,6 @@ This potentially makes the receiver more difficult to operate and understandable #### Flag for current Receiver: --receive-route -Idea would be similar same as in [Proposal](#Proposal), but there will be explicit flag to turn off local storage capabilities. +Idea would be similar same as in [Proposal](#proposal), but there will be explicit flag to turn off local storage capabilities. I think we can have much more understandable logic if *we simply not* configure hashring for **ingesting receivers** and not configure local hashring endpoint to notify that such Receiver instance will never store anything. diff --git a/docs/proposals-accepted/202101-endpoint-discovery.md b/docs/proposals-accepted/202101-endpoint-discovery.md index 1ee2423471..7e123da9a0 100644 --- a/docs/proposals-accepted/202101-endpoint-discovery.md +++ b/docs/proposals-accepted/202101-endpoint-discovery.md @@ -17,7 +17,7 @@ We want to propose a new flag called `--endpoint=
` that will be passed ### Motivation -Currently, in Thanos Query, the discovery of rules APIs happens via Store API's Info method. This makes it harder if we ever want to not have a coupling to the Store API (which is already planned for [scalable ruler proposal](https://github.com/thanos-io/thanos/blob/main/docs/proposals/202005_scalable-rule-storage.md)). +Currently, in Thanos Query, the discovery of rules APIs happens via Store API's Info method. This makes it harder if we ever want to not have a coupling to the Store API (which is already planned for [scalable ruler proposal](https://github.com/thanos-io/thanos/blob/main/docs/proposals-accepted/202005-scalable-rule-storage.md)). We also require passing two different flags to the Thanos Query component `--store=
` and `--rule=
`. If users use both flags, the Query component performs DNS discovery on often the same address multiple times, which can cause occasional DNS issues as this resolution happens so frequently. This is especially confusing when one DNS lookup works and the other doesn't. Adding new APIs in the future would exacerbate this issue with DNS requests. diff --git a/docs/proposals-accepted/_index.md b/docs/proposals-accepted/README.md similarity index 59% rename from docs/proposals-accepted/_index.md rename to docs/proposals-accepted/README.md index fd2a7d5cb5..17eed2e877 100644 --- a/docs/proposals-accepted/_index.md +++ b/docs/proposals-accepted/README.md @@ -1,5 +1,3 @@ ---- -title: 'Accepted Proposals:' ---- +# Accepted Proposals: List of approved, but not yet implemented proposals. diff --git a/docs/proposals-done/201909-thanos-sharding.md b/docs/proposals-done/201909-thanos-sharding.md index f3fdecbf3e..3700bd5dc5 100644 --- a/docs/proposals-done/201909-thanos-sharding.md +++ b/docs/proposals-done/201909-thanos-sharding.md @@ -86,7 +86,7 @@ On each component that works on the object storage (e.g Store GW and Compactor), ### Relabelling -Similar to [promtail](https://github.com/grafana/loki/blob/master/docs/clients/promtail/scraping.md#relabeling) this config will follow native [Prometheus relabel-config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config) syntax. +Similar to [promtail](https://grafana.com/docs/loki/latest/clients/promtail/scraping/#relabeling) this config will follow native [Prometheus relabel-config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config) syntax. The relabel config will define filtering process done on **every** synchronization with object storage. @@ -105,7 +105,7 @@ Output: By default, on empty relabel-config, all external labels are assumed. Intuitively blocks without any external labels will be ignored. -All blocks should compose as set of labels to advertise. The input should be based from original meta files. NOT from relabelling. The reasoning is covered in [`Next Steps`](#Future-Work) section +All blocks should compose as set of labels to advertise. The input should be based from original meta files. NOT from relabelling. The reasoning is covered in [`Next Steps`](#future-work) section Example usages would be: diff --git a/docs/proposals-done/201912-thanos-binary-index-header.md b/docs/proposals-done/201912-thanos-binary-index-header.md index d33dacedd0..e016ce3da3 100644 --- a/docs/proposals-done/201912-thanos-binary-index-header.md +++ b/docs/proposals-done/201912-thanos-binary-index-header.md @@ -72,7 +72,7 @@ This design is trying to address those four problems. ## No Goals * Removing initial startup for Thanos Store Gateway completely as designed in [Cortex, no initial block sync](https://github.com/thanos-io/thanos/issues/1813) - * However this proposal might be a step towards that as we might be able to load and build index-cache/index quickly on demand from disk. See [Future Work](./201912_thanos_binary_index_header.md#Future-work) + * However this proposal might be a step towards that as we might be able to load and build index-cache/index quickly on demand from disk. See [Future Work](#future-work) * At the same time be able to load `index-header` at query time directly from bucket is not a goal of this proposal. * Decreasing size. While it would be nice to use less space; our aim is latency of building/loading the block. That might be correlated with size, but not necessarily (e.g when extra considering compression) @@ -99,7 +99,7 @@ With that effort building time and resources should be compared with downloading Thanks to this format we can reuse most of the [FileReader](https://github.com/prometheus/prometheus/blob/de0a772b8e7d27dc744810a1a693d97be027049a/tsdb/index/index.go#L664) code to load file. -Thanos will build/compose all index-headers on the startup for now, however in theory we can load and build those blocks on demand. Given the minimal memory that each loaded block should take now, this is described as [Future Work](./201912_thanos_binary_index_header.md#Future-Work) +Thanos will build/compose all index-headers on the startup for now, however in theory we can load and build those blocks on demand. Given the minimal memory that each loaded block should take now, this is described as [Future Work](#future-work) ### Length of Posting to fetch @@ -107,7 +107,7 @@ While idea of combing different pieces of TSDB index as our index-header is grea We need to know apriori how to partition and how many bytes we need to fetch from the storage to get each posting: https://github.com/thanos-io/thanos/blob/7e11afe64af0c096743a3de8a594616abf52be45/pkg/store/bucket.go#L1567 -To calculate those sizes we use [`indexr.PostingsRanges()`](https://github.com/thanos-io/thanos/blob/7e11afe64af0c096743a3de8a594616abf52be45/pkg/block/index.go#L156-L155) which scans through `posting` section of the TSDB index. Having to fetch whole postings section just to get size of each posting makes this proposal less valuable as we still need to download big part of index and traverse through it instead of what we propose in [#Proposal](./201912_thanos_binary_index_header.md#proposal) +To calculate those sizes we use [`indexr.PostingsRanges()`](https://github.com/thanos-io/thanos/blob/7e11afe64af0c096743a3de8a594616abf52be45/pkg/block/index.go#L156-L155) which scans through `posting` section of the TSDB index. Having to fetch whole postings section just to get size of each posting makes this proposal less valuable as we still need to download big part of index and traverse through it instead of what we propose in [#Proposal](#proposal) For series we don't know the exact size either, however we estimate max size of each series to be 64*1024. It depends on sane number of label pairs and chunks per series. We have really only one potential case when this was too low: https://github.com/thanos-io/thanos/issues/552. Decision about series this was made here: https://github.com/thanos-io/thanos/issues/146 diff --git a/docs/proposals-done/202004-embedd-cortex-frontend.md b/docs/proposals-done/202004-embedd-cortex-frontend.md index 670d150587..212122de72 100644 --- a/docs/proposals-done/202004-embedd-cortex-frontend.md +++ b/docs/proposals-done/202004-embedd-cortex-frontend.md @@ -82,7 +82,7 @@ To make this happen we will propose a small refactor in Cortex code to avoid unn #### Don't add anything, document Cortex query frontend and add examples of usage -Unfortunately we tried this path already without success. Reasons were mentioned in [Motivation](202004_embedd_cortex_frontend.md#Motivation) +Unfortunately we tried this path already without success. Reasons were mentioned in [Motivation](#motivation) #### Add response caching to Querier itself, in the same process. diff --git a/docs/proposals-done/README.md b/docs/proposals-done/README.md new file mode 100644 index 0000000000..739bf7d56c --- /dev/null +++ b/docs/proposals-done/README.md @@ -0,0 +1,3 @@ +# Done Proposals: + +List of implemented proposals. diff --git a/docs/proposals-done/_index.md b/docs/proposals-done/_index.md deleted file mode 100644 index 806e6d7a36..0000000000 --- a/docs/proposals-done/_index.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -title: 'Done Proposals:' ---- - -List of implemented proposals. diff --git a/docs/proposals-rejected/README.md b/docs/proposals-rejected/README.md new file mode 100644 index 0000000000..3fc0a4890c --- /dev/null +++ b/docs/proposals-rejected/README.md @@ -0,0 +1,3 @@ +# Rejected Proposals: + +List of rejected proposals. diff --git a/docs/proposals-rejected/_index.md b/docs/proposals-rejected/_index.md deleted file mode 100644 index 652f92390a..0000000000 --- a/docs/proposals-rejected/_index.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -title: 'Rejected Proposals:' ---- - -List of rejected proposals. diff --git a/docs/quick-tutorial.md b/docs/quick-tutorial.md index 0906efa68f..86f4ef1abc 100644 --- a/docs/quick-tutorial.md +++ b/docs/quick-tutorial.md @@ -1,10 +1,3 @@ ---- -weight: 1 -type: docs -title: Quick Tutorial -menu: thanos ---- - # Quick Tutorial Feel free to check the free, in-browser interactive tutorial [as Katacoda Thanos Course](https://katacoda.com/thanos/courses/thanos) We will be progressively updating our Katacoda Course with more scenarios. @@ -57,7 +50,7 @@ The purpose of the Sidecar is to backup Prometheus data into an Object Storage b The Sidecar makes use of the `reload` Prometheus endpoint. Make sure it's enabled with the flag `--web.enable-lifecycle`. -[Component sidecar documentation](../components/sidecar.md) +[Component sidecar documentation](components/sidecar.md) ### External storage @@ -82,7 +75,7 @@ If you are not interested in backing up any data, the `--objstore.config-file` f ### Store API -The Sidecar component implements and exposes a gRPC *[Store API](/pkg/store/storepb/rpc.proto#L19)*. The sidecar implementation allows you to query the metric data stored in Prometheus. +The Sidecar component implements and exposes a gRPC *[Store API](https://github.com/thanos-io/thanos/blob/main/pkg/store/storepb/rpc.proto#L27)*. The sidecar implementation allows you to query the metric data stored in Prometheus. Let's extend the Sidecar in the previous section to connect to a Prometheus server, and expose the Store API. @@ -117,7 +110,7 @@ global: ## Querier/Query -Now that we have setup the Sidecar for one or more Prometheus instances, we want to use Thanos' global [Query Layer](../components/query.md) to evaluate PromQL queries against all instances at once. +Now that we have setup the Sidecar for one or more Prometheus instances, we want to use Thanos' global [Query Layer](components/query.md) to evaluate PromQL queries against all instances at once. The Query component is stateless and horizontally scalable and can be deployed with any number of replicas. Once connected to the Sidecars, it automatically detects which Prometheus servers need to be contacted for a given PromQL query. @@ -135,7 +128,7 @@ thanos query \ Go to the configured HTTP address that should now show a UI similar to that of Prometheus. If the cluster formed correctly you can now query across all Prometheus instances within the cluster. You can also check the Stores page to check up on your stores. -[Query documentation](../components/query.md) +[Query documentation](components/query.md) ### Deduplicating Data from Prometheus HA pairs @@ -204,7 +197,7 @@ The store gateway occupies small amounts of disk space for caching basic informa * *[Example Kubernetes manifest](https://github.com/thanos-io/kube-thanos/blob/master/manifests/thanos-store-statefulSet.yaml)* -[Store Gateway documentation](../components/store.md) +[Store Gateway documentation](components/store.md) ## Compactor @@ -225,10 +218,10 @@ The compactor is not in the critical path of querying or data backup. It can eit * *[Example Kubernetes manifest](https://github.com/thanos-io/kube-thanos/blob/master/examples/all/manifests/thanos-compact-statefulSet.yaml)* -[Compactor documentation](../components/compact.md) +[Compactor documentation](components/compact.md) ## Ruler/Rule -In case of Prometheus with Thanos sidecar does not have enough retention, or if you want to have alerts or recording rules that requires global view, Thanos has just the component for that: the [Ruler](../components/rule.md), which does rule and alert evaluation on top of a given Thanos Querier. +In case of Prometheus with Thanos sidecar does not have enough retention, or if you want to have alerts or recording rules that requires global view, Thanos has just the component for that: the [Ruler](components/rule.md), which does rule and alert evaluation on top of a given Thanos Querier. -[Rule documentation](../components/rule.md) +[Rule documentation](components/rule.md) diff --git a/docs/release-process.md b/docs/release-process.md index c4be87c6ad..9381ca1ff0 100644 --- a/docs/release-process.md +++ b/docs/release-process.md @@ -1,9 +1,3 @@ ---- -type: docs -title: Release Process -menu: thanos ---- - # Release Process This page describes the release cadence and process for Thanos project. diff --git a/docs/service-discovery.md b/docs/service-discovery.md index b8bd368cbc..45dfd6fcb1 100644 --- a/docs/service-discovery.md +++ b/docs/service-discovery.md @@ -1,9 +1,3 @@ ---- -type: docs -title: Service Discovery -menu: thanos ---- - # Service Discovery Service Discovery (SD) is a vital part of several Thanos components. It allows Thanos to discover API targets required to perform certain operations. diff --git a/docs/sharding.md b/docs/sharding.md index 78f3d49144..2a6bf1c7c4 100644 --- a/docs/sharding.md +++ b/docs/sharding.md @@ -1,8 +1,4 @@ ---- -type: docs -title: Sharding -menu: thanos ---- +# Sharding # Background @@ -22,7 +18,7 @@ Queries against store gateway which are touching large number of blocks (no matt # Relabelling -Similar to [promtail](https://github.com/grafana/loki/blob/master/docs/clients/promtail/configuration.md#relabel_config) this config will follow native [Prometheus relabel-config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config) syntax. +Similar to [promtail](https://grafana.com/docs/loki/latest/clients/promtail/configuration/#relabel_configs) this config will follow native [Prometheus relabel-config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config) syntax. Now, thanos only support following relabel actions: diff --git a/docs/storage.md b/docs/storage.md index cdd4cfaabe..fc974253f0 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -1,9 +1,3 @@ ---- -type: docs -title: Object Storage & Data Format -menu: thanos ---- - # Object Storage & Data Format Thanos uses object storage as primary storage for metrics and metadata related to them. In this document you can learn how to configure your object storage and what is the data layout and format for primary Thanos components that are "block" aware, like: `sidecar` `compact`, `receive` and `store gateway`. @@ -446,7 +440,7 @@ At that point, anyone can use your provider by spec. ## Data in Object Storage -Thanos supports writing and reading data in native Prometheus `TSDB blocks` in [TSDB format](https://github.com/prometheus/prometheus/tree/master/tsdb/docs/format). This is the format used by [Prometheus](https://prometheus.io) TSDB database for persisting data on the local disk. With the efficient index and [chunk](design.md/#chunk) binary formats, it also fits well to be used directly from object storage using range GET API. +Thanos supports writing and reading data in native Prometheus `TSDB blocks` in [TSDB format](https://github.com/prometheus/prometheus/tree/master/tsdb/docs/format). This is the format used by [Prometheus](https://prometheus.io) TSDB database for persisting data on the local disk. With the efficient index and [chunk](design.md#chunk) binary formats, it also fits well to be used directly from object storage using range GET API. Following sections explain this format in details with the additional files and entries that Thanos system supports. @@ -644,7 +638,7 @@ From high level it allows to find: * Label names * Label values for label name * All series labels -* Given (or all) series' chunk reference. This can be used to find [chunk](design.md/#chunk) with samples in the [chunk files](#chunks-file-format) +* Given (or all) series' chunk reference. This can be used to find [chunk](design.md#chunk) with samples in the [chunk files](#chunks-file-format) The following describes the format of the `index` file found in each block directory. It is terminated by a table of contents which serves as an entry point into the index. @@ -706,7 +700,7 @@ The section contains a sequence of the string entries, each prefixed with the st ##### Series -The section contains a sequence of series that hold the label set of the series as well as its [chunks](design.md/#chunk) within the block. The series are sorted lexicographically by their label sets. Each series section is aligned to 16 bytes. The ID for a series is the `offset/16`. This serves as the series' ID in all subsequent references. Thereby, a sorted list of series IDs implies a lexicographically sorted list of series label sets. +The section contains a sequence of series that hold the label set of the series as well as its [chunks](design.md#chunk) within the block. The series are sorted lexicographically by their label sets. Each series section is aligned to 16 bytes. The ID for a series is the `offset/16`. This serves as the series' ID in all subsequent references. Thereby, a sorted list of series IDs implies a lexicographically sorted list of series label sets. ``` ┌───────────────────────────────────────┐ @@ -720,9 +714,9 @@ The section contains a sequence of series that hold the label set of the series └───────────────────────────────────────┘ ``` -Every series entry first holds its number of labels, followed by tuples of symbol table references that contain the label name and value. The label pairs are lexicographically sorted. After the labels, the number of indexed [chunks](design.md/#chunk) is encoded, followed by a sequence of metadata entries containing the chunks minimum (`mint`) and maximum (`maxt`) timestamp and a reference to its position in the chunk file. The `mint` is the time of the first sample and `maxt` is the time of the last sample in the chunk. Holding the time range data in the index allows dropping chunks irrelevant to queried time ranges without accessing them directly. +Every series entry first holds its number of labels, followed by tuples of symbol table references that contain the label name and value. The label pairs are lexicographically sorted. After the labels, the number of indexed [chunks](design.md#chunk) is encoded, followed by a sequence of metadata entries containing the chunks minimum (`mint`) and maximum (`maxt`) timestamp and a reference to its position in the chunk file. The `mint` is the time of the first sample and `maxt` is the time of the last sample in the chunk. Holding the time range data in the index allows dropping chunks irrelevant to queried time ranges without accessing them directly. -`mint` of the first [chunk](design.md/#chunk) is stored, it's `maxt` is stored as a delta and the `mint` and `maxt` are encoded as deltas to the previous time for subsequent chunks. Similarly, the reference of the first chunk is stored and the next ref is stored as a delta to the previous one. +`mint` of the first [chunk](design.md#chunk) is stored, it's `maxt` is stored as a delta and the `mint` and `maxt` are encoded as deltas to the previous time for subsequent chunks. Similarly, the reference of the first chunk is stored and the next ref is stored as a delta to the previous one. ``` ┌──────────────────────────────────────────────────────────────────────────┐ @@ -888,7 +882,7 @@ The table of contents serves as an entry point to the entire index and points to The following describes the format of a chunks file, which is created in the `chunks/` directory of a block. The maximum size per segment file is 512MiB. -[Chunks](design.md/#chunk) in the files are referenced from the index by uint64 composed of in-file offset (lower 4 bytes) and segment sequence number (upper 4 bytes). +[Chunks](design.md#chunk) in the files are referenced from the index by uint64 composed of in-file offset (lower 4 bytes) and segment sequence number (upper 4 bytes). ``` ┌──────────────────────────────┐ diff --git a/docs/tracing.md b/docs/tracing.md index 37eedc02af..458b3a3cee 100644 --- a/docs/tracing.md +++ b/docs/tracing.md @@ -1,9 +1,3 @@ ---- -type: docs -title: Tracing -menu: thanos ---- - # Tracing Thanos supports different tracing backends that implements `opentracing.Tracer` interface. diff --git a/scripts/website/contentpreprocess.sh b/scripts/website/contentpreprocess.sh deleted file mode 100755 index 470c922adc..0000000000 --- a/scripts/website/contentpreprocess.sh +++ /dev/null @@ -1,114 +0,0 @@ -#!/usr/bin/env bash - -OUTPUT_CONTENT_DIR=$1 -TOP_WEIGHT=$2 -COMMIT_SHA=$(git rev-parse HEAD) - -echo ">> preprocessing content of dir ${OUTPUT_CONTENT_DIR}" - -# Add headers to special CODE_OF_CONDUCT.md, CONTRIBUTING.md and CHANGELOG.md files. -echo "$( - cat <${OUTPUT_CONTENT_DIR}/CODE_OF_CONDUCT.md -cat CODE_OF_CONDUCT.md >>${OUTPUT_CONTENT_DIR}/CODE_OF_CONDUCT.md - -echo "$( - cat <${OUTPUT_CONTENT_DIR}/CONTRIBUTING.md -cat CONTRIBUTING.md >>${OUTPUT_CONTENT_DIR}/CONTRIBUTING.md - -echo "$( - cat <${OUTPUT_CONTENT_DIR}/CHANGELOG.md -cat CHANGELOG.md >>${OUTPUT_CONTENT_DIR}/CHANGELOG.md - -echo "$( - cat <${OUTPUT_CONTENT_DIR}/MAINTAINERS.md -cat MAINTAINERS.md >>${OUTPUT_CONTENT_DIR}/MAINTAINERS.md - -echo "$( - cat <${OUTPUT_CONTENT_DIR}/SECURITY.md -cat SECURITY.md >>${OUTPUT_CONTENT_DIR}/SECURITY.md - -# Move root things to "thanos" and "contributing" dir for easy organizing. -mkdir -p ${OUTPUT_CONTENT_DIR}/thanos/ -mv ${OUTPUT_CONTENT_DIR}/*.md ${OUTPUT_CONTENT_DIR}/thanos/ -mv ${OUTPUT_CONTENT_DIR}/thanos/CONTRIBUTING.md ${OUTPUT_CONTENT_DIR}/contributing/CONTRIBUTING.md -mv ${OUTPUT_CONTENT_DIR}/thanos/CODE_OF_CONDUCT.md ${OUTPUT_CONTENT_DIR}/contributing/CODE_OF_CONDUCT.md -# This is needed only for v0.13-0.16 versions. -mv ${OUTPUT_CONTENT_DIR}/thanos/community.md ${OUTPUT_CONTENT_DIR}/contributing/community.md - -# Create an _index.md in Thanos dir. -echo "$( - cat <${OUTPUT_CONTENT_DIR}/thanos/_index.md - -# Create an _index.md in this dir to enable sorting capabilities and make this version appear top in version picker -echo "$( - cat <${OUTPUT_CONTENT_DIR}/_index.md - -# Add edit footer to all markdown files assumed as content. -ALL_DOC_CONTENT_FILES=$(echo "${OUTPUT_CONTENT_DIR}/**/*.md") -for file in ${ALL_DOC_CONTENT_FILES}; do - relFile=${file##${OUTPUT_CONTENT_DIR}/} - echo "$( - cat <>${file} - -done - -# All the absolute links are replaced with a direct link to the file on github, including the current commit SHA. -perl -pi -e 's/]\(\//]\(https:\/\/github.com\/thanos-io\/thanos\/tree\/'${COMMIT_SHA}'\//g' ${ALL_DOC_CONTENT_FILES} - -# All the relative links needs to have ../ This is because Hugo is missing: https://github.com/gohugoio/hugo/pull/3934 -perl -pi -e 's/]\((?!http)/]\(..\//g' ${ALL_DOC_CONTENT_FILES} -# All the relative links in src= needs to have ../ as well. -perl -pi -e 's/src=\"(?!http)/src=\"..\//g' ${ALL_DOC_CONTENT_FILES} diff --git a/scripts/website/mdoxpostprocess.sh b/scripts/website/mdoxpostprocess.sh new file mode 100755 index 0000000000..01783c7641 --- /dev/null +++ b/scripts/website/mdoxpostprocess.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +OUTPUT_CONTENT_DIR=$1 +TOP_WEIGHT=$2 +COMMIT_SHA=$(git rev-parse HEAD) + +echo ">> postprocessing content of dir ${OUTPUT_CONTENT_DIR}" + +# Create an _index.md in this dir to enable sorting capabilities and make this version appear top in version picker +echo "$( + cat <${OUTPUT_CONTENT_DIR}/_index.md + +# Create a thanos/_index.md file to make sure links work. +echo "$( + cat <${OUTPUT_CONTENT_DIR}/thanos/_index.md diff --git a/scripts/website/websitepreprocess.sh b/scripts/website/websitepreprocess.sh index f7e11060ce..18359b32ab 100755 --- a/scripts/website/websitepreprocess.sh +++ b/scripts/website/websitepreprocess.sh @@ -5,23 +5,31 @@ RELEASE_FILTER_RE="release-(0|[1-9]\d*)\.(0|[1-9]\d*)$" WEBSITE_DIR="website" ORIGINAL_CONTENT_DIR="docs" -OUTPUT_CONTENT_DIR="${WEBSITE_DIR}/docs-pre-processed" FILES="${WEBSITE_DIR}/docs-pre-processed/*" +MDOX_TIP_CONFIG=".mdox.yaml" +MDOX_PREV_CONFIG=".mdox.prev-release.yaml" + +# Support gtar and ggrep on OSX (installed via brew), falling back to tar and grep. On Linux +# systems gtar or ggrep won't be installed, so will use tar and grep as expected. +TAR=$(which gtar 2>/dev/null || which tar) +GREP=$(which ggrep 2>/dev/null || which grep) + +# Exported for use in .mdox.prev-release.yaml +export OUTPUT_CONTENT_DIR="${WEBSITE_DIR}/docs-pre-processed" git remote add upstream https://github.com/thanos-io/thanos.git git remote add origin https://github.com/thanos-io/thanos.git git remote -v git fetch origin -RELEASE_BRANCHES=$(git branch --all | grep -P "remotes/origin/${RELEASE_FILTER_RE}" | egrep --invert-match '(:?HEAD|main)$' | sort -V) +RELEASE_BRANCHES=$(git branch --all | $GREP -P "remotes/origin/${RELEASE_FILTER_RE}" | egrep --invert-match '(:?HEAD|main)$' | sort -V) echo ">> chosen $(echo ${RELEASE_BRANCHES}) releases to deploy docs from" +# preprocess tip separately rm -rf ${OUTPUT_CONTENT_DIR} -mkdir -p "${OUTPUT_CONTENT_DIR}/tip" - -# Copy original content from current state first. -cp -r ${ORIGINAL_CONTENT_DIR}/* "${OUTPUT_CONTENT_DIR}/tip" -scripts/website/contentpreprocess.sh "${OUTPUT_CONTENT_DIR}/tip" 100000 +PATH=$PATH:$GOBIN +$MDOX transform --log.level=debug --config-file=$MDOX_TIP_CONFIG +scripts/website/mdoxpostprocess.sh "${OUTPUT_CONTENT_DIR}/tip" 100000 #create variable for weight value WEIGHT_VALUE=0 @@ -29,9 +37,12 @@ WEIGHT_VALUE=0 for branchRef in ${RELEASE_BRANCHES}; do WEIGHT_VALUE=$((WEIGHT_VALUE + 1)) branchName=${branchRef##*/} - tags=${branchName/release-/v} + # Exported for use in .mdox.prev-release.yaml + export tags=${branchName/release-/v} echo ">> cloning docs for versioning ${tags}" - mkdir -p "${OUTPUT_CONTENT_DIR}/${tags}" - git archive --format=tar "refs/${branchRef}" | tar -C${OUTPUT_CONTENT_DIR}/${tags} -x "docs/" --strip-components=1 - scripts/website/contentpreprocess.sh "${OUTPUT_CONTENT_DIR}/${tags}" ${WEIGHT_VALUE} + mkdir -p "${OUTPUT_CONTENT_DIR}/${tags}-git-docs" + git archive --format=tar "refs/${branchRef}" | $TAR -C${OUTPUT_CONTENT_DIR}/${tags}-git-docs -x "docs/" --strip-components=1 + $MDOX transform --log.level=debug --config-file=$MDOX_PREV_CONFIG + scripts/website/mdoxpostprocess.sh "${OUTPUT_CONTENT_DIR}/${tags}" ${WEIGHT_VALUE} + rm -rf ${OUTPUT_CONTENT_DIR}/${tags}-git-docs done