Add semantic search ingestion and throughput blog

.github/vale/styles/OpenSearch/HeadingCapitalization.yml

@@ -2,4 +2,9 @@ extends: capitalization
 message: "'%s' is a heading and should be in sentence case."
 level: error
 scope: heading
-match: $sentence
+match: $sentence
+threshold: 1
+indicators:
+  - ":"
+exceptions:
+  - k    # ignores lowercase k-NN

.github/vale/styles/OpenSearch/Inclusive.yml

@@ -7,8 +7,6 @@ swap:
   'abort': stop, end, or cancel
   'black day': blocked day
   'blacklist': deny list
-  'execute': run, process, apply, start, launch, or initiate
-  'hang': stop responding
   'kill': stop, end, clear, remove, or cancel
   'master account': 'management account'
   'master': cluster manager

.github/vale/styles/OpenSearch/SignatureV4.yml

@@ -0,0 +1,10 @@
+extends: substitution
+message: "'%s': Use 'AWS Signature Version 4' instead of '%s' on first appearance. Then, Signature Version 4 may be used. Only use SigV4 when space is limited."
+ignorecase: true
+link: 'https://github.com/opensearch-project/documentation-website/blob/main/TERMS.md'
+level: warning
+action:
+  name: replace
+swap:
+  sigv4: Signature Version 4
+  AWS sigv4: AWS Signature Version 4

.github/vale/styles/OpenSearch/SubstitutionsError.yml

@@ -10,9 +10,13 @@ swap:
   'data set': dataset
   'for information on': for information about
   'for more information on': for more information about
+  'geo bounds': geobounds
+  'geo distance': geodistance
   'geo hash': geohash
+  'geo hex': geohex
   'geo point': geopoint
   'geo shape': geoshape
+  'geo tile': geotile
   'geospacial': geospatial
   'hard code': hardcode
   'high-performance computing': high performance computing
@@ -43,4 +47,5 @@ swap:
   'U.S.': US
   'web page': webpage
   'web site': website
+  'whitespace': white space
   'user interface \(UI\)': UI

.github/vale/styles/OpenSearch/TableHeadings.yml

@@ -3,3 +3,5 @@ message: "'%s' is a table heading and should be in sentence case."
 level: error
 scope: table.header
 match: $sentence
+exceptions:
+  - k    # ignores lowercase k-NN

.github/vale/styles/Vocab/OpenSearch/Plugins/accept.txt

@@ -2,6 +2,7 @@ Alerting plugin
 Anomaly Detection plugin
 Asynchronous Search plugin
 Cross-Cluster Replication plugin
+Crypto plugin
 Maps plugin
 Notebooks plugin
 Notifications plugin

.github/vale/styles/Vocab/OpenSearch/Products/accept.txt

@@ -36,6 +36,7 @@ ElastiFlow
 Eliatra
 Faiss
 Filebeat
+Flickr
 Flink
 Fluent Bit
 Fluentd
@@ -105,6 +106,7 @@ Okta
 Open Distro
 OpenID Connect
 OpenSearch
+OpenSearch Benchmark
 OpenSearch Dashboards
 OpenSearch Project
 OpenSearch Service
@@ -143,7 +145,6 @@ Searchium
 Searchium.ai
 Sease
 Serverless
-SigV4 
 Simple Schema for Observability
 SnappyFlow
 SquareShift

.github/vale/styles/Vocab/OpenSearch/Words/accept.txt

@@ -4,9 +4,10 @@
 [Aa]utopopulate
 [Aa]utoscaler?
 [Bb]ackoff
-[Bb]ackport
+[Bb]ackport(s|ing)?
 [Bb]ackpressure
 [Bb]asemap
+[Bb]enchmarked
 [Bb]igram
 Boolean
 [Cc]allout
@@ -16,20 +17,24 @@ Boolean
 [Cc]onfig
 [Cc]ron
 [Cc]ybersecurity
+[Dd]ashboarding
 [Dd]atagram
 [Dd]eallocate
 [Dd]eduplication
 [Dd]eserialize
 [Dd]eserializing
 [Dd]eserialization
 Dev
+[Dd]iscoverability
 Distro
+[Dd]uplicative
 [Ee]gress
 [Ee]num
 [Ff]ailover
 [Ff]lyout
 [Ff]sync
 Gantt
+[Gg]eobounds
 [Gg]eodistance
 [Gg]eohash
 GeoHex
@@ -53,16 +58,19 @@ k-NN
 [Kk]eystore
 kibibyte
 mebibyte
+[Ll]earnings
 [Ll]emmatization
 Levenshtein
 [Ll]inestring
 [Ll]ookups?
 [Ll]ossy
 [Ll]owercases?d?
+[Mm]acrobenchmarks?
 [Mm]isorder
 [Mm]odularization
 [Mm]ultifield
 [Mm]ultiline
+[Mm]ultimodal
 [Mm]ultipoint
 [Mm]ultipolygon
 [Mm]ultithreaded
@@ -80,6 +88,7 @@ pebibyte
 [Pp]repper
 [Pp]reprocess
 [Pp]retrain
+[Pp]seudocode
 [Rr]ebalance
 [Rr]ebalancing
 [Rr]edownload
@@ -88,19 +97,23 @@ pebibyte
 [Rr]eindex
 [Rr]eingest
 [Rr]epo
-[Rr]erank
+[Rr]erank(er|ed|ing)?
 [Rr]erankers?
 [Rr]ewriters?
+[Rr]ollout
 [Rr]ollup
 [Rr]unbooks?
 [Ss]erverless
+[Ss]harding
 [Ss]ignificand
 stdout
 [Ss]ubaggregation
 [Ss]ubcalculation
+[Ss]ubcollectors?
 [Ss]ubcommands?
 [Ss]ubfield
-[Ss]ubquery
+[Ss]ubquer(y|ies)
+[Ss]ubstrings?
 [Ss]ubtag
 [Ss]ubtree
 [Ss]ubvector

.github/workflows/jekyll-build.yml

@@ -13,4 +13,10 @@ jobs:
         ruby-version: '3.0'
         bundler-cache: true
     - run: |
+        malformed_event_dates=$(grep -Er --files-without-match '^eventdate: [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2} [-+][0-9]{4}$' _events)
+        if [ -n "${malformed_event_dates}" ]; then
+          echo "Malformed event date in:" >&2
+          echo "${malformed_event_dates}" >&2
+          exit 1
+        fi
         bundle exec jekyll build --future

.pre-commit-config.yaml

@@ -0,0 +1,17 @@
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v2.3.0
+    hooks:
+    -   id: check-yaml
+    -   id: end-of-file-fixer
+    -   id: trailing-whitespace
+
+- repo: local
+  hooks:
+    - id: malformed-event-dates
+      name: Malformed event date check
+      entry: ./_scripts/_malformeddates.sh
+      language: system
+      types: [file]
+      files: \.(markdown|md)$
+      pass_filenames: false

.vale.ini

@@ -48,6 +48,7 @@ OpenSearch.RolloverNoun = YES
 OpenSearch.RolloverVerb = YES
 OpenSearch.SetupNoun = YES
 OpenSearch.SetupVerb = YES
+OpenSearch.SignatureV4 = YES
 OpenSearch.SpacingPunctuation = YES
 OpenSearch.SpacingSlash = YES
 OpenSearch.SpacingWords = YES

.whitesource

@@ -0,0 +1,33 @@
+{
+  "scanSettings": {
+    "configMode": "AUTO",
+    "configExternalURL": "",
+    "projectToken": "",
+    "baseBranches": []
+  },
+  "scanSettingsSAST": {
+    "enableScan": false,
+    "scanPullRequests": false,
+    "incrementalScan": true,
+    "baseBranches": [],
+    "snippetSize": 10
+  },
+  "checkRunSettings": {
+    "vulnerableCheckRunConclusionLevel": "failure",
+    "displayMode": "diff",
+    "useMendCheckNames": true
+  },
+  "checkRunSettingsSAST": {
+    "checkRunConclusionLevel": "failure",
+    "severityThreshold": "high"
+  },
+  "issueSettings": {
+    "minSeverityLevel": "LOW",
+    "issueType": "DEPENDENCY"
+  },
+  "remediateSettings": {
+    "workflowRules": {
+      "enabled": true
+    }
+  }
+}

BLOG_GUIDE.md

@@ -0,0 +1,127 @@
+# Blog Guide
+
+This guide provides an overview of formatting blogs for the project website. It references the [semantic blog post](/_posts/2023-03-30-semantic-science-benchmarks.md) as an example. You can [view the post](https://opensearch.org/blog/semantic-science-benchmarks/) in its rendered form on the project website.
+
+* * *
+
+### Table of contents
+
+* [Adding a blog post](#adding-a-blog-post)
+* [Front matter](#front-matter)
+* [Title](#title)
+* [Tables](#tables)
+* [Images](#images)
+* [Authors](#authors)
+* * *
+
+## Adding a blog post
+
+To add a new blog post, create a new `.md` file in the `_posts` directory. The name of the file must start with the date in the format `yyyy-MM-dd` (for example, `2023-03-30-semantic-science-benchmarks.md`).
+
+## Front matter
+
+Every post must start with front matter in YAML format, for example:
+
+```
+---
+layout: post
+title:  "The ABCs of semantic search in OpenSearch: Architectures, benchmarks, and combination strategies"
+authors:
+ - mshyani
+ - dhrubo
+ - nmishra
+ - kolchfa
+date: 2023-03-30
+categories:
+ - technical-post
+meta_keywords: semantic search engine, neural search engine, keyword and natural language search, search relevance, benchmarking tests
+meta_description: Learn how to create a semantic search engine in OpenSearch, including architecture and model options, benchmarking tests, and effects of different combination strategies and normalization protocols.
+excerpt: In an earlier blog post, we described different ways of building a semantic search engine in OpenSearch. In this post, we'll dive further into the science behind it. We'll discuss the benefits of combining keyword-based search with neural search, the architecture and model options, and benchmarking tests and results. First, we'll provide an overview of our proposed solutions and a summary of the main results. Next, we'll outline the steps for creating a solution and fine-tuning it for your own document corpus. Finally, we'll discuss the effects of different combination strategies and normalization protocols on search relevance. 
+has_math: true
+has_science_table: true
+---
+```
+
+The following table describes all front matter variables.
+
+Variable | Description
+:--- | :---
+`layout` | The page layout. Must be `post`.
+`title` | The blog title.
+`authors` | A list of authors' short names.
+`date` | The publish date of the blog post. To test your blog by running the project website locally, this date must be no later than the current date. You can change this date to the publish date once the blog post has been reviewed and is ready for publishing.
+`categories` | A list of categories to which the blog post applies. Common options are `community-updates`, `technical-posts`, `update`, `community`, `feature`, `updates`, `partners`, and `releases`.
+`meta_keywords` | Meta keywords are provided by the marketing team once you put up a PR with the blog post.
+`meta_description` | Meta keywords are provided by the marketing team once you put up a PR with the blog post.
+`excerpt` | (Optional) A blog excerpt you want to appear on the [blog front page](https://opensearch.org/blog). If you don't provide this variable, the excerpt will contain the first paragraph of the blog. If you do provide your own excerpt, make sure it does not contain any special Markdown formatting because this formatting will be ignored and displayed as is. For example, if you surround a word with tic marks, the tic marks will be displayed rather than formatting the variable in code font.
+`has_math` | (Optional) If your blog post contains mathematical formulas, set this variable to `true` so you can use the [MathJax](https://www.mathjax.org/) syntax to render the formulas.
+`has_science_table` | (Optional) By default, tables do not render grid lines. To add grid lines to your table, set this variable to `true`.
+
+## Title
+
+The blog title is sourced from the `title` variable in the front matter. Do not include a heading after the front matter; start with the introductory paragraph directly. If you include a heading after the front matter, the heading will be displayed twice.
+
+## Tables
+
+You can use either Markdown or HTML syntax for tables in your blog post. Markdown tables do not support row or column span. By default, tables do not render grid lines. To add grid lines to your table, include `has_science_table: true` in the front matter of the blog post.
+
+## Images
+
+If your blog post contains images, add a folder containing the images in the `assets/media/blog-images` directory. Name the folder the same as you named the blog file except for the file extension. For example, if your blog file is named `2023-03-30-semantic-science-benchmarks.md`, name the image folder `2023-03-30-semantic-science-benchmarks` You can use either Markdown or HTML syntax for images. By default, images include a standard border and are responsive. 
+
+To insert a Markdown image, use the `![<alternate text>](link)` syntax:
+
+```
+![Similar vectors](/assets/media/blog-images/2023-02-13-semantic-search-solutions/vectors.jpg)
+```
+
+If you want to specify the image width or another style, use HTML syntax:
+
+```
+<img src="/assets/media/blog-images/2023-02-13-semantic-search-solutions/vectors.jpg" alt="Similar vectors" width="700"/>
+```
+
+To center a Markdown image, specify the `img-centered` class for the image:
+
+```
+![Similar vectors](/assets/media/blog-images/2023-02-13-semantic-search-solutions/vectors.jpg){:class="img-centered"}
+```
+
+To center an HTML image, include `class="centered"` in the image tag:
+
+```
+<img src="/assets/media/blog-images/2023-02-13-semantic-search-solutions/vectors.jpg" alt="Similar vectors" class="img-centered"/>
+```
+
+## Authors
+
+For each author, add an author [bio](#author-bio) and [picture](#author-picture).
+
+### Author bio
+
+For an author bio, create an author file in the `_authors` directory. The name the author file must be the same as the variable you are referencing in the `authors` list of the blog front matter. For example, suppose your front matter contains the following author list:
+
+```
+authors:
+ - krisfreedain
+```
+
+In this example, the file name is `krisfreedain.markdown`. The author file contains the following front matter and bio:
+
+```
+---
+short_name: krisfreedain
+name: Kris Freedain
+photo: '/assets/media/authors/krisfreedain.jpg'
+twitter: 'KrisFreedain'
+github: krisfreedain
+linkedin: 'krisfreedain'
+---
+**Kris Freedain** is the OpenSearch Project Community Manager; his hobbies include gardening, garage gym powerlifting, and meditation.
+```
+
+The Twitter, GitHub, and LinkedIn profile names are optional.
+
+### Author picture
+
+Add an author picture file in the `assets/media/authors` directory. The name of the file must be the same as the `short_name` variable in the author file front matter. Common image file extensions are `.png` and `.jpg`. In this example, the file name is `krisfreedain.jpg`.

CODE_OF_CONDUCT.md

@@ -17,6 +17,7 @@ This code of conduct applies to all spaces provided by the OpenSource project in
 * Publishing private information, such as physical or electronic address, without permission;
 * Other conduct which could reasonably be considered inappropriate in a professional setting;
 * Advocating for or encouraging any of the above behaviors.
-* Enforcement and Reporting Code of Conduct Issues:
+
+**Enforcement and Reporting Code of Conduct Issues:**
 
-Instances of abusive, harassing, or otherwise unacceptable behavior may be reported. [Contact us](mailto:[email protected]). All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances.
+Instances of abusive, harassing, or otherwise unacceptable behavior may be reported. [Contact us](mailto:[email protected]). All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances.