diff --git a/.gitattributes b/.gitattributes
new file mode 100644
index 0000000..ab02d70
--- /dev/null
+++ b/.gitattributes
@@ -0,0 +1,3 @@
+
+Dockerfile text eol=lf
+*.sh eol=lf
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 221813d..e5c93da 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -3,120 +3,23 @@ name: CI
on: [pull_request]
jobs:
-
build-test:
- runs-on: windows-latest
- env:
- api_keyman_com_host: http://127.0.0.1:8888
- api_keyman_com_mssql_pw: Password1!
- api_keyman_com_mssql_user: sa
- api_keyman_com_mssqlconninfo: sqlsrv:Server=(local)\KEYMANAPI; Database=
- api_keyman_com_mssql_create_database: true
- api_keyman_com_mssqldb: keyboards
- KEYMANHOSTS_TIER: TIER_TEST
+ runs-on: ubuntu-latest
steps:
-
- name: Checkout
- uses: actions/checkout@v2
+ uses: actions/checkout@v3
- - name: Setup PHP 7.4
- uses: shivammathur/setup-php@6972aed899fa2dd4016a7e314c46e6902bcafb7b
- with:
- php-version: '7.4'
- extensions: curl, intl, mbstring, openssl
- coverage: none
+ - name: Build the Docker image
+ shell: bash
+ run: |
+ echo "TIER_TEST" > tier.txt
+ ./build.sh build start
env:
fail-fast: true
- #
- # Configure IIS and setup site for running unit tests
- # * Installs IIS, CGI extensions, URLRewrite and configures to connect to PHP
- # * Sets up http://127.0.0.1:8888 as host for tests
- # * Enables detailed error reporting
- #
- - name: Download and install IIS and setup a local website
- shell: powershell
- run: |
- Enable-WindowsOptionalFeature -Online -FeatureName IIS-WebServerRole -NoRestart
- Enable-WindowsOptionalFeature -Online -FeatureName IIS-CGI -NoRestart
- Enable-WindowsOptionalFeature -Online -FeatureName IIS-ISAPIExtensions -NoRestart
- Enable-WindowsOptionalFeature -Online -FeatureName IIS-ISAPIFilter -NoRestart
- Choco-Install -PackageName urlrewrite -ArgumentList "--no-progress"
- Import-Module WebAdministration
- New-WebAppPool -name "NewWebSiteAppPool" -force
- New-WebSite -name "NewWebSite" -PhysicalPath "$ENV:GITHUB_WORKSPACE" -ApplicationPool "NewWebSiteAppPool" -port 8888 -force
- Set-WebConfigurationproperty -filter "system.webServer/httpErrors" -pspath "MACHINE/WEBROOT/APPHOST" -name errorMode -value Detailed
-
- #
- # This step configures FastCGI according to the documentation at https://www.php.net/manual/en/install.windows.manual.php
- # This alternative doesn't work: New-WebHandler -name "PHP" -Path *.php -Modules FastCgiModule -ScriptProcessor "c:\tools\php\php-cgi.exe" -Verb 'GET,POST' -Force
- #
- - name: Setup FastCGI
- shell: cmd
- run: |
- set phpdir=c:\tools
- set phppath=php
-
- REM Clear current PHP handlers
- %windir%\system32\inetsrv\appcmd clear config /section:system.webServer/fastCGI
- %windir%\system32\inetsrv\appcmd set config /section:system.webServer/handlers /-[name='PHP_via_FastCGI']
-
- REM Set up the PHP handler
- %windir%\system32\inetsrv\appcmd set config /section:system.webServer/fastCGI /+[fullPath='%phpdir%\%phppath%\php-cgi.exe']
- %windir%\system32\inetsrv\appcmd set config /section:system.webServer/handlers /+[name='PHP_via_FastCGI',path='*.php',verb='*',modules='FastCgiModule',scriptProcessor='%phpdir%\%phppath%\php-cgi.exe',resourceType='Unspecified']
- %windir%\system32\inetsrv\appcmd set config /section:system.webServer/handlers /accessPolicy:Read,Script
-
- REM Configure FastCGI Variables
- %windir%\system32\inetsrv\appcmd set config -section:system.webServer/fastCgi /[fullPath='%phpdir%\%phppath%\php-cgi.exe'].instanceMaxRequests:10000
- %windir%\system32\inetsrv\appcmd.exe set config -section:system.webServer/fastCgi /+"[fullPath='%phpdir%\%phppath%\php-cgi.exe'].environmentVariables.[name='PHP_FCGI_MAX_REQUESTS',value='10000']"
- %windir%\system32\inetsrv\appcmd.exe set config -section:system.webServer/fastCgi /+"[fullPath='%phpdir%\%phppath%\php-cgi.exe'].environmentVariables.[name='PHPRC',value='%phpdir%\%phppath%\php.ini']"
-
- #
- # Write environment to localenv.php (so PHP under IIS can see it as env is not available)
- #
- - name: Setup localenv.php
- shell: cmd
- run: |
- echo ^ tools\db\localenv.php
- echo $mssqlpw='%api_keyman_com_mssql_pw%'; >> tools\db\localenv.php
- echo $mssqluser='%api_keyman_com_mssql_user%'; >> tools\db\localenv.php
- echo $mssqldb = '%api_keyman_com_mssqldb%'; >> tools\db\localenv.php
- echo $mssqlconninfo='%api_keyman_com_mssqlconninfo%'; >> tools\db\localenv.php
- echo $mssql_create_database = true; >> tools\db\localenv.php
-
- #
- # Install SQL Server Developer Edition with FullText Search module
- #
- - name: Download and install SQL Server Express
- shell: powershell
- run: |
- Choco-Install -PackageName sql-server-2019 -ArgumentList "--no-progress", "--params", "'/IGNOREPENDINGREBOOT /INSTANCEID=KEYMANAPI /INSTANCENAME=KEYMANAPI /SAPWD=Password1! /SECURITYMODE=SQL /UPDATEENABLED=FALSE /FEATURES=SQLENGINE,FULLTEXT'"
-
- #
- # Install website PHP dependencies
- #
- - name: Install dependencies
- shell: cmd
- run: composer install --no-progress
-
- #
- # Install PHP SQL Server PDO Driver
- # * Copy driver to PHP extensions
- # * Configure extension
- # * Allow PHP errors to be displayed
- #
- - name: Install PHP SQL Server PDO Driver
- shell: powershell
+ - name: Run tests
+ shell: bash
run: |
- Invoke-WebRequest -outfile pdo.zip https://github.com/microsoft/msphpsql/releases/download/v5.8.0/Windows-7.4.zip
- Expand-Archive pdo.zip -DestinationPath pdo\
- copy pdo\Windows-7.4\x64\php_pdo_sqlsrv_74_nts.dll c:\tools\php\ext\
- Add-Content -path c:\tools\php\php.ini -value '','extension=php_pdo_sqlsrv_74_nts.dll','mssql.secure_connection=Off','display_errors = On','upload_tmp_dir = c:\tools\php\tmp'
+ ./build.sh test
- #
- # Finally, run the unit tests!
- #
- - name: Run unit tests
- shell: cmd
- run: composer test
diff --git a/.github/workflows/cleanup.yml b/.github/workflows/cleanup.yml
new file mode 100644
index 0000000..6d9b8f9
--- /dev/null
+++ b/.github/workflows/cleanup.yml
@@ -0,0 +1,57 @@
+name: Prune images
+on:
+ workflow_run:
+ workflows: [Docker-build]
+ types: [completed]
+
+jobs:
+ prune:
+ runs-on: ubuntu-latest
+ steps:
+ - name: prune-staging-api
+ uses: vlaurin/action-ghcr-prune@v0.5.0
+ with:
+ token: ${{ secrets.GITHUB_TOKEN }}
+ organization: keymanapp
+ container: api-keyman-com-app
+ dry-run: true
+ keep-younger-than: 7
+ keep-last: 3
+ prune-tags-regexes: staging
+ prune-untagged: true
+
+ - name: prune-staging-db
+ uses: vlaurin/action-ghcr-prune@v0.5.0
+ with:
+ token: ${{ secrets.GITHUB_TOKEN }}
+ organization: keymanapp
+ container: api-keyman-com-db
+ dry-run: true
+ keep-younger-than: 7
+ keep-last: 3
+ prune-tags-regexes: staging
+ prune-untagged: true
+
+ - name: prune-production-api
+ uses: vlaurin/action-ghcr-prune@v0.5.0
+ with:
+ token: ${{ secrets.GITHUB_TOKEN }}
+ organization: keymanapp
+ container: api-keyman-com-app
+ dry-run: true
+ keep-younger-than: 7
+ keep-last: 3
+ prune-tags-regexes: master
+ prune-untagged: true
+
+ - name: prune-production-db
+ uses: vlaurin/action-ghcr-prune@v0.5.0
+ with:
+ token: ${{ secrets.GITHUB_TOKEN }}
+ organization: keymanapp
+ container: api-keyman-com-db
+ dry-run: true
+ keep-younger-than: 7
+ keep-last: 3
+ prune-tags-regexes: master
+ prune-untagged: true
\ No newline at end of file
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
new file mode 100644
index 0000000..6301426
--- /dev/null
+++ b/.github/workflows/docker.yml
@@ -0,0 +1,84 @@
+name: Docker-build
+on:
+ push:
+ branches: [ "master", "staging" ]
+ paths:
+ - .github/**
+ - composer.*
+ - resources/**
+ - "*Dockerfile"
+env:
+ REGISTRY: ghcr.io
+ IMAGE_NAME: keymanapp/api-keyman-com
+
+jobs:
+ build:
+
+ runs-on: ubuntu-latest
+ permissions:
+ contents: read
+ packages: write
+
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v3
+
+ # Workaround: https://github.com/docker/build-push-action/issues/461
+ - name: Setup Docker buildx
+ uses: docker/setup-buildx-action@v2
+
+ # Login against a Docker registry except on PR
+ - name: Log into registry ${{ env.REGISTRY }}
+ if: github.event_name != 'pull_request'
+ uses: docker/login-action@v2
+ with:
+ registry: ${{ env.REGISTRY }}
+ username: ${{ github.actor }}
+ password: ${{ secrets.GITHUB_TOKEN }}
+
+ # Extract metadata (tags, labels) for PHP container
+ - name: Extract Docker metadata for PHP runtime
+ id: meta-app
+ uses: docker/metadata-action@v4
+ with:
+ images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-app
+ tags: |
+ type=raw,value={{branch}}
+ type=raw,value=latest
+ labels: |
+ org.opencontainers.image.description=PHP api runtime
+
+ # Extract metadata (tags, labels) for sqlserver container
+ - name: Extract Docker metadata for SQLServer container
+ id: meta-db
+ uses: docker/metadata-action@v4
+ with:
+ images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-db
+ tags: |
+ type=raw,value={{branch}}
+ type=raw,value=latest
+ labels: |
+ org.opencontainers.image.description=SQLServer
+
+ # Build and push Docker image with Buildx (don't push on PR)
+ - name: Build and push Docker image for PHP runtime
+ uses: docker/build-push-action@v4
+ with:
+ context: .
+ push: ${{ github.event_name != 'pull_request' }}
+ tags: ${{ steps.meta-app.outputs.tags }}
+ labels: ${{ steps.meta-app.outputs.labels }}
+ cache-from: type=gha
+ cache-to: type=gha,mode=max
+
+ # Build and push Docker image with Buildx (don't push on PR)
+ - name: Build and push Docker image for SQLServer containe
+ uses: docker/build-push-action@v4
+ with:
+ context: .
+ file: mssql.Dockerfile
+ push: ${{ github.event_name != 'pull_request' }}
+ tags: ${{ steps.meta-db.outputs.tags }}
+ labels: ${{ steps.meta-db.outputs.labels }}
+ cache-from: type=gha
+ cache-to: type=gha,mode=max
\ No newline at end of file
diff --git a/.gitignore b/.gitignore
index a8e7454..ceec8be 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,6 +1,11 @@
/.data/
**/.idea/**/*.xml
**/*.iml
-/vendor/
+vendor*
/node_modules/
.vscode/
+
+# Shared files are bootstrapped:
+resources/bootstrap.inc.sh
+resources/.bootstrap-version
+_common/
diff --git a/.htaccess b/.htaccess
new file mode 100644
index 0000000..52fe189
--- /dev/null
+++ b/.htaccess
@@ -0,0 +1,135 @@
+
+RewriteEngine on
+RewriteBase /
+
+# Ready
+RewriteRule "^_control/ready$" "_control/ready.php" [END]
+
+###### Rewrites for /script folder: /search
+
+# rule name="/search/2.0" stopProcessing="true"
+RewriteRule "^search/2\.0" "/script/search/2.0/search.php" [END]
+
+# rule name="/search" stopProcessing="true"
+RewriteRule "^search(/1\.0)?" "/script/search/1.0/search.php" [END]
+
+# Rewrites for /script folder: /keyboard
+
+# rule name="/keyboard" stopProcessing="true"
+RewriteRule "^keyboard/(.*)$" "/script/keyboard/keyboard.php?id=$1" [END]
+
+
+###### Rewrites for /script folder: /increment-download
+
+# rule name="/increment-download" stopProcessing="true"
+# TODO: Conditions
+#
+#
+#
+# RewriteCond
+RewriteRule "^increment-download/(.*)$" "/script/increment-download/increment-download.php?id=$1" [END]
+
+##### Rewrites for /script folder: /model
+
+# rule name="/model?q=..." stopProcessing="true"
+RewriteRule "^model(/)?$" "/script/model-search/model-search.php" [END]
+
+# rule name="/model" stopProcessing="true"
+RewriteRule "^model/(.*)$" "/script/model/model.php?id=$1" [END]
+
+##### Rewrites for /script folder: /version
+
+# rule name="/version/platform/level" stopProcessing="true"
+RewriteRule "^version(\/([^/]+)(\/([^/]+))?)?$" "/script/version/version.php?platform=$2&level=$4" [END]
+
+# rule name="/version" stopProcessing="true"
+RewriteRule "^version(\/)?$" "/script/version/version.php" [END]
+
+# rule name="/package-version[/1.0]" stopProcessing="true" TODO appendQueryString="true"
+RewriteRule "^package-version(\/1\.0)?(\/)?$" "/script/package-version/package-version.php" [END]
+
+##### Rewrites for /script folder: /cloud to /script/legacy/...
+
+# rule name="Language + keyboard map 4.0" stopProcessing="true"
+RewriteRule "cloud/(4\.0\/)languages(\/([a-z0-9-]{2,}))(\/([a-z0-9_]+))" "/script/legacy/legacy40.php?context=language&languageid=$3&keyboardid=$5" [QSA,END]
+
+# rule name="Language map 4.0" stopProcessing="true"
+RewriteRule "cloud/(4\.0\/)languages(\/([a-z0-9-]{2,}))?" "/script/legacy/legacy40.php?context=language&languageid=$3" [QSA,END]
+
+# rule name="Keyboard + Language Map 4.0" stopProcessing="true"
+RewriteRule "cloud/(4\.0\/)keyboards(\/([a-z0-9_]+))(\/([a-z0-9-]{2,}))" "/script/legacy/legacy40.php?context=keyboard&keyboardid=$3&languageid=$5" [QSA,END]
+
+# rule name="Keyboard map 4.0" stopProcessing="true"
+RewriteRule "cloud/(4\.0\/)keyboards(\/([a-z0-9_]+))?" "/script/legacy/legacy40.php?context=keyboard&keyboardid=$3" [QSA,END]
+
+# rule name="Language + keyboard map 3.0" stopProcessing="true"
+RewriteRule "cloud/(3\.0\/)languages(\/([a-z0-9-]{2,}))(\/([a-z0-9_]+))" "/script/legacy/legacy30.php?context=language&languageid=$3&keyboardid=$5" [END]
+
+# rule name="Language map 3.0" stopProcessing="true"
+RewriteRule "cloud/(3\.0\/)languages(\/([a-z0-9-]{2,}))?" "/script/legacy/legacy30.php?context=language&languageid=$3" [END]
+
+# rule name="Keyboard + Language Map 3.0" stopProcessing="true"
+RewriteRule "cloud/(3\.0\/)keyboards(\/([a-z0-9_]+))(\/([a-z0-9-]{2,}))" "/script/legacy/legacy30.php?context=keyboard&keyboardid=$3&languageid=$5" [END]
+
+# rule name="Keyboard map 3.0" stopProcessing="true"
+RewriteRule "cloud/(3\.0\/)keyboards(\/([a-z0-9_]+))?" "/script/legacy/legacy30.php?context=keyboard&keyboardid=$3" [END]
+
+# rule name="Language map 2.0" stopProcessing="true"
+RewriteRule "cloud/(2\.0\/)languages(\/([a-z0-9-]{2,}))?" "/script/legacy/legacy20.php?context=language&languageid=$3" [END]
+
+# rule name="Keyboard + Language Map 2.0" stopProcessing="true"
+RewriteRule "cloud/(2\.0\/)keyboards(\/([a-z0-9_]+))(\/([a-z0-9-]{2,}))" "/script/legacy/legacy20.php?context=keyboard&keyboardid=$3&languageid=$5" [END]
+
+# rule name="Keyboard map 2.0" stopProcessing="true"
+RewriteRule "cloud/(2\.0\/)keyboards(\/([a-z0-9_]+))?" "/script/legacy/legacy20.php?context=keyboard&keyboardid=$3" [END]
+
+# rule name="Language map" stopProcessing="true"
+RewriteRule "cloud/(1\.0\/)?languages(\/([a-z0-9-]{2,}))?" "/script/legacy/legacy10.php?context=language&languageid=$3" [END]
+
+# rule name="Keyboard map" stopProcessing="true"
+RewriteRule "cloud/(1\.0\/)?keyboards(\/([a-z0-9_]+))?" "/script/legacy/legacy10.php?context=keyboard&keyboardid=$3" [END]
+
+##### Rewrites for /hooks
+
+# rule name="/hooks/keyboards-build-success.json" stopProcessing="true"
+# RewriteRule "^hooks/keyboards-build-success.json" "/script/hooks/keyboards-build-success.php?format=application/json" [QSA,END]
+
+
+# rule name="/hooks/keyboards-build-success" stopProcessing="true"
+# RewriteRule "^hooks/keyboards-build-success" "/script/hooks/keyboards-build-success.php?format=text/plain" [QSA,END]
+
+
+##### Keyman for Windows 14.0+ API endpoints
+
+# rule name="windows/14.0+/update" stopProcessing="true"
+RewriteRule "^windows/[1-9][0-9]\.[0-9]/update(.*)" "/script/windows/14.0/update/index.php$1" [END]
+
+
+##### Keyman Desktop 10.0, 11.0, 12.0, etc API endpoints
+
+# rule name="desktop/10.0-13.0/update" stopProcessing="true"
+RewriteRule "^desktop/(10|11|12|13)\.[0-9]/update(.*)" "/script/desktop/10.0/update/index.php$2" [END]
+
+
+# rule name="desktop/10.0/exception" stopProcessing="true"
+# Note: this endpoint is also used by developer
+RewriteRule "^desktop/[1-9][0-9]\.[0-9]/exception(.*)" "/script/desktop/10.0/exception/index.php$2" [END]
+
+
+# rule name="desktop/10.0/isonline" stopProcessing="true"
+RewriteRule "^desktop/[1-9][0-9]\.[0-9]/isonline(/?)$" "/script/desktop/10.0/isonline/index.php" [END]
+
+
+# rule name="desktop/10.0/submitdiag" stopProcessing="true"
+RewriteRule "^desktop/[1-9][0-9]\.[0-9]/submitdiag(/?)$" "/script/desktop/10.0/submitdiag/index.php" [END]
+
+##### Keyman Developer 10.0 - 13.0 API endpoints
+
+# rule name="developer/10.0/update" stopProcessing="true"
+RewriteRule "^developer/(10|11|12|13)\.0/update(.*)" "/script/developer/10.0/update/index.php$2" [END]
+
+
+##### Keyman Developer 14.0+ API endpoints
+
+# rule name="developer/14.0+/update" stopProcessing="true"
+RewriteRule "^developer/[1-9][0-9]\.[0-9]/update(.*)" "/script/developer/14.0/update/index.php$1" [END]
diff --git a/Dockerfile b/Dockerfile
new file mode 100644
index 0000000..f109b08
--- /dev/null
+++ b/Dockerfile
@@ -0,0 +1,47 @@
+# syntax=docker/dockerfile:1
+FROM php:7.4-apache@sha256:c9d7e608f73832673479770d66aacc8100011ec751d1905ff63fae3fe2e0ca6d AS composer-builder
+
+# Install Zip to use composer
+RUN apt-get update && apt-get install -y \
+ zlib1g-dev \
+ libzip-dev \
+ unzip
+RUN docker-php-ext-install zip
+
+# Install and update composer
+COPY --from=composer /usr/bin/composer /usr/bin/composer
+RUN composer self-update
+
+USER www-data
+WORKDIR /composer
+COPY composer.* /composer/
+RUN composer install
+
+# Site
+FROM php:7.4-apache@sha256:c9d7e608f73832673479770d66aacc8100011ec751d1905ff63fae3fe2e0ca6d
+COPY resources/keyman-site.conf /etc/apache2/conf-available/
+RUN cp /usr/local/etc/php/php.ini-production /usr/local/etc/php/php.ini
+RUN echo memory_limit = 1024M >> /usr/local/etc/php/php.ini
+RUN chown -R www-data:www-data /var/www/html/
+
+# Install SQL drivers
+# https://learn.microsoft.com/en-us/sql/connect/php/installation-tutorial-linux-mac?view=sql-server-ver16
+# https://stackoverflow.com/a/72176870
+RUN apt-get update && apt-get install -y gnupg2
+
+# Adding custom MS repo
+RUN curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add -
+RUN curl https://packages.microsoft.com/config/ubuntu/22.04/prod.list > /etc/apt/sources.list.d/mssql-release.list
+
+## Install SQL Server drivers and Zip
+RUN apt-get update && ACCEPT_EULA=Y apt-get -y --no-install-recommends install msodbcsql18 unixodbc-dev zip libzip-dev
+RUN pecl install sqlsrv-5.10.1
+RUN pecl install pdo_sqlsrv-5.10.1
+RUN docker-php-ext-install pdo pdo_mysql zip
+RUN docker-php-ext-enable sqlsrv pdo_sqlsrv pdo pdo_mysql
+COPY --from=composer-builder /composer/vendor /var/www/vendor
+
+# This is handled in init-container.sh
+# RUN ls -l /var/www/ && php /var/www/html/tools/db/build/build_cli.php
+RUN a2enmod rewrite; a2enconf keyman-site
+# service apache2 restart
diff --git a/README.md b/README.md
index d2dafbd..858deab 100644
--- a/README.md
+++ b/README.md
@@ -1,5 +1,7 @@
# api.keyman.com
+The following is outdated and will be replaced with Docker/Apache
+
## Configuration
Currently, this site runs only on a Windows host with IIS and Microsoft SQL Server.
diff --git a/_common/JsonApiFailure.php b/_common/JsonApiFailure.php
deleted file mode 100644
index cbf1270..0000000
--- a/_common/JsonApiFailure.php
+++ /dev/null
@@ -1,23 +0,0 @@
- $errorcode, 'message' => $message];
- echo json_encode($data, JSON_UNESCAPED_SLASHES);
- exit;
- }
-
- static function InvalidParameters($expected) {
- self::Failure(400, JsonApiFailure::ERROR_InvalidParameters, "Invalid parameters passed to function (expected: $expected)");
- }
- }
\ No newline at end of file
diff --git a/_common/KeymanHosts.php b/_common/KeymanHosts.php
deleted file mode 100644
index 36ad3e1..0000000
--- a/_common/KeymanHosts.php
+++ /dev/null
@@ -1,133 +0,0 @@
-tier;
- }
-
- public static function Rebuild() {
- self::$instance = new KeymanHosts();
- }
-
- /**
- * Returns $contents after regex'ing the Keyman live hosts for Markdown files
- * @param $contents
- * @return $contents
- */
- public function fixupHostReferences($contents) {
- // Regex Keyman hosts
- $contents = str_replace("https://s.keyman.com", $this->s_keyman_com, $contents);
- $contents = str_replace("https://api.keyman.com", $this->api_keyman_com, $contents);
- $contents = str_replace("https://help.keyman.com", $this->help_keyman_com, $contents);
- $contents = str_replace("https://downloads.keyman.com", $this->downloads_keyman_com, $contents);
- $contents = str_replace("https://keyman.com", $this->keyman_com, $contents);
- $contents = str_replace("https://keymanweb.com", $this->keymanweb_com, $contents);
- $contents = str_replace("https://r.keymanweb.com", $this->r_keymanweb_com, $contents);
- $contents = str_replace("https://blog.keyman.com", $this->blog_keyman_com, $contents);
- $contents = str_replace("https://donate.keyman.com", $this->donate_keyman_com, $contents);
- $contents = str_replace("https://translate.keyman.com", $this->translate_keyman_com, $contents);
- $contents = str_replace("https://sentry.keyman.com", $this->sentry_keyman_com, $contents);
-
- return $contents;
- }
-
- function __construct() {
- if(isset($_SERVER['KEYMANHOSTS_TIER']) && in_array($_SERVER['KEYMANHOSTS_TIER'],
- [KeymanHosts::TIER_DEVELOPMENT, KeymanHosts::TIER_STAGING,
- KeymanHosts::TIER_PRODUCTION, KeymanHosts::TIER_TEST])) {
- $this->tier = $_SERVER['KEYMANHOSTS_TIER'];
- } else if(file_exists(__DIR__ . '/../tier.txt')) {
- $this->tier = trim(file_get_contents(__DIR__ . '/../tier.txt'));
- } else {
- $this->tier = KeymanHosts::TIER_DEVELOPMENT;
- }
-
- switch($this->tier) {
- // Not all these are currently used but helps to cleanup confusion
- case KeymanHosts::TIER_PRODUCTION:
- case KeymanHosts::TIER_STAGING:
- $site_suffix = '';
- $site_protocol = 'https://';
- break;
- case KeymanHosts::TIER_TEST:
- $site_suffix = '';
- $site_protocol = 'http://';
- break;
- case KeymanHosts::TIER_DEVELOPMENT:
- $site_suffix = '.local';
- $site_protocol = 'https://';
- break;
- }
-
- $this->blog_keyman_com = "https://blog.keyman.com";
- $this->donate_keyman_com = "https://donate.keyman.com";
- $this->translate_keyman_com = "https://translate.keyman.com";
- $this->sentry_keyman_com = "https://sentry.keyman.com";
-
- if(in_array($this->tier, [KeymanHosts::TIER_STAGING, KeymanHosts::TIER_TEST])) {
- // As we build more staging areas, change these over as well. Assumption that we'll stage across multiple sites is a
- // little presumptuous but we can live with it.
- $this->s_keyman_com = "https://s.keyman.com";
- $this->api_keyman_com = "https://api.keyman-staging.com";
- $this->help_keyman_com = "https://help.keyman-staging.com";
- $this->downloads_keyman_com = "https://downloads.keyman.com";
- $this->keyman_com = "https://keyman-staging.com";
- $this->keymanweb_com = "https://keymanweb.com";
- $this->r_keymanweb_com = "https://r.keymanweb.com";
- } else {
- // TODO: allow override of these with e.g. KEYMANHOSTS_API_KEYMAN_COM='https://api.keyman.com';
- $this->s_keyman_com = "{$site_protocol}s.keyman.com{$site_suffix}";
- $this->api_keyman_com = "{$site_protocol}api.keyman.com{$site_suffix}";
- $this->help_keyman_com = "{$site_protocol}help.keyman.com{$site_suffix}";
- $this->downloads_keyman_com = "{$site_protocol}downloads.keyman.com{$site_suffix}";
- $this->keyman_com = "{$site_protocol}keyman.com{$site_suffix}";
- $this->keymanweb_com = "{$site_protocol}keymanweb.com{$site_suffix}";
- $this->r_keymanweb_com = "https://r.keymanweb.com"; /// local dev domain is usually not available
- }
-
- $this->blog_keyman_com_host = preg_replace('/^http(s)?:\/\/(.+)$/', '$2', $this->blog_keyman_com);
- $this->s_keyman_com_host = preg_replace('/^http(s)?:\/\/(.+)$/', '$2', $this->s_keyman_com);
- $this->api_keyman_com_host = preg_replace('/^http(s)?:\/\/(.+)$/', '$2', $this->api_keyman_com);
- $this->help_keyman_com_host = preg_replace('/^http(s)?:\/\/(.+)$/', '$2', $this->help_keyman_com);
- $this->downloads_keyman_com_host = preg_replace('/^http(s)?:\/\/(.+)$/', '$2', $this->downloads_keyman_com);
- $this->keyman_com_host = preg_replace('/^http(s)?:\/\/(.+)$/', '$2', $this->keyman_com);
- $this->keymanweb_com_host = preg_replace('/^http(s)?:\/\/(.+)$/', '$2', $this->keymanweb_com);
- $this->r_keymanweb_com_host = preg_replace('/^http(s)?:\/\/(.+)$/', '$2', $this->r_keymanweb_com);
- $this->donate_keyman_com_host = preg_replace('/^http(s)?:\/\/(.+)$/', '$2', $this->donate_keyman_com);
- $this->translate_keyman_com_host = preg_replace('/^http(s)?:\/\/(.+)$/', '$2', $this->translate_keyman_com);
- $this->sentry_keyman_com_host = preg_replace('/^http(s)?:\/\/(.+)$/', '$2', $this->sentry_keyman_com);
- }
- }
diff --git a/_common/KeymanSentry.php b/_common/KeymanSentry.php
deleted file mode 100644
index 7358d32..0000000
--- a/_common/KeymanSentry.php
+++ /dev/null
@@ -1,32 +0,0 @@
- $dsn,
- 'environment' => $environment
- ]);
- }
- }
\ No newline at end of file
diff --git a/_common/MarkdownHost.php b/_common/MarkdownHost.php
deleted file mode 100644
index 517c52f..0000000
--- a/_common/MarkdownHost.php
+++ /dev/null
@@ -1,83 +0,0 @@
-pagetitle;
- }
-
- function Content() {
- return $this->content;
- }
-
- function __construct($file) {
- $this->pagetitle = 'TODO'; // If page title is not set, this hints to the developer to fix it
-
- $file = realpath(__DIR__ . '/../') . DIRECTORY_SEPARATOR . $file;
- $contents = trim(file_get_contents($file));
- $contents = str_replace("\r\n", "\n", $contents);
-
- $contents = KeymanHosts::Instance()->fixupHostReferences($contents);
-
- // This header specification comes from YAML originally and is not common across
- // markdown implementations. While Parsedown does not currently parse this out,
- // it seems a sensible approach to use. The header section is delineated by `---`
- // and `---` must be the first three characters of the file (no BOM!); note that
- // the full spec supports metadata sections anywhere but we only support top-of-file.
- //
- // Currently we support only the 'title' and 'redirect' keywords.
- //
- // title: must be a plain text title
- // redirect: must be a relative or absolute url
- //
- // ---
- // keyword: content
- // keyword: content
- // ---
- //
- // source: https://yaml.org/spec/1.2/spec.html#id2760395
- // source: https://pandoc.org/MANUAL.html#extension-yaml_metadata_block
- //
-
- $lines = explode("\n", $contents);
-
- $found = count($lines) > 3 && rtrim($lines[0]) == '---';
- $headers = [];
- for($i = 1; $i < count($lines); $i++) {
- if($lines[$i] == '---') break;
- if(!preg_match('/^([a-z0-9_-]+):(.+)$/', $lines[$i], $match)) {
- $found = false;
- break;
- } else {
- $headers[$match[1]] = trim($match[2]);
- }
- }
- $found = $found && $i < count($lines);
-
- if($found) $contents = implode("\n", array_slice($lines, $i));
-
- if(isset($headers['redirect'])) {
- header("Location: {$headers['redirect']}");
- exit;
- }
-
- $this->pagetitle = isset($headers['title']) ? $headers['title'] : 'Untitled';
-
- // Performs the parsing + prettification of Markdown for display through PHP.
- $Parsedown = new \ParsedownExtra();
-
- // Does the magic.
- $this->content =
- "
" . htmlentities($this->pagetitle) . "
\n" .
- "" .
- $Parsedown->text($contents) .
- "
";
- }
- }
-
diff --git a/_common/README.md b/_common/README.md
deleted file mode 100644
index 6413998..0000000
--- a/_common/README.md
+++ /dev/null
@@ -1,16 +0,0 @@
-# Common Files
-
-These files are common to keyman.com sites. They must be kept identical to each other.
-
-## Namespace
-
-The root namespace for all PHP modules is `Keyman\Site\Common`.
-
-## Validation
-
-`composer test` on api.keyman.com will compare the contents of the `_common` folder across
-keyman.com, api.keyman.com and help.keyman.com (and in future, other sites), if the
-corresponding folders can be found. Currently, this test assumes that each site is in
-a sibling folder with corresponding name (e.g. `~/keyman/sites/keyman.com`,
-`~/keyman/sites/api.keyman.com`, etc), so the test will be skipped in CI at this time
-(especially as the development cycle may be out of sync for the sites).
diff --git a/_control/alive b/_control/alive
new file mode 100644
index 0000000..e7ad0e7
--- /dev/null
+++ b/_control/alive
@@ -0,0 +1,2 @@
+Alive
+
diff --git a/_control/info.php b/_control/info.php
new file mode 100644
index 0000000..951065f
--- /dev/null
+++ b/_control/info.php
@@ -0,0 +1,28 @@
+getActiveSchema();
+if(file_exists(__DIR__ . '/../.data/BUILDING')) $status = 'Building ' . $dci->getInactiveSchema();
+else if(file_exists(__DIR__ . '/../.data/MUST_REBUILD')) $status .= 'Must Rebuild';
+else $status = 'Ready';
+
+$date = file_exists(__DIR__ . '/../.data/LAST_REBUILD_DATE') ?
+ file_get_contents(__DIR__ . '/../.data/LAST_REBUILD_DATE') :
+ 'Unknown';
+
+echo <<api.keyman.com
+
+host | {$kh->api_keyman_com_host} |
+tier | {$kh->Tier()} |
+schema | $schema |
+database build status | $status |
+last database build completed | $date |
+
+END;
diff --git a/_control/ready.php b/_control/ready.php
new file mode 100644
index 0000000..b055dbe
--- /dev/null
+++ b/_control/ready.php
@@ -0,0 +1,66 @@
+prepare('EXEC sp_keyboard_search_by_id ?, ?');
+ $id = 'khmer_angkor';
+ $obsolete = FALSE;
+ $stmt->bindParam(1, $id);
+ $stmt->bindParam(2, $obsolete);
+ try {
+ if ($stmt->execute()) {
+ $data = $stmt->fetchAll(PDO::FETCH_NUM);
+ //json_print($data);
+ }
+ } catch(PDOException $e) {
+ die('Error: ' . $e->getMessage());
+ }
+
+ // Test chinese_pinyin_import.sql ready with query
+ $stmt = $mssql->prepare(
+ 'SELECT pinyin_key, chinese_text, tip FROM kmw_chinese_pinyin WHERE pinyin_key=? ORDER BY frequency DESC, id');
+ $py = 'biguansuoguo';
+ $stmt->bindParam(1, $py);
+ try {
+ if ($stmt->execute()) {
+ $data = $stmt->fetchAll(PDO::FETCH_NUM);
+ //json_print($data);
+ }
+ } catch(PDOException $e) {
+ die('chinese_pinyin_import.sql not ready: ' . $e->getMessage());
+ }
+
+ // Test japanese_import.sql ready with query
+ $stmt = $mssql->prepare(
+ 'SELECT DISTINCT kanji, gloss, pri FROM kmw_japanese WHERE (kana=?) ORDER BY pri');
+ $kana = 'あいでし';
+ $stmt->bindParam(1, $kana);
+ try {
+ if ($stmt->execute()) {
+ $data = $stmt->fetchAll(PDO::FETCH_NUM);
+ //json_print($data);
+ }
+ } catch(PDOException $e) {
+ die('japanese_import.sql not ready: ' . $e->getMessage());
+ }
+
+ if (!file_exists(__DIR__ . '/../.data/activeschema.txt')) {
+ die('/.data/activeschema.txt not ready');
+ }
+
+ if (!file_exists(__DIR__ . '/../_common/KeymanHosts.php')) {
+ die('/_common not ready');
+ }
+
+ if (!is_dir(__DIR__ . '/../vendor')) {
+ die('/vendor not ready');
+ }
+
+ echo "ready\n";
diff --git a/build.sh b/build.sh
new file mode 100755
index 0000000..e867f9e
--- /dev/null
+++ b/build.sh
@@ -0,0 +1,175 @@
+#!/usr/bin/env bash
+## START STANDARD SITE BUILD SCRIPT INCLUDE
+readonly THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")"
+readonly BOOTSTRAP="$(dirname "$THIS_SCRIPT")/resources/bootstrap.inc.sh"
+readonly BOOTSTRAP_VERSION=v0.4
+[ -f "$BOOTSTRAP" ] && source "$BOOTSTRAP" || source <(curl -fs https://raw.githubusercontent.com/keymanapp/shared-sites/$BOOTSTRAP_VERSION/bootstrap.inc.sh)
+## END STANDARD SITE BUILD SCRIPT INCLUDE
+
+readonly API_KEYMAN_DB_CONTAINER_NAME=api-keyman-com-db
+readonly API_KEYMAN_DB_CONTAINER_DESC=api-keyman-com-db
+readonly API_KEYMAN_DB_IMAGE_NAME=api-keyman-com-db
+
+readonly API_KEYMAN_CONTAINER_NAME=api-keyman-com-website
+readonly API_KEYMAN_CONTAINER_DESC=api-keyman-com-app
+readonly API_KEYMAN_IMAGE_NAME=api-keyman-com-website
+readonly HOST_API_KEYMAN_COM=api.keyman.com.localhost
+
+source _common/keyman-local-ports.inc.sh
+source _common/docker.inc.sh
+
+################################ Main script ################################
+
+builder_describe \
+ "Setup api.keyman.com site to run via Docker." \
+ "configure" \
+ "clean" \
+ "build" \
+ "start" \
+ "stop" \
+ "test" \
+ ":db Build the database" \
+ ":app Build the site"
+
+builder_parse "$@"
+
+function test_docker_container() {
+ echo "TIER_TEST" > tier.txt
+ # Note: ci.yml replicates these
+
+ # Run unit tests
+ docker exec $API_KEYMAN_CONTAINER_DESC sh -c "vendor/bin/phpunit --testdox"
+
+ # Lint .php files for obvious errors
+ docker exec $API_KEYMAN_CONTAINER_DESC sh -c "find . -name '*.php' | grep -v '/vendor/' | xargs -n 1 -d '\\n' php -l"
+
+ # Check all internal links
+ # NOTE: link checker runs on host rather than in docker image
+ npx broken-link-checker http://localhost:8058 --ordered --recursive --host-requests 10 -e --filter-level 3
+
+ rm tier.txt
+}
+
+builder_run_action configure bootstrap_configure
+
+builder_run_action clean:db clean_docker_container $API_KEYMAN_DB_IMAGE_NAME $API_KEYMAN_DB_CONTAINER_NAME
+builder_run_action clean:app clean_docker_container $API_KEYMAN_IMAGE_NAME $API_KEYMAN_CONTAINER_NAME
+builder_run_action stop:db stop_docker_container $API_KEYMAN_DB_IMAGE_NAME $API_KEYMAN_DB_CONTAINER_NAME
+builder_run_action stop:app stop_docker_container $API_KEYMAN_IMAGE_NAME $API_KEYMAN_CONTAINER_NAME
+
+# Build the Docker containers
+function build_docker_container_db() {
+ local IMAGE_NAME=$1
+ local CONTAINER_NAME=$2
+
+ # Download docker image. --mount option requires BuildKit
+ DOCKER_BUILDKIT=1 docker build -t $API_KEYMAN_DB_IMAGE_NAME -f mssql.Dockerfile .
+}
+
+builder_run_action build:db build_docker_container_db $API_KEYMAN_DB_IMAGE_NAME $API_KEYMAN_DB_CONTAINER_NAME
+builder_run_action build:app build_docker_container $API_KEYMAN_IMAGE_NAME $API_KEYMAN_CONTAINER_NAME
+
+# Custom start actions for db and app different from shared-sites
+function start_docker_container_db() {
+ local IMAGE_NAME=$1
+ local CONTAINER_NAME=$2
+ local CONTAINER_DESC=$3
+ # HOST not applicable
+ local PORT=$4
+
+ local CONTAINER_ID=$(get_docker_container_id $CONTAINER_NAME)
+ if [ ! -z "$CONTAINER_ID" ]; then
+ builder_die "container $CONTAINER_ID has already been started"
+ fi
+
+ # Start the Docker container
+ if [ -z $(get_docker_image_id $IMAGE_NAME) ]; then
+ builder_die "ERROR: Docker container doesn't exist. Run \"./build.sh build\" first"
+ fi
+
+ # Setup database
+ builder_echo "Setting up DB container"
+ docker run --rm -d -p $PORT:1433 \
+ -e "ACCEPT_EULA=Y" \
+ -e "MSSQL_AGENT_ENABLED=true" \
+ -e "MSSQL_SA_PASSWORD=yourStrong(\!)Password" \
+ --name $CONTAINER_DESC \
+ $CONTAINER_NAME
+
+ builder_echo green "Listening on http://localhost:$PORT"
+}
+
+function start_docker_container_app() {
+ local IMAGE_NAME=$1
+ local CONTAINER_NAME=$2
+ local CONTAINER_DESC=$3
+ local HOST=$4
+ local PORT=$5
+
+ _verify_vendor_is_not_folder
+
+ local CONTAINER_ID=$(get_docker_container_id $CONTAINER_NAME)
+ if [ ! -z "$CONTAINER_ID" ]; then
+ builder_die "$HOST container $CONTAINER_ID has already been started"
+ fi
+
+ # Start the Docker container
+ if [ -z $(get_docker_image_id $IMAGE_NAME) ]; then
+ builder_die "ERROR: Docker container doesn't exist. Run \"./build.sh build\" first"
+ fi
+
+ if [[ $OSTYPE =~ msys|cygwin ]]; then
+ # Windows needs leading slashes for path
+ SITE_HTML="//$(pwd):/var/www/html/"
+ else
+ SITE_HTML="$(pwd):/var/www/html/"
+ fi
+
+ ADD_HOST=
+ if [[ $OSTYPE =~ linux-gnu ]]; then
+ # Linux needs --add-host parameter
+ ADD_HOST="--add-host host.docker.internal:host-gateway"
+ fi
+
+ db_ip=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' ${API_KEYMAN_DB_IMAGE_NAME})
+
+ builder_echo "Spooling up site container"
+
+ docker run --rm -d -p $PORT:80 -v ${SITE_HTML} \
+ -e 'api_keyman_com_mssql_pw=yourStrong(\!)Password' \
+ -e api_keyman_com_mssql_user=sa \
+ -e 'api_keyman_com_mssqlconninfo=sqlsrv:Server='$db_ip',1433;TrustServerCertificate=true;Encrypt=false;Database=' \
+ -e api_keyman_com_mssql_create_database=true \
+ -e api_keyman_com_mssqldb=keyboards \
+ --name $CONTAINER_DESC \
+ ${ADD_HOST} \
+ $CONTAINER_NAME
+
+ # Skip if link already exists
+ if [ ! -L vendor ]; then
+ # Create link to vendor/ folder
+ CONTAINER_ID=$(get_docker_container_id $CONTAINER_NAME)
+ if [ -z "$CONTAINER_ID" ]; then
+ builder_die "Docker container appears to have failed to start in order to create link to vendor/"
+ fi
+
+ docker exec -i $CONTAINER_ID sh -c "ln -s /var/www/vendor vendor && chown -R www-data:www-data vendor"
+ fi
+
+ # after starting container, we want to run an init script if it is present
+ if [ -f resources/init-container.sh ]; then
+ CONTAINER_ID=$(get_docker_container_id $CONTAINER_NAME)
+ if [ -z "$CONTAINER_ID" ]; then
+ builder_die "Docker container appears to have failed to start in order to run init-container.sh script"
+ fi
+
+ docker exec -i $CONTAINER_ID sh -c "./resources/init-container.sh"
+ fi
+
+ builder_echo green "Listening on http://$HOST:$PORT"
+}
+
+builder_run_action start:db start_docker_container_db $API_KEYMAN_DB_IMAGE_NAME $API_KEYMAN_DB_CONTAINER_NAME $API_KEYMAN_DB_CONTAINER_DESC $PORT_API_KEYMAN_COM_DB
+builder_run_action start:app start_docker_container_app $API_KEYMAN_IMAGE_NAME $API_KEYMAN_CONTAINER_NAME $API_KEYMAN_CONTAINER_DESC $HOST_API_KEYMAN_COM $PORT_API_KEYMAN_COM
+
+builder_run_action test:app test_docker_container
diff --git a/composer.json b/composer.json
index 6aa104b..0c39a76 100644
--- a/composer.json
+++ b/composer.json
@@ -13,11 +13,11 @@
"scripts": {
"test": [
"Composer\\Config::disableProcessTimeout",
- "vendor\\bin\\phpunit --testdox"
+ "vendor/bin/phpunit --testdox"
],
"build": [
"Composer\\Config::disableProcessTimeout",
- "php tools\\db\\build\\build_cli.php"
+ "php ./tools/db/build/build_cli.php"
],
"lint": "find . -name '*.php' | grep -v '/vendor/' | xargs -n 1 php -l"
}
diff --git a/mssql.Dockerfile b/mssql.Dockerfile
new file mode 100644
index 0000000..869fe2c
--- /dev/null
+++ b/mssql.Dockerfile
@@ -0,0 +1,16 @@
+# syntax=docker/dockerfile:1
+FROM mcr.microsoft.com/mssql/server:2022-latest@sha256:ffef32dda16cc5abd70db1d0654c01cbe9f7093d66e0c83ade20738156cb7d0e
+USER root
+
+RUN export DEBIAN_FRONTEND=noninteractive && \
+apt-get update --fix-missing && \
+apt-get install -y gnupg2 && \
+apt-get install -yq curl apt-transport-https && \
+curl https://packages.microsoft.com/keys/microsoft.asc | tac | tac | apt-key add - && \
+curl https://packages.microsoft.com/config/ubuntu/22.04/mssql-server-2022.list | tac | tac | tee /etc/apt/sources.list.d/mssql-server.list && \
+apt-get update
+
+RUN apt-get install -y mssql-server-fts
+
+# Run SQL Server process
+CMD /opt/mssql/bin/sqlservr
diff --git a/resources/init-container.sh b/resources/init-container.sh
new file mode 100755
index 0000000..5276105
--- /dev/null
+++ b/resources/init-container.sh
@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+echo "---- Sleep 15 Before Generating DB ----"
+sleep 15;
+
+# If we know we are immediately going to run tests, there's no need to build
+# the database and then rebuild it again as a test database!
+if [[ ! -f /var/www/html/tier.txt ]] || [[ $(
+ SetHandler text/html
+
+
+DirectoryIndex index.md index.php index.html
+
+
+ Options +Includes +FollowSymLinks -MultiViews
+ AllowOverride All
+
+
+php_value include_path "/var/www/html/_includes:."
+
diff --git a/schemas/.htaccess b/schemas/.htaccess
new file mode 100644
index 0000000..13a49ea
--- /dev/null
+++ b/schemas/.htaccess
@@ -0,0 +1,44 @@
+# Rewrite old schema endpoints: note, all future references should be to the
+# versioned schema files rather than to the base, so we will not add extra
+# redirects here for new schemas
+
+# keyboard_info.distribution.json (deprecated by keyboard_info.schema.json)
+RewriteRule "^keyboard_info\.distribution\.json$" "/schemas/keyboard_info.distribution/1.0.6/keyboard_info.distribution.json" [END]
+
+# keyboard_info.source.json (deprecated by keyboard_info.schema.json)
+RewriteRule "^keyboard_info\.source\.json$" "/schemas/keyboard_info.source/1.0.6/keyboard_info.source.json" [END]
+
+# keyboard_json.json
+RewriteRule "^keyboard_json\.json$" "/schemas/keyboard_json/1.0/keyboard_json.json" [END]
+
+# model_info.distribution.json"
+RewriteRule "^model_info\.distribution\.json$" "/schemas/model_info.distribution/1.0.1/model_info.distribution.json" [END]
+
+# model_info.source.json
+RewriteRule "^model_info\.source\.json$" "/schemas/model_info.source/1.0.1/model_info.source.json" [END]
+
+# model-search.json
+RewriteRule "^model-search\.json$" "/schemas/model-search/1.0.1/model-search.json" [END]
+
+# package.json (renamed to kmp.schema.json)
+
+# note: package.json has been renamed to kmp.schema.json to reduce confusion with
+# NPM's standard filename, and these redirects added to keep things clear
+RewriteRule "^package\.json$" "/schemas/package/1.1.0/kmp.schema.json" [END]
+
+RewriteRule "^package/1\.0/package\.json$" "/schemas/kmp/1.0/kmp.schema.json" [END]
+RewriteRule "^package/1\.0\.1/package\.json$" "/schemas/kmp/1.0.1/kmp.schema.json" [END]
+RewriteRule "^package/1\.0\.2/package\.json$" "/schemas/kmp/1.0.2/kmp.schema.json" [END]
+RewriteRule "^package/1\.1\.0/package\.json$" "/schemas/kmp/1.1.0/kmp.schema.json" [END]
+
+# package-version.json
+RewriteRule "^package-version\.json$" "/schemas/package-version/1.0.1/package-version.json" [END]
+
+# search.json
+RewriteRule "^search\.json$" "/schemas/search/3.0/search.json" [END]
+
+# version.json
+RewriteRule "^version\.json$" "/schemas/version/2.0/version.json" [END]
+
+# windows-update.json
+RewriteRule "^windows-update\.json$" "/schemas/windows-update/17.0/windows-update.json" [END]
diff --git a/schemas/README.md b/schemas/README.md
index bff21b3..fa094df 100644
--- a/schemas/README.md
+++ b/schemas/README.md
@@ -1,199 +1,28 @@
-# keyboard_info
-
-* **keyboard_info.source.json**
-* **keyboard_info.distribution.json**
-
-Documentation at https://help.keyman.com/developer/cloud/keyboard_info
-
-New versions should be deployed to
-- **keymanapp/keyman/windows/src/global/inst/data/keyboard_info**
-- **keymanapp/keyboards/tools**
-- **keymanapp/keyboards-starter/tools**
-
-# .keyboard_info version history
-
-## 2018-11-26 1.0.5 stable
-* Add deprecated field - true if the keyboard is deprecated (generated at deployment time).
-
-## 2018-11-26 1.0.4 stable
-* Add helpLink field - a link to a keyboard's help page on help.keyman.com if it exists.
-
-## 2018-02-12 1.0.3 stable
-* Renamed minKeymanDesktopVersion to minKeymanVersion to clarify that this version information applies to all platforms.
-
-## 2018-02-10 1.0.2 stable
-* Add dictionary to platform support choices. Fixed default for platform to 'none'.
-
-## 2018-01-31 1.0.1 stable
-* Add file sizes, isRTL, sourcePath fields so we can supply these to the legacy KeymanWeb Cloud API endpoints.
-* Remove references to .kmx being a valid package format.
-
-## 2017-09-14 1.0 stable
-* Initial version
-
-------------------------------------------------------------
-
-# search
-
-* search.json
-
-Documentation at https://help.keyman.com/developer/cloud/search
-
-# search version history
-
-## 2019-12-13 1.0.2
-* Added deprecated field for keyboard_info
-
-## 2018-02-06 1.0.1
-* Added SearchCountry definition.
-
-## 2017-11-07 1.0 beta
-* Initial version
-
-------------------------------------------------------------
-
-# keyboard_json
-
-* keyboard_json.json
-
-Note: this format is deprecated as of Keyman 10.0.
-
-Documentation at https://help.keyman.com/developer/9.0/guides/distribute/mobile-apps
-
-# keyboard_json version history
-
-## 2017-11-23 1.0 beta
-* Initial version
-
-------------------------------------------------------------
-
-# package
-
-* package.json
-
-Documentation at https://help.keyman.com/developer/10.0/reference/file-types/metadata
-
-# package.json version history
-
-## 2019-01-31 1.1.0
-* Add lexicalModels properties (note: `version` is optional and currently unused)
-
-## 2018-02-13 1.0.2
-* Add rtl property for keyboard layouts
-
-## 2018-01-22 1.0.1
-* Remove id field as it is derived from the filename anyway
-
-## 2017-11-30 1.0 beta
-* Initial version
-
-------------------------------------------------------------
-
-# version
-
-* version.json
-
-Documentation at https://help.keyman.com/developer/cloud/version/2.0
-
-## version.json version history
-
-## 2021-09-30 2.0.2
-* Add 'all' semantics
-
-## 2019-10-23 2.0.1 alpha
-* Add linux platform
-
-## 2018-03-07 2.0 beta
-* Initial version
-
-------------------------------------------------------------
-
-# keymanweb-cloud-api
-
-* keymanweb-cloud-api-1.0.json
-* keymanweb-cloud-api-2.0.json
-* keymanweb-cloud-api-3.0.json
-* keymanweb-cloud-api-4.0.json
-
-Formal specification of legacy KeymanWeb cloud API endpoints at https://r.keymanweb.com/api/
-
-Documentation at https://help.keyman.com/developer/cloud/
-
-# keyman-web-cloud-api version history
-
-## 2018-01-31
-* Created schema files for existing json api endpoints
-
-------------------------------------------------------------
-
-# model_info
-
-* model_info.source.json
-* model_info.distribution.json
-
-Documentation at https://help.keyman.com/developer/cloud/model_info
-
-## 2019-01-31 1.0 beta
-* Initial version, seeded from .keyboard_info specification
-
-## 2020-09-21 1.0.1
-* Relaxed the URL definitions in the schema so extension is no longer tested
-
-------------------------------------------------------------
-
-# visualkeyboard
-
-* visualkeyboard.dtd
-
-XML Document Type Defintion for the .kvks file format. Previously, this was
-at http://tavultesoft.com/keymandev/visualkeyboard.dtd.
-
-## 2019-08-20
-* Moved to api.keyman.com (redirect on tavultesoft.com)
-
-------------------------------------------------------------
-
-# package-version
-
-* package-version.json
-
-Documentation at https://help.keyman.com/developer/cloud/package-version
-
-## 2020-04-30 1.0
-* Initial version 1.0 (alpha)
-
-## 2020-09-21 1.0.1
-
-* Fixed bugs with missing .kmp files
-* Added deprecation links
-* Updated .kmp URLs to use keyman.com/go/package format
-
-------------------------------------------------------------
-
-# windows-update
-
-* windows-update.json
-
-## 2020-07-23 1.0
-* Initial version 1.0 (alpha)
-
-## 2020-11-17 1.0.1
-* Relaxed `file` to `optional-file` to allow for responses without an update available.
-
-------------------------------------------------------------
-
-# developer-update
-
-* developer-update.json
-
-## 2020-11-17 1.0
-* Initial version 1.0 (alpha)
-
-------------------------------------------------------------
-
-# kps
-
-* kps.xsd
-
-## 2021-07-19 7.0
-* Initial version 7.0
+# Documentation for individual schemas
+
+* [developer-update](developer-update/README.md)
+* [displaymap](displaymap/README.md)
+* [keyboard_info](keyboard_info/README.md) (formerly named 'keyboard_info.distribution' and 'keyboard_info.source')
+* [keyboard_json](keyboard_json/README.md)
+* [keyman-touch-layout](keyman-touch-layout/README.md)
+* [keymanweb-cloud-api](keymanweb-cloud-api/README.md)
+* [kmp](kmp/README.md) (formerly named 'package')
+* [kpj](kpj/README.md)
+* [kpj-9.0](kpj-9.0/README.md)
+* [kps](kps/README.md)
+* [kvks](kvks/README.md) (replaces visualkeyboard)
+* [model_info.distribution](model_info.distribution/README.md)
+* [model_info.source](model_info.source/README.md)
+* [package-version](package-version/README.md)
+* [regtest](regtest/README.md)
+* [search](search/README.md)
+* [version](version/README.md)
+* [visualkeyboard](visualkeyboard/README.md)
+* [windows-update](windows-update/README.md)
+
+# Other schemas
+
+The following schemas are currently not documented:
+* increment-download
+* kvk (binary schema in Kaitai struct format)
+* model-search
diff --git a/schemas/developer-update/README.md b/schemas/developer-update/README.md
new file mode 100644
index 0000000..016aaf9
--- /dev/null
+++ b/schemas/developer-update/README.md
@@ -0,0 +1,6 @@
+# developer-update
+
+* developer-update.json
+
+## 2020-11-17 1.0
+* Initial version 1.0 (alpha)
diff --git a/schemas/displaymap/1.0/displaymap.schema.json b/schemas/displaymap/1.0/displaymap.schema.json
new file mode 100644
index 0000000..a5e6419
--- /dev/null
+++ b/schemas/displaymap/1.0/displaymap.schema.json
@@ -0,0 +1,36 @@
+{
+ "$ref": "#/definitions/displayMap",
+
+ "definitions": {
+ "displayMap": {
+ "type": "object",
+ "properties": {
+ "map": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/map"
+ }
+ }
+ }
+ },
+ "map": {
+ "type": "object",
+ "properties": {
+ "pua": { "type": "string" },
+ "str": { "type": "string" },
+ "unicode": { "type": "string" },
+ "usages": { "anyOf": [
+ { "type": "array", "items": { "$ref": "#/definitions/usage" } },
+ { "type": "array", "items": { "type": "string" } }
+ ] }
+ }
+ },
+ "usage": {
+ "type": "object",
+ "properties": {
+ "filename": { "type": "string" },
+ "count": { "type": "number" }
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/schemas/displaymap/README.md b/schemas/displaymap/README.md
new file mode 100644
index 0000000..1958c30
--- /dev/null
+++ b/schemas/displaymap/README.md
@@ -0,0 +1,77 @@
+# displaymap.schema.json
+
+This mapping file provides data for remapping the touch layout and visual
+keyboard key caps. The primary purpose of this file is to provide a pathway for
+consistent display of diacritics and other unattached marks which may be
+displayed on the keyboard by use of a special font with formatted glyphs in the
+Private Use Area, which will have consistent display across all platforms and
+not rely on platform-specific or font-specific rendering behaviors.
+
+This file can be generated by `kmc analyze osk-char-use` command, or hand
+crafted. The compiler uses only the `str` and `pua` values in the file, although
+there may be additional data in the file provided for reference purposes. The
+file should have the following structure, in this example, mapping the Unicode
+values `U+17BB U+17C7` (ុះ) to the Private Use Area code `U+F19F`:
+
+```json
+{
+ "map": [
+ {
+ "pua": "F19F",
+ "str": "ុះ",
+ "unicode": "17BB 17C7",
+ "usages": [
+ "khmer_angkor.kvks",
+ "khmer_angkor.keyman-touch-layout"
+ ]
+ },
+ ...
+ ]
+}
+```
+
+The file can be passed without modification to the [ttkbdfont.py script][2]
+(unsupported) to generate a Kbd font. The font may need some manual editing as
+insertion of dotted circle (`U+25CC`) as a base may not always be possible
+automatically, and combined marks may not render as a cluster in some scenarios.
+The open source tool [FontForge][3] is suitable for making these kinds of minor
+adjustments to the generated font.
+
+## Standard conventions for use of displayMaps
+
+In the Keyman keyboards repository, the PUA range used should start at `U+F100`.
+
+`&displayMap` JSON files should be named `Kbd