use meilisearch

Signed-off-by: ManAnRuck <[email protected]>

.github/workflows/deploy-meilisearch-sync-democracy.yaml

@@ -0,0 +1,30 @@
+name: Docker – meilisearch-sync-democracy
+
+on:
+  push:
+    paths:
+      - "services/cron-jobs/meilisearch-sync-democracy/**"
+    # branches:
+    #   - master
+
+jobs:
+  build:
+    name: Docker – meilisearch-sync-democracy
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Read package.json
+        uses: tyankatsu0105/read-package-version-actions@v1
+        id: package-version
+        with:
+          path: "./services/cron-jobs/meilisearch-sync-democracy"
+
+      - uses: docker/build-push-action@v1
+        with:
+          username: ${{ github.actor }}
+          password: ${{ github.token }}
+          registry: docker.pkg.github.com
+          repository: ${{ github.repository }}/meilisearch-sync-democracy
+          path: ./services/cron-jobs/meilisearch-sync-democracy
+          tags: ${{ steps.package-version.outputs.version }}

infra/kustomize/base/cluster/meilisearch-persistent-volume-claim.yaml

@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: meilisearch-persistent-volume-claim
+spec:
+  storageClassName: do-block-storage
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 1Gi

infra/kustomize/base/local/meilisearch-persistent-volume-claim.yaml

@@ -0,0 +1,25 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: meilisearch-persistent-volume-claim
+spec:
+  storageClassName: hostpath
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 1Gi
+---
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: meilisearch-volume
+spec:
+  storageClassName: hostpath
+  capacity:
+    storage: 1Gi
+  accessModes:
+    - ReadWriteOnce
+  persistentVolumeReclaimPolicy: Delete
+  hostPath:
+    path: /tmp/democracy-meilisearch/data.ms

infra/kustomize/base/main/meilisearch-depl.yaml

@@ -0,0 +1,40 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: meilisearch-depl
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: meilisearch
+  template:
+    metadata:
+      labels:
+        app: meilisearch
+    spec:
+      volumes:
+        - name: meilisearch-storage
+          persistentVolumeClaim:
+            claimName: meilisearch-persistent-volume-claim
+      containers:
+        - name: meilisearch
+          image: getmeili/meilisearch
+          volumeMounts:
+            - name: meilisearch-storage
+              mountPath: /data.ms
+          envFrom:
+            - secretRef:
+                name: meilisearch-config
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: meilisearch-srv
+spec:
+  selector:
+    app: meilisearch
+  ports:
+    - name: db
+      protocol: TCP
+      port: 7700
+      targetPort: 7700

infra/kustomize/base/main/meilisearch-sync-democracy.yaml

@@ -0,0 +1,24 @@
+apiVersion: batch/v1beta1
+kind: CronJob
+metadata:
+  name: meilisearch-sync-democracy-cronjob
+spec:
+  schedule: "0 6 * * *"
+  concurrencyPolicy: Forbid
+  jobTemplate:
+    spec:
+      backoffLimit: 0
+      template:
+        spec:
+          restartPolicy: Never
+          containers:
+            - name: meilisearch-sync-democracy
+              image: docker.pkg.github.com/demokratie-live/democracy-development/meilisearch-sync-democracy:0.1.0
+              env:
+                - name: DB_URL
+                  valueFrom:
+                    configMapKeyRef:
+                      name: democracy-api-config
+                      key: DB_URL
+          imagePullSecrets:
+            - name: regcred

services/cron-jobs/meilisearch-sync-democracy/.dockerignore

@@ -0,0 +1,2 @@
+node_modules
+build

services/cron-jobs/meilisearch-sync-democracy/.gitignore

@@ -0,0 +1,2 @@
+node_modules
+build

services/cron-jobs/meilisearch-sync-democracy/Dockerfile

@@ -0,0 +1,35 @@
+FROM node:12-alpine AS BUILD_IMAGE
+
+# install next-optimized-images requirements
+RUN apk --no-cache update \ 
+    && apk --no-cache add curl bash git \
+    &&  rm -fr /var/cache/apk/*
+
+# install node-prune (https://github.com/tj/node-prune)
+RUN curl -sfL https://install.goreleaser.com/github.com/tj/node-prune.sh | bash -s -- -b /usr/local/bin
+
+WORKDIR /app
+COPY package.json yarn.lock ./
+RUN yarn --frozen-lockfile
+COPY . .
+
+RUN yarn build
+
+RUN npm prune --production
+
+# run node prune
+RUN /usr/local/bin/node-prune
+
+FROM node:12-alpine
+
+WORKDIR /app
+
+COPY . .
+
+# copy from build image
+COPY --from=BUILD_IMAGE /app/build ./build
+COPY --from=BUILD_IMAGE /app/node_modules ./node_modules
+
+ENV NODE_ENV=production
+
+ENTRYPOINT [ "yarn", "start" ]

services/cron-jobs/meilisearch-sync-democracy/Dockerfile.dev

@@ -0,0 +1,15 @@
+FROM node:12-alpine
+
+RUN apk --no-cache update \ 
+    && apk --no-cache add git \
+    &&  rm -fr /var/cache/apk/*
+
+WORKDIR /app
+COPY package.json yarn.lock ./
+RUN yarn --frozen-lockfile
+
+COPY . .
+
+ENV NODE_ENV=development
+
+ENTRYPOINT [ "yarn", "dev" ]

services/cron-jobs/meilisearch-sync-democracy/package.json

@@ -0,0 +1,23 @@
+{
+  "name": "meilisearch-sync-democracy",
+  "version": "0.1.0",
+  "main": "build/index.js",
+  "license": "Apache-2.0",
+  "scripts": {
+    "dev": "ts-node-dev ./src/index",
+    "lint": "yarn lint:ts && yarn lint:exports",
+    "lint:ts": "tsc --noEmit",
+    "lint:exports": "ts-unused-exports ./tsconfig.json --excludePathsFromReport=generated --excludePathsFromReport=resolvers --excludePathsFromReport=/schemas",
+    "build": "tsc",
+    "start": "node ./build/index.js"
+  },
+  "dependencies": {
+    "@democracy-deutschland/democracy-common": "^0.2.8",
+    "meilisearch": "^0.12.0"
+  },
+  "devDependencies": {
+    "ts-node-dev": "^1.0.0-pre.50",
+    "ts-unused-exports": "^6.2.1",
+    "typescript": "^3.9.6"
+  }
+}

services/cron-jobs/meilisearch-sync-democracy/src/index.ts

@@ -0,0 +1,55 @@
+import mongoConnect from "./mongoose";
+import MeiliSearch, { Index } from "meilisearch";
+
+import {
+  ProcedureModel,
+  getCron,
+  IProcedure,
+} from "@democracy-deutschland/democracy-common";
+
+const CRON_NAME = "Meilisearch";
+
+const client = new MeiliSearch({
+  host: process.env.MEILI_SEARCH_HOST!,
+  apiKey: process.env.MEILI_SEARCH_SECRET!,
+});
+
+const start = async () => {
+  let index: Index<IProcedure>;
+  const cron = await getCron({ name: CRON_NAME });
+  console.log(cron);
+  const indexes = await client.listIndexes();
+  console.log(indexes);
+  if (!indexes.some((index) => index.uid === "procedures")) {
+    index = await client.createIndex("procedures", {
+      primaryKey: "procedureId",
+    });
+  } else {
+    index = client.getIndex("procedures");
+  }
+  const sumProcedures = await ProcedureModel.countDocuments();
+  console.log(sumProcedures);
+  const pageSize = 50;
+  console.log(`start indexing`);
+  for (let i = 0; i < sumProcedures; i += pageSize) {
+    const procedures = await ProcedureModel.find().limit(pageSize).skip(i);
+    await index.addDocuments(procedures as any);
+    console.log(`indexed ${i} - ${i + pageSize}`);
+  }
+  console.log(`finish indexing`);
+};
+
+(async () => {
+  console.info("START");
+  console.info("process.env", process.env.DB_URL);
+  if (!process.env.DB_URL) {
+    throw new Error("you have to set environment variable: DB_URL");
+  }
+  await mongoConnect();
+  console.log("procedures", await ProcedureModel.countDocuments({}));
+  await start().catch((e) => {
+    console.log(e);
+    process.exit(1);
+  });
+  process.exit(0);
+})();

services/cron-jobs/meilisearch-sync-democracy/src/mongoose.ts

@@ -0,0 +1,23 @@
+import { mongoose } from "@democracy-deutschland/democracy-common";
+
+export default () =>
+  new Promise(async (resolve, reject) => {
+    mongoose.set("useFindAndModify", false);
+    // Mongo Debug
+    mongoose.set("debug", false);
+
+    mongoose.connect(process.env.DB_URL!, {
+      useNewUrlParser: true,
+      useUnifiedTopology: true,
+    });
+
+    mongoose.connection.once("connected", () => {
+      console.info("MongoDB is running");
+      resolve();
+    });
+    mongoose.connection.on("error", (e: Error) => {
+      // Unknown if this ends up in main - therefore we log here
+      console.error(e.stack);
+      reject(e);
+    });
+  });

services/cron-jobs/meilisearch-sync-democracy/tsconfig.json

@@ -0,0 +1,69 @@
+{
+  "compilerOptions": {
+    /* Visit https://aka.ms/tsconfig.json to read more about this file */
+
+    /* Basic Options */
+    // "incremental": true,                   /* Enable incremental compilation */
+    "target": "es6" /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019', 'ES2020', or 'ESNEXT'. */,
+    "module": "commonjs" /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', 'es2020', or 'ESNext'. */,
+    // "lib": [],                             /* Specify library files to be included in the compilation. */
+    // "allowJs": true,                       /* Allow javascript files to be compiled. */
+    // "checkJs": true,                       /* Report errors in .js files. */
+    // "jsx": "preserve",                     /* Specify JSX code generation: 'preserve', 'react-native', or 'react'. */
+    // "declaration": true,                   /* Generates corresponding '.d.ts' file. */
+    // "declarationMap": true,                /* Generates a sourcemap for each corresponding '.d.ts' file. */
+    // "sourceMap": true,                     /* Generates corresponding '.map' file. */
+    // "outFile": "./",                       /* Concatenate and emit output to single file. */
+    "outDir": "./build" /* Redirect output structure to the directory. */,
+    "rootDir": "./src" /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */,
+    // "composite": true,                     /* Enable project compilation */
+    // "tsBuildInfoFile": "./",               /* Specify file to store incremental compilation information */
+    // "removeComments": true,                /* Do not emit comments to output. */
+    // "noEmit": true,                        /* Do not emit outputs. */
+    // "importHelpers": true,                 /* Import emit helpers from 'tslib'. */
+    // "downlevelIteration": true,            /* Provide full support for iterables in 'for-of', spread, and destructuring when targeting 'ES5' or 'ES3'. */
+    // "isolatedModules": true,               /* Transpile each file as a separate module (similar to 'ts.transpileModule'). */
+
+    /* Strict Type-Checking Options */
+    "strict": true /* Enable all strict type-checking options. */,
+    // "noImplicitAny": true,                 /* Raise error on expressions and declarations with an implied 'any' type. */
+    // "strictNullChecks": true,              /* Enable strict null checks. */
+    // "strictFunctionTypes": true,           /* Enable strict checking of function types. */
+    // "strictBindCallApply": true,           /* Enable strict 'bind', 'call', and 'apply' methods on functions. */
+    // "strictPropertyInitialization": true,  /* Enable strict checking of property initialization in classes. */
+    // "noImplicitThis": true,                /* Raise error on 'this' expressions with an implied 'any' type. */
+    // "alwaysStrict": true,                  /* Parse in strict mode and emit "use strict" for each source file. */
+
+    /* Additional Checks */
+    "noUnusedLocals": true /* Report errors on unused locals. */,
+    // "noUnusedParameters": true,            /* Report errors on unused parameters. */
+    // "noImplicitReturns": true,             /* Report error when not all code paths in function return a value. */
+    // "noFallthroughCasesInSwitch": true,    /* Report errors for fallthrough cases in switch statement. */
+
+    /* Module Resolution Options */
+    // "moduleResolution": "node",            /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */
+    // "baseUrl": "./src/" /* Base directory to resolve non-absolute module names. */,
+    // "paths": {},                           /* A series of entries which re-map imports to lookup locations relative to the 'baseUrl'. */
+    // "rootDirs": [],                        /* List of root folders whose combined content represents the structure of the project at runtime. */
+    // "typeRoots": [],                       /* List of folders to include type definitions from. */
+    // "types": [],                           /* Type declaration files to be included in compilation. */
+    // "allowSyntheticDefaultImports": true,  /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */
+    "esModuleInterop": true /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */,
+    // "preserveSymlinks": true,              /* Do not resolve the real path of symlinks. */
+    // "allowUmdGlobalAccess": true,          /* Allow accessing UMD globals from modules. */
+
+    /* Source Map Options */
+    // "sourceRoot": "",                      /* Specify the location where debugger should locate TypeScript files instead of source locations. */
+    // "mapRoot": "",                         /* Specify the location where debugger should locate map files instead of generated locations. */
+    // "inlineSourceMap": true,               /* Emit a single file with source maps instead of having a separate file. */
+    // "inlineSources": true,                 /* Emit the source alongside the sourcemaps within a single file; requires '--inlineSourceMap' or '--sourceMap' to be set. */
+
+    /* Experimental Options */
+    // "experimentalDecorators": true,        /* Enables experimental support for ES7 decorators. */
+    // "emitDecoratorMetadata": true,         /* Enables experimental support for emitting type metadata for decorators. */
+
+    /* Advanced Options */
+    "skipLibCheck": true /* Skip type checking of declaration files. */,
+    "forceConsistentCasingInFileNames": true /* Disallow inconsistently-cased references to the same file. */
+  }
+}