Merge pull request #8 from theosli/feature/page/subscription

Subscription feature

.github/pull_request_template.md

@@ -9,9 +9,3 @@ _Describe the solution this PR implements_
 ## How To Test
 
 _Describe the steps required to test the changes_
-
-## Additional Checks
-
-- [ ] The PR targets `master` for a bugfix, or `next` for a feature
-- [ ] The PR includes **unit tests** (if not possible, describe why)
-- [ ] The **documentation** is up to date

Makefile

@@ -48,7 +48,6 @@ conv_agent_test: ## Test the conversational agent
 	npm --workspace conversational_agent run test
 
 clean: ## To clean the project
-	rm -f package-lock.json
 	rm -rf node_modules
 
 # Start ngrok on a specific port

README.md

@@ -31,7 +31,7 @@ Then you can copy the `.env.sample` file as `.env`, and fill it with your info:
 
 ## Start the webpage
 
-To run the webpage localy:
+To run the webpage locally:
 
 ```sh
 make webpage
@@ -43,7 +43,13 @@ To start Next in dev mode:
 make dev
 ```
 
-## The conversational Agent
+To start the supabase database (in order to test the subscription feature)
+
+```sh
+npx supabase start
+```
+
+Warning : the supabase containers use Docker Desktop. The image is quite consequent (~5Gb needed)
 
 ## Test the interest scrapper (without the mail)
 

conversational_agent/package.json

@@ -13,7 +13,10 @@
     "dependencies": {
         "dotenv": "^16.3.1",
         "express": "^4.21.1",
+        "ngrok": "^5.0.0-beta.2",
         "openai": "^4.68.4",
+        "postmark": "^4.0.5",
+        "ts-node": "^10.9.2",
         "zod": "^3.23.8"
     },
     "devDependencies": {

conversational_agent/src/buildEmail.ts

@@ -1,31 +1,44 @@
 import { getUserPreferences } from './getUserPreferences';
+import { JSDOM } from 'jsdom';
+import DOMPurify from 'dompurify';
 
 export const buildResponse = async (body: any) => {
     // Generate a response from AI based on the received email text
     const aiResponse = await getUserPreferences(body['TextBody']);
 
+    const window = new JSDOM('').window;
+    const purify = DOMPurify(window);
+    const cleanBodyFrom = purify.sanitize(body['From']);
+    const cleanBodyDate = purify.sanitize(body['Date']);
+    const cleanBodyTo = purify.sanitize(body['To']);
+    const cleanBodySubject = purify.sanitize(body['Subject']);
+    const cleanBodyTextBody = purify.sanitize(body['TextBody']);
+    const cleanThemes = purify.sanitize(
+        aiResponse?.themes.length == 1 ? 'theme' : 'themes'
+    );
+
     const emailMetadata = `
         
         -------- Previous Message --------
         
-        From: ${body['From']}
+        From: ${cleanBodyFrom}
         
-        Sent: ${body['Date']}
+        Sent: ${cleanBodyDate}
         
-        To: ${body['To']}
+        To: ${cleanBodyTo}
         
-        Subject: ${body['Subject']}
+        Subject: ${cleanBodySubject}
         
-        ${body['TextBody']}
+        ${cleanBodyTextBody}
     
     `;
 
-    if (aiResponse?.themes?.length) {
+    if (!aiResponse?.themes?.length) {
         return `Sorry, we didn't find any preferences in your E-Mail. ${emailMetadata}`;
     }
     return `Hello!
     
-    The following ${aiResponse?.themes.length == 1 ? 'theme' : 'themes'} have been added to your next newsletters :
+    The following ${cleanThemes} have been added to your next newsletters :
     - ${aiResponse?.themes.join('\n  - ')}
     ${emailMetadata}`;
 };

conversational_agent/src/sendEmail.ts

@@ -1,14 +1,14 @@
 import dotenv from 'dotenv';
-import postmark from 'postmark';
+import { JSDOM } from 'jsdom';
+import DOMPurify from 'dompurify';
+import { ServerClient } from 'postmark';
 
 // Load environment variables from the .env file
 dotenv.config({ path: './../.env' });
 
 export const sendMail = async (to: string, subject: string, body: string) => {
     // Use the Postmark API key from environment variables
-    const client = new postmark.ServerClient(
-        process.env.POSTMARK_API_KEY || ''
-    );
+    const client = new ServerClient(process.env.POSTMARK_API_KEY || '');
 
     try {
         // Send an email
@@ -32,9 +32,12 @@ export const sendMail = async (to: string, subject: string, body: string) => {
  * @returns String
  */
 function formatTextBody(content: string) {
+    const window = new JSDOM('').window;
+    const purify = DOMPurify(window);
+    const cleanContent = purify.sanitize(content);
     return `Curator AI
 
-    ${content}
+    ${cleanContent}
     
     See you soon for your next newsletter!`;
 }
@@ -44,13 +47,16 @@ function formatTextBody(content: string) {
  * @param content String : The content of the mail
  * @returns String
  */
-function formatHtmlBody(content: String) {
+function formatHtmlBody(content: string) {
+    const window = new JSDOM('').window;
+    const purify = DOMPurify(window);
+    const cleanContent = purify.sanitize(content).replace(/\n/g, '<br/>');
     return `
   <div style="font-family: Arial, sans-serif; background-color: #f9f9f9; color: #333; padding: 20px; border-radius: 10px; max-width: 800px; margin: 0 auto;">
   <h1 style="color: #164e63; text-align: center; font-size: 32px;">Curator AI</h1>
   <p style="font-size: 18px; text-align: center;">Incoming message :</p>
   <div style="margin-bottom: 30px; padding: 20px; background-color: #fff; border-radius: 8px; box-shadow: 0 4px 8px rgba(0, 0, 0, 0.1);">
-  ${content.replace(/\n/g, '<br/>')}
+  ${cleanContent}
   </div>
   <p style="font-size: 18px; text-align: center;">See you soon for your next newsletter!</p>
 `;

conversational_agent/src/server.ts

@@ -2,6 +2,8 @@ import express, { Request, Response } from 'express';
 import dotenv from 'dotenv';
 import { buildResponse } from './buildEmail';
 import { sendMail } from './sendEmail';
+import { JSDOM } from 'jsdom';
+import DOMPurify from 'dompurify';
 
 // Load environment variables from the .env file
 dotenv.config({ path: './../.env' });
@@ -19,13 +21,19 @@ app.post('/webhook', async (req: Request, res: Response) => {
     const body = req.body;
     const isSpam = req.headers['X-Spam-Status'];
 
+    const window = new JSDOM('').window;
+    const purify = DOMPurify(window);
+    const cleanBodyFrom = purify.sanitize(body['From']);
+    const cleanBodyDate = purify.sanitize(body['Date']);
+    const cleanBodyTextBody = purify.sanitize(body['TextBody']);
+
     if (isSpam) {
         console.log('Spam received from ' + body['From']);
         return;
     }
     console.log(
-        `Received email from ${body['From']} on ${body['Date']} : 
-${body['TextBody']}`
+        `Received email from ${cleanBodyFrom} on ${cleanBodyDate} : 
+${cleanBodyTextBody}`
     );
 
     const response = await buildResponse(body);

conversational_agent/src/test/testScrapper.ts

@@ -1,5 +1,7 @@
 import { getUserPreferences } from '../getUserPreferences';
 import { promises as fs } from 'fs';
+import { JSDOM } from 'jsdom';
+import DOMPurify from 'dompurify';
 
 async function getStringFromFile(filePath: string): Promise<string> {
     try {
@@ -11,18 +13,26 @@ async function getStringFromFile(filePath: string): Promise<string> {
     }
 }
 
+async function formatResponse(aiResponse: { themes: string[] } | null) {
+    const window = new JSDOM('').window;
+    const purify = DOMPurify(window);
+    const cleanThemes = purify.sanitize(
+        aiResponse?.themes.length == 1 ? 'theme' : 'themes'
+    );
+    if (!aiResponse?.themes?.length) {
+        return `Hello!
+Sorry, we didn't find any preferences in your E-Mail.`;
+    }
+    return `Hello!
+The following ${cleanThemes} have been added to your next newsletters :
+- ${aiResponse?.themes.join('\n  - ')}`;
+}
+
 (async () => {
     let userMail = await getStringFromFile(__dirname + '/myMessage.txt');
 
     // Generate a response from AI based on the received email text
     const aiResponse = await getUserPreferences(userMail);
 
-    if (!aiResponse?.themes?.length) {
-        console.log(`Hello!
-Sorry, we didn't find any preferences in your E-Mail.`);
-    } else {
-        console.log(`Hello!
-The following ${aiResponse?.themes.length == 1 ? 'theme' : 'themes'} have been added to your next newsletters :
-- ${aiResponse?.themes.join('\n  - ')}`);
-    }
+    console.log(await formatResponse(aiResponse));
 })();

conversational_agent/tsconfig.json

@@ -2,27 +2,20 @@
     "include": ["./src/**/*"],
     "exclude": ["node_modules", "dist"],
     "compilerOptions": {
-        /* Visit https://aka.ms/tsconfig to read more about this file */
-        /* Projects */
-        /* Language and Environment */
-        "target": "es2016" /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */,
-        /* Modules */
-        "module": "commonjs" /* Specify what module code is generated. */,
-        "rootDir": "./src" /* Specify the root folder within your source files. */,
+        "target": "es2016",
+        "module": "commonjs",
+        "rootDir": "./src",
         "paths": {
             "@/*": ["./src/*"]
-        } /* Specify a set of entries that re-map imports to additional lookup locations. */,
-        "resolveJsonModule": true /* Enable importing .json files. */,
-        /* Emit */
-        "declaration": true /* Generate .d.ts files from TypeScript and JavaScript files in your project. */,
-        "declarationMap": true /* Create sourcemaps for d.ts files. */,
-        "sourceMap": true /* Create source map files for emitted JavaScript files. */,
-        "outDir": "./dist" /* Specify an output folder for all emitted files. */,
-        "esModuleInterop": true /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */,
-        "forceConsistentCasingInFileNames": true /* Ensure that casing is correct in imports. */,
-        /* Type Checking */
-        "strict": true /* Enable all strict type-checking options. */,
-        /* Completeness */
-        "skipLibCheck": true /* Skip type checking all .d.ts files. */
+        },
+        "resolveJsonModule": true,
+        "declaration": true,
+        "declarationMap": true,
+        "sourceMap": true,
+        "outDir": "./dist",
+        "esModuleInterop": true,
+        "forceConsistentCasingInFileNames": true,
+        "strict": true,
+        "skipLibCheck": true
     }
 }

curator/README.md

@@ -4,7 +4,7 @@ Mailing part of the Newsletter. Uses the curate agent to generate the content an
 
 ## Send a sample email
 
-First, define the POSTMARK_SERVER_API_KEY and POSTMARK_DEFAULT_MAIL in .env located at root of the repository.
+First, define the `POSTMARK_SERVER_API_KEY` and `POSTMARK_DEFAULT_MAIL` in `.env` located at root of the repository.
 
 ```txt
 POSTMARK_SERVER_API_KEY=XXX

curator/package.json

@@ -42,13 +42,13 @@
         "cli-progress": "^3.12.0",
         "commander": "^11.1.0",
         "dotenv": "^16.3.1",
-        "jsdom": "^24.0.0",
         "openai": "^4.24.3",
         "postmark": "^4.0.5",
         "rss-parser": "^3.13.0"
     },
     "scripts": {
         "build": "npx tsc",
-        "start": "ts-node ./src/cli.ts"
+        "start": "ts-node ./src/cli.ts",
+        "test:newsletter": "cp -n ./../.env.sample ./../.env && npx ts-node src/mail_agent/newsletterScript.ts"
     }
 }