From ccb56cd9ff03f5c52ff70612d1544e5748d2686c Mon Sep 17 00:00:00 2001
From: "gru-agent[bot]" <185149714+gru-agent[bot]@users.noreply.github.com>
Date: Tue, 4 Mar 2025 00:45:39 +0800
Subject: [PATCH] =?UTF-8?q?=E2=9C=85=20test:=20Add=20unit=20tests=20for=20?=
 =?UTF-8?q?the=20jina=20crawler=20functionality=20in=20jina.test.ts.=20(#6?=
 =?UTF-8?q?671)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: gru-agent[bot] <185149714+gru-agent[bot]@users.noreply.github.com>
---
 .../src/crawImpl/__test__/jina.test.ts        | 169 ++++++++++++++++++
 1 file changed, 169 insertions(+)
 create mode 100644 packages/web-crawler/src/crawImpl/__test__/jina.test.ts

diff --git a/packages/web-crawler/src/crawImpl/__test__/jina.test.ts b/packages/web-crawler/src/crawImpl/__test__/jina.test.ts
new file mode 100644
index 0000000000000..cb96a969e1daa
--- /dev/null
+++ b/packages/web-crawler/src/crawImpl/__test__/jina.test.ts
@@ -0,0 +1,169 @@
+import { describe, expect, it, vi } from 'vitest';
+
+import { jina } from '../jina';
+
+describe('jina crawler', () => {
+  const mockFetch = vi.fn();
+  global.fetch = mockFetch;
+
+  beforeEach(() => {
+    vi.resetAllMocks();
+  });
+
+  it('should crawl url successfully', async () => {
+    const mockResponse = {
+      ok: true,
+      json: () =>
+        Promise.resolve({
+          code: 200,
+          data: {
+            content: 'test content',
+            description: 'test description',
+            siteName: 'test site',
+            title: 'test title',
+          },
+        }),
+    };
+
+    mockFetch.mockResolvedValue(mockResponse);
+
+    const result = await jina('https://example.com', {
+      apiKey: 'test-key',
+      filterOptions: {},
+    });
+
+    expect(mockFetch).toHaveBeenCalledWith('https://r.jina.ai/https://example.com', {
+      headers: {
+        'Accept': 'application/json',
+        'Authorization': 'Bearer test-key',
+        'x-send-from': 'LobeChat Community',
+      },
+    });
+
+    expect(result).toEqual({
+      content: 'test content',
+      contentType: 'text',
+      description: 'test description',
+      length: 12,
+      siteName: 'test site',
+      title: 'test title',
+      url: 'https://example.com',
+    });
+  });
+
+  it('should use JINA_READER_API_KEY from env if apiKey not provided', async () => {
+    process.env.JINA_READER_API_KEY = 'env-reader-key';
+
+    const mockResponse = {
+      ok: true,
+      json: () =>
+        Promise.resolve({
+          code: 200,
+          data: {
+            content: 'test content',
+          },
+        }),
+    };
+
+    mockFetch.mockResolvedValue(mockResponse);
+
+    await jina('https://example.com', { filterOptions: {} });
+
+    expect(mockFetch).toHaveBeenCalledWith('https://r.jina.ai/https://example.com', {
+      headers: {
+        'Accept': 'application/json',
+        'Authorization': 'Bearer env-reader-key',
+        'x-send-from': 'LobeChat Community',
+      },
+    });
+
+    delete process.env.JINA_READER_API_KEY;
+  });
+
+  it('should use JINA_API_KEY from env if apiKey and JINA_READER_API_KEY not provided', async () => {
+    process.env.JINA_API_KEY = 'env-key';
+
+    const mockResponse = {
+      ok: true,
+      json: () =>
+        Promise.resolve({
+          code: 200,
+          data: {
+            content: 'test content',
+          },
+        }),
+    };
+
+    mockFetch.mockResolvedValue(mockResponse);
+
+    await jina('https://example.com', { filterOptions: {} });
+
+    expect(mockFetch).toHaveBeenCalledWith('https://r.jina.ai/https://example.com', {
+      headers: {
+        'Accept': 'application/json',
+        'Authorization': 'Bearer env-key',
+        'x-send-from': 'LobeChat Community',
+      },
+    });
+
+    delete process.env.JINA_API_KEY;
+  });
+
+  it('should send empty Authorization header if no api key provided', async () => {
+    const mockResponse = {
+      ok: true,
+      json: () =>
+        Promise.resolve({
+          code: 200,
+          data: {
+            content: 'test content',
+          },
+        }),
+    };
+
+    mockFetch.mockResolvedValue(mockResponse);
+
+    await jina('https://example.com', { filterOptions: {} });
+
+    expect(mockFetch).toHaveBeenCalledWith('https://r.jina.ai/https://example.com', {
+      headers: {
+        'Accept': 'application/json',
+        'Authorization': '',
+        'x-send-from': 'LobeChat Community',
+      },
+    });
+  });
+
+  it('should return undefined if response is not ok', async () => {
+    mockFetch.mockResolvedValue({ ok: false });
+
+    const result = await jina('https://example.com', { filterOptions: {} });
+
+    expect(result).toBeUndefined();
+  });
+
+  it('should return undefined if response code is not 200', async () => {
+    const mockResponse = {
+      ok: true,
+      json: () =>
+        Promise.resolve({
+          code: 400,
+          message: 'Bad Request',
+        }),
+    };
+
+    mockFetch.mockResolvedValue(mockResponse);
+
+    const result = await jina('https://example.com', { filterOptions: {} });
+
+    expect(result).toBeUndefined();
+  });
+
+  it('should return undefined if fetch throws error', async () => {
+    mockFetch.mockRejectedValue(new Error('Network error'));
+
+    const result = await jina('https://example.com', { filterOptions: {} });
+
+    expect(result).toBeUndefined();
+  });
+});