.

.vscode/launch.json

@@ -4,6 +4,15 @@
     // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
     "version": "0.2.0",
     "configurations": [
+    {
+        "name": "Launch Program",
+        "program": "${workspaceFolder}/pdf.mjs",
+        "request": "launch",
+        "skipFiles": [
+            "<node_internals>/**"
+        ],
+        "type": "node"
+    },
       {
         "type": "node",
         "request": "attach",

package.json

@@ -23,6 +23,7 @@
   },
   "dependencies": {
     "@alicloud/dysmsapi20170525": "2.0.23",
+    "@cyber2024/pdf-parse-fixed": "^1.2.5",
     "@dqbd/tiktoken": "^1.0.6",
     "@esm2cjs/p-timeout": "^6.0.0",
     "@fidm/x509": "^1.2.1",
@@ -43,16 +44,19 @@
     "@nestjs/typeorm": "^9.0.1",
     "ali-oss": "^6.17.1",
     "axios": "^1.3.6",
+    "azure-openai": "^0.9.4",
     "bcrypt": "^5.1.0",
     "big.js": "^6.2.1",
     "bignumber.js": "^9.1.1",
     "class-transformer": "^0.5.1",
     "class-validator": "^0.13.0",
+    "cld": "^2.9.0",
     "crypto": "^1.0.1",
     "date-fns": "^2.29.3",
     "eventsource-parser": "^1.0.0",
     "express-basic-auth": "^1.2.1",
     "express-handlebars": "^7.0.6",
+    "fastq": "^1.15.0",
     "handlebars": "^4.7.7",
     "hbs": "^4.2.0",
     "ioredis": "^5.3.2",
@@ -69,6 +73,7 @@
     "passport-jwt": "^4.0.1",
     "pem": "^1.14.7",
     "pg": "^8.11.0",
+    "queue": "^7.0.0",
     "quick-lru": "^5.0.0",
     "randomstring": "^1.2.3",
     "reflect-metadata": "^0.1.13",

pdf.mjs

@@ -0,0 +1,72 @@
+import PdfParse from '@cyber2024/pdf-parse-fixed'
+import { readFileSync } from 'fs'
+import cld from 'cld'
+import { Configuration, OpenAIApi } from 'azure-openai'
+import pg from 'pg'
+
+async function pdf2text(path) {
+    const pdf = await PdfParse(readFileSync(path))
+    const contents = []
+    let newParagraph = ''
+    pdf.text
+        .trim()
+        .split('\n')
+        .forEach((line) => {
+            line = line.trim()
+            newParagraph += line
+            if (isFullSentence(line)) {
+                contents.push(newParagraph)
+                newParagraph = ''
+            }
+        })
+    if (newParagraph) {
+        contents.push(newParagraph)
+    }
+
+    const lang = await cld.detect(contents.join('\n'))
+    console.log(contents.length)
+}
+
+function isFullSentence(str) {
+    return /[.!？。！？…;；:：”’）】》」』〕〉》〗〞〟»"'\])}]+$/.test(str)
+}
+
+await pdf2text('/Users/drew/Downloads/《Python 3学习笔记（上卷）》_1-50.pdf')
+
+const openai = new OpenAIApi(
+    new Configuration({
+        apiKey: 'beb32e4625a94b65ba8bc0ba1688c4d2',
+        // add azure info into configuration
+        azure: {
+            apiKey: 'beb32e4625a94b65ba8bc0ba1688c4d2',
+            endpoint: 'https://zouma.openai.azure.com/'
+        }
+    })
+)
+// const response = await openai.createEmbedding({
+//     model: 'embedding',
+//     input: 'The food was delicious and the waiter...'
+// })
+// console.log(JSON.stringify(response.data, null, 4))
+
+const client = new pg.Client({
+    host: '47.97.42.229',
+    port: 5432,
+    user: 'postgres',
+    password: 'D$&g3a9BCJH&$Nzh',
+    database: 'gpt_test',
+    connectionTimeoutMillis: 5000
+})
+await client.connect()
+
+//table exists
+client.query(`create table if not exists public.chat_embedding (
+    id integer primary key not null default nextval('embedding_id_seq'::regclass),
+    name character varying,
+    text character varying,
+    embedding vector(1536)
+);`)
+const res = await client.query('SELECT * FROM chat_embedding')
+
+console.log(res.rows)
+client.end()

src/app.module.ts

@@ -73,6 +73,7 @@ import { LikesModule } from './likes/likes.module'
         TypeOrmModule.forRootAsync({
             imports: [ConfigModule],
             inject: [ConfigService],
+            name: 'db1',
             useFactory: (config: ConfigService) => ({
                 name: 'db1',
                 type: 'postgres',

src/chat-pdf/chat-pdf.controller.ts

@@ -0,0 +1,16 @@
+import { Controller, Post, UploadedFile, UseInterceptors } from '@nestjs/common'
+import { Public } from '../auth/public.decorator'
+import { FileInterceptor } from '@nestjs/platform-express'
+import { ChatPdfService } from './chat-pdf.service'
+
+@Controller('chat-pdf')
+export class ChatPdfController {
+    constructor(private readonly chatPdfService: ChatPdfService) {}
+
+    @Public()
+    @Post('upload')
+    @UseInterceptors(FileInterceptor('file'))
+    public async uploadFile(@UploadedFile() file: Express.Multer.File) {
+        return await this.chatPdfService.upload(file)
+    }
+}

src/chat-pdf/chat-pdf.module.ts

@@ -1,7 +1,11 @@
-import { Module } from '@nestjs/common';
-import { ChatPdfService } from './chat-pdf.service';
+import { Module } from '@nestjs/common'
+import { ChatPdfService } from './chat-pdf.service'
+import { ChatPdfController } from './chat-pdf.controller'
+import { TypeOrmModule } from '@nestjs/typeorm'
 
 @Module({
-  providers: [ChatPdfService]
+    imports: [TypeOrmModule.forFeature([])],
+    providers: [ChatPdfService],
+    controllers: [ChatPdfController]
 })
 export class ChatPdfModule {}

src/chat-pdf/chat-pdf.service.spec.ts

@@ -1,18 +1,29 @@
-import { Test, TestingModule } from '@nestjs/testing';
-import { ChatPdfService } from './chat-pdf.service';
+import { Test, TestingModule } from '@nestjs/testing'
+import { ChatPdfService } from './chat-pdf.service'
+import { readFileSync } from 'fs'
+jest.useFakeTimers()
 
 describe('ChatPdfService', () => {
-  let service: ChatPdfService;
+    let service: ChatPdfService
 
-  beforeEach(async () => {
-    const module: TestingModule = await Test.createTestingModule({
-      providers: [ChatPdfService],
-    }).compile();
+    beforeEach(async () => {
+        const module: TestingModule = await Test.createTestingModule({
+            providers: [ChatPdfService]
+        }).compile()
 
-    service = module.get<ChatPdfService>(ChatPdfService);
-  });
+        service = module.get<ChatPdfService>(ChatPdfService)
+    })
 
-  it('should be defined', () => {
-    expect(service).toBeDefined();
-  });
-});
+    it('should be defined', () => {
+        expect(service).toBeDefined()
+    })
+
+    describe('parse', () => {
+        it('parse', async () => {
+            expect(service).toBeDefined()
+            console.log(
+                await service.parsePdf(readFileSync('/Users/drew/Downloads/《Python 3学习笔记（上卷）》_1-50.pdf'))
+            )
+        })
+    })
+})

src/chat-pdf/chat-pdf.service.ts

@@ -1,4 +1,110 @@
-import { Injectable } from '@nestjs/common';
+import { Injectable, InternalServerErrorException, Logger } from '@nestjs/common'
+import { mkdtempSync } from 'fs'
+import { tmpdir } from 'os'
+import * as PdfParse from '@cyber2024/pdf-parse-fixed'
+import { createHash } from 'crypto'
+import { DataSource } from 'typeorm'
+import { InjectDataSource } from '@nestjs/typeorm'
+import { get_encoding } from '@dqbd/tiktoken'
+import { Configuration, OpenAIApi } from 'azure-openai'
+import * as queue from 'fastq'
+import { setTimeout } from 'timers/promises'
 
 @Injectable()
-export class ChatPdfService {}
+export class ChatPdfService {
+    tokenizer = get_encoding('cl100k_base')
+    constructor(
+        @InjectDataSource('db1')
+        private dataSource: DataSource
+    ) {}
+
+    public async upload(file: Express.Multer.File) {
+        const { originalname, buffer, mimetype } = file
+        const md5 = this.calculateMD5(buffer)
+        const pdf = await PdfParse(buffer)
+        const contents = []
+        let newParagraph = ''
+        pdf.text
+            .trim()
+            .split('\n')
+            .forEach((line) => {
+                line = line.trim()
+                newParagraph += line
+                if (this.isFullSentence(line)) {
+                    contents.push(newParagraph)
+                    newParagraph = ''
+                }
+            })
+        if (newParagraph) {
+            contents.push(newParagraph)
+        }
+
+        const embeddings = await this.createEmbeddings(contents)
+        for (const item of embeddings) {
+            const sql = `INSERT INTO chat_embedding (id, name, text, embedding) VALUES (default, '${md5}', '${item.text
+                .replace(/'/g, "''")
+                .replace(/\\/g, '\\\\')}', '[${item.embedding.join(',')}]')`
+            try {
+                await this.dataSource.query(sql)
+            } catch (error) {
+                Logger.error(sql)
+            }
+        }
+    }
+
+    isFullSentence(str) {
+        return /[.!？。！？…;；:：”’）】》」』〕〉》〗〞〟»"'\])}]+$/.test(str)
+    }
+
+    calculateMD5(buffer) {
+        const hash = createHash('md5')
+        hash.update(buffer)
+        return hash.digest('hex')
+    }
+
+    async createEmbeddings(content: string[]) {
+        const self = this
+        const result = Array(content.length)
+        async function worker(arg) {
+            result[arg.index] = await self.getEmbedding(arg.text)
+        }
+        const q = queue.promise(worker, 64)
+        content.forEach((text, index) => {
+            q.push({
+                text,
+                index
+            })
+        })
+        await q.drained()
+        return result
+    }
+
+    async getEmbedding(content: string, retry = 0) {
+        const openai = new OpenAIApi(
+            new Configuration({
+                apiKey: 'beb32e4625a94b65ba8bc0ba1688c4d2',
+                // add azure info into configuration
+                azure: {
+                    apiKey: 'beb32e4625a94b65ba8bc0ba1688c4d2',
+                    endpoint: 'https://zouma.openai.azure.com/'
+                }
+            })
+        )
+        try {
+            const response = await openai.createEmbedding({
+                model: 'embedding',
+                input: content
+            })
+            return {
+                text: content,
+                embedding: response.data.data[0].embedding
+            }
+        } catch (error) {
+            if (retry < 3) {
+                await setTimeout(1000)
+                return await this.getEmbedding(content, retry + 1)
+            }
+            throw new InternalServerErrorException(error.message)
+        }
+    }
+}

src/weixin/weixin.service.spec.ts

@@ -6,13 +6,18 @@ import { ConfigModule } from '@nestjs/config'
 import { TypeOrmModule } from '@nestjs/typeorm'
 import weixinConfig from './weixin.config'
 import { AccessTokenCache } from './entities/access-token-cache.entity'
+import { JsapiTicketCache } from './entities/jsapi-ticket-cache.entity'
 
 describe('WeixinService', () => {
     let weixinService: WeixinService
 
     beforeEach(async () => {
         const moduleRef = await Test.createTestingModule({
-            imports: [AppModule, ConfigModule.forFeature(weixinConfig), TypeOrmModule.forFeature([AccessTokenCache])],
+            imports: [
+                AppModule,
+                ConfigModule.forFeature(weixinConfig),
+                TypeOrmModule.forFeature([AccessTokenCache, JsapiTicketCache])
+            ],
             controllers: [],
             providers: [WeixinService]
         }).compile()

yarn.lock

@@ -450,6 +450,15 @@
   dependencies:
     "@jridgewell/trace-mapping" "0.3.9"
 
+"@cyber2024/pdf-parse-fixed@^1.1.1", "@cyber2024/pdf-parse-fixed@^1.2.5":
+  version "1.2.5"
+  resolved "https://registry.npmmirror.com/@cyber2024/pdf-parse-fixed/-/pdf-parse-fixed-1.2.5.tgz#1e2743bdc35f0761b76441a5cbe4c874da9d7618"
+  integrity sha512-EEYLmbqVo8+RqQEo6dWtBWDgYF5oDzUKN0J1C04wJ67H613Uu/HqPwhDPYbMnCMTp3UTox0L/X26+3Is7F0mPg==
+  dependencies:
+    "@cyber2024/pdf-parse-fixed" "^1.1.1"
+    debug "^3.1.0"
+    node-ensure "^0.0.0"
+
 "@dqbd/tiktoken@^1.0.6":
   version "1.0.6"
   resolved "https://registry.npmmirror.com/@dqbd/tiktoken/-/tiktoken-1.0.6.tgz#96bfd0a4909726c61551a8c783493f01841bd163"
@@ -2017,6 +2026,13 @@ axios@^0.19.0:
   dependencies:
     follow-redirects "1.5.10"
 
+axios@^0.26.0:
+  version "0.26.1"
+  resolved "https://registry.npmmirror.com/axios/-/axios-0.26.1.tgz#1ede41c51fcf51bbbd6fd43669caaa4f0495aaa9"
+  integrity sha512-fPwcX4EvnSHuInCMItEhAGnaSEXRBjtzh9fOtsE6E1G6p7vl7edEeZe11QHf18+6+9gR5PbKV/sGKNaD8YaMeA==
+  dependencies:
+    follow-redirects "^1.14.8"
+
 axios@^1.3.6:
   version "1.3.6"
   resolved "https://registry.npmmirror.com/axios/-/axios-1.3.6.tgz#1ace9a9fb994314b5f6327960918406fa92c6646"
@@ -2026,6 +2042,14 @@ axios@^1.3.6:
     form-data "^4.0.0"
     proxy-from-env "^1.1.0"
 
+azure-openai@^0.9.4:
+  version "0.9.4"
+  resolved "https://registry.npmmirror.com/azure-openai/-/azure-openai-0.9.4.tgz#4aa21c71d015f6e044ad411b4e312055eb4e142c"
+  integrity sha512-7uii4ZInxzu2zjLg45PdvgOaw3ps18tEAw0Yux9mo8anX4PwnCMSS9xdlKNiNQyyEKPogvAcxH2PIufHXFLx6Q==
+  dependencies:
+    axios "^0.26.0"
+    form-data "^4.0.0"
+
 babel-jest@^29.4.2:
   version "29.4.2"
   resolved "https://registry.npmjs.org/babel-jest/-/babel-jest-29.4.2.tgz"
@@ -2399,6 +2423,15 @@ class-validator@^0.13.0:
     libphonenumber-js "^1.9.43"
     validator "^13.7.0"
 
+cld@^2.9.0:
+  version "2.9.0"
+  resolved "https://registry.npmmirror.com/cld/-/cld-2.9.0.tgz#8a4cec2541ac54fa45cf3aa3f13e561fed913fe7"
+  integrity sha512-eYK2WvupoNCCHnDjWiuEcpysqTsqQ8mZpAvFA6aFIQOBz0W77Ak+yRvvcf/Cvx3afcb86CMUBZ4W7b8IoKRuOg==
+  dependencies:
+    glob "7"
+    node-addon-api "*"
+    underscore "^1.12.1"
+
 cli-cursor@^3.1.0:
   version "3.1.0"
   resolved "https://registry.npmjs.org/cli-cursor/-/cli-cursor-3.1.0.tgz"
@@ -2716,6 +2749,13 @@ debug@=3.1.0:
   dependencies:
     ms "2.0.0"
 
+debug@^3.1.0:
+  version "3.2.7"
+  resolved "https://registry.npmmirror.com/debug/-/debug-3.2.7.tgz#72580b7e9145fb39b6676f9c5e5fb100b934179a"
+  integrity sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==
+  dependencies:
+    ms "^2.1.1"
+
 dedent@^0.7.0:
   version "0.7.0"
   resolved "https://registry.npmjs.org/dedent/-/dedent-0.7.0.tgz"
@@ -3291,6 +3331,13 @@ fast-safe-stringify@2.1.1, fast-safe-stringify@^2.1.1:
   resolved "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-2.1.1.tgz"
   integrity sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA==
 
+fastq@^1.15.0:
+  version "1.15.0"
+  resolved "https://registry.npmmirror.com/fastq/-/fastq-1.15.0.tgz#d04d07c6a2a68fe4599fea8d2e103a937fae6b3a"
+  integrity sha512-wBrocU2LCXXa+lWBt8RoIRD89Fi8OdABODa/kEnyeyjS5aZO5/GNvI5sEINADqP/h8M29UHTHUb53sUu5Ihqdw==
+  dependencies:
+    reusify "^1.0.4"
+
 fastq@^1.6.0:
   version "1.15.0"
   resolved "https://registry.npmjs.org/fastq/-/fastq-1.15.0.tgz"
@@ -3388,7 +3435,7 @@ follow-redirects@1.5.10:
   dependencies:
     debug "=3.1.0"
 
-follow-redirects@^1.15.0:
+follow-redirects@^1.14.8, follow-redirects@^1.15.0:
   version "1.15.2"
   resolved "https://registry.npmmirror.com/follow-redirects/-/follow-redirects-1.15.2.tgz#b460864144ba63f2681096f274c4e57026da2c13"
   integrity sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==
@@ -3639,18 +3686,7 @@ glob-to-regexp@^0.4.1:
   resolved "https://registry.npmjs.org/glob-to-regexp/-/glob-to-regexp-0.4.1.tgz"
   integrity sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw==
 
-glob@^10.1.0:
-  version "10.2.2"
-  resolved "https://registry.npmmirror.com/glob/-/glob-10.2.2.tgz#ce2468727de7e035e8ecf684669dc74d0526ab75"
-  integrity sha512-Xsa0BcxIC6th9UwNjZkhrMtNo/MnyRL8jGCP+uEwhA5oFOCY1f2s1/oNKY47xQ0Bg5nkjsfAEIej1VeH62bDDQ==
-  dependencies:
-    foreground-child "^3.1.0"
-    jackspeak "^2.0.3"
-    minimatch "^9.0.0"
-    minipass "^5.0.0"
-    path-scurry "^1.7.0"
-
-glob@^7.0.0, glob@^7.1.2, glob@^7.1.3, glob@^7.1.4:
+glob@7, glob@^7.0.0, glob@^7.1.2, glob@^7.1.3, glob@^7.1.4:
   version "7.2.3"
   resolved "https://registry.npmmirror.com/glob/-/glob-7.2.3.tgz#b8df0fb802bbfa8e89bd1d938b4e16578ed44f2b"
   integrity sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==
@@ -3662,6 +3698,17 @@ glob@^7.0.0, glob@^7.1.2, glob@^7.1.3, glob@^7.1.4:
     once "^1.3.0"
     path-is-absolute "^1.0.0"
 
+glob@^10.1.0:
+  version "10.2.2"
+  resolved "https://registry.npmmirror.com/glob/-/glob-10.2.2.tgz#ce2468727de7e035e8ecf684669dc74d0526ab75"
+  integrity sha512-Xsa0BcxIC6th9UwNjZkhrMtNo/MnyRL8jGCP+uEwhA5oFOCY1f2s1/oNKY47xQ0Bg5nkjsfAEIej1VeH62bDDQ==
+  dependencies:
+    foreground-child "^3.1.0"
+    jackspeak "^2.0.3"
+    minimatch "^9.0.0"
+    minipass "^5.0.0"
+    path-scurry "^1.7.0"
+
 glob@^8.1.0:
   version "8.1.0"
   resolved "https://registry.npmjs.org/glob/-/glob-8.1.0.tgz"
@@ -5247,6 +5294,11 @@ node-abort-controller@^3.0.1:
   resolved "https://registry.npmjs.org/node-abort-controller/-/node-abort-controller-3.1.1.tgz"
   integrity sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ==
 
+node-addon-api@*:
+  version "6.1.0"
+  resolved "https://registry.npmmirror.com/node-addon-api/-/node-addon-api-6.1.0.tgz#ac8470034e58e67d0c6f1204a18ae6995d9c0d76"
+  integrity sha512-+eawOlIgy680F0kBzPUNFhMZGtJ1YmqM6l4+Crf4IkImjYrO/mqPwRMh352g23uIaQKFItcQ64I7KMaJxHgAVA==
+
 node-addon-api@^5.0.0:
   version "5.1.0"
   resolved "https://registry.npmjs.org/node-addon-api/-/node-addon-api-5.1.0.tgz"
@@ -5259,6 +5311,11 @@ node-emoji@1.11.0:
   dependencies:
     lodash "^4.17.21"
 
+node-ensure@^0.0.0:
+  version "0.0.0"
+  resolved "https://registry.npmmirror.com/node-ensure/-/node-ensure-0.0.0.tgz#ecae764150de99861ec5c810fd5d096b183932a7"
+  integrity sha512-DRI60hzo2oKN1ma0ckc6nQWlHU69RH6xN0sjQTjMpChPfTYvKZdcQFfdYK2RWbJcKyUizSIy/l8OTGxMAM1QDw==
+
 node-fetch@^2.6.1, node-fetch@^2.6.7, node-fetch@^2.6.9:
   version "2.6.9"
   resolved "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.9.tgz"
@@ -5870,6 +5927,11 @@ queue-microtask@^1.2.2:
   resolved "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz"
   integrity sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==
 
+queue@^7.0.0:
+  version "7.0.0"
+  resolved "https://registry.npmmirror.com/queue/-/queue-7.0.0.tgz#2f43841ac492a4848007089810702704f5b2c4ae"
+  integrity sha512-sphwS7HdfQnvrJAXUNAUgpf9H/546IE3p/5Lf2jr71O4udEYlqAhkevykumas2FYuMkX/29JMOgrRdRoYZ/X9w==
+
 quick-lru@^5.0.0:
   version "5.1.1"
   resolved "https://registry.npmmirror.com/quick-lru/-/quick-lru-5.1.1.tgz#366493e6b3e42a3a6885e2e99d18f80fb7a8c932"
@@ -6942,6 +7004,11 @@ uid@2.0.1:
   dependencies:
     "@lukeed/csprng" "^1.0.0"
 
+underscore@^1.12.1:
+  version "1.13.6"
+  resolved "https://registry.npmmirror.com/underscore/-/underscore-1.13.6.tgz#04786a1f589dc6c09f761fc5f45b89e935136441"
+  integrity sha512-+A5Sja4HP1M08MaXya7p5LvjuM7K6q/2EaC0+iovj/wOcMsTzMvDFbasi/oSapiwOlt252IqsKqPjCl7huKS0A==
+
 unescape@^1.0.1:
   version "1.0.1"
   resolved "https://registry.npmmirror.com/unescape/-/unescape-1.0.1.tgz#956e430f61cad8a4d57d82c518f5e6cc5d0dda96"