x1ongzhu 1 jaar geleden
bovenliggende
commit
1bce8df324
3 gewijzigde bestanden met toevoegingen van 31 en 63 verwijderingen
  1. 1 2
      package.json
  2. 21 46
      src/paper/paper-gen/general.ts
  3. 9 15
      yarn.lock

+ 1 - 2
package.json

@@ -65,7 +65,7 @@
     "execa": "^8.0.1",
     "express-basic-auth": "^1.2.1",
     "express-handlebars": "^7.0.6",
-    "fastq": "^1.15.0",
+    "fastq": "^1.17.1",
     "handlebars": "^4.7.7",
     "hbs": "^4.2.0",
     "hnswlib-node": "^2.0.0",
@@ -80,7 +80,6 @@
     "nestjs-typeorm-paginate": "^4.0.3",
     "node-xlsx": "^0.23.0",
     "nodemailer": "^6.9.1",
-    "p-queue": "^8.0.1",
     "p-retry": "^6.1.0",
     "p-timeout": "^6.1.1",
     "passport": "^0.6.0",

+ 21 - 46
src/paper/paper-gen/general.ts

@@ -6,6 +6,7 @@ import { Logger } from '@nestjs/common'
 import { HumanMessage, SystemMessage } from 'langchain/schema'
 import * as NumberToChinese from '@siakhooi/number-to-chinese-words'
 import { createAbstract } from './abstract'
+import * as fastq from 'fastq'
 
 function chapters2md(chapters, sub = []) {
     let md = ''
@@ -86,60 +87,34 @@ ${chapters2md(chapters)}
             //     const { content } = await llm.call([new SystemMessage(sysPrompt), new HumanMessage(prompt)])
             //     paper.write('\n\n' + content)
             // }
-            const PQueue = (await eval("import('p-queue')")).default
-            const queue = new PQueue({concurrency: 4});
-            let content: any = []
-            chapter.children.forEach(async (section, j) => {
-                queue.add(async () => {
-                    let prompt
-                    if (section.children?.length) {
-                        prompt = `你可以适当加入一些口语,让文章更接近真人的书写的感觉
+            let content: any = Array(chapter.children.length)
+
+            const q = fastq.promise(async ({ section, j }) => {
+                let prompt
+                if (section.children?.length) {
+                    prompt = `你可以适当加入一些口语,让文章更接近真人的书写的感觉
 请务必不能让人觉得你是一个AI,否则我可能会丢掉我的学位
 
 ${chapters2md(section.children, [i + 1, j + 1])}
 请按照以上结构编写第${i + 1}章${chapter.title}的${i + 1}.${j + 1}小节(请注意只需要编写这一小节的内容)
 ## ${i + 1}.${j + 1} ${section.title} :`
-                    } else {
-                        prompt = `请务必不能让人觉得你是一个AI,否则我可能会丢掉我的学位\n\n请编写第${i + 1}章${
-                            chapter.title
-                        }的${i + 1}.${j + 1}小节(请注意只需要编写这一小节的内容)
+                } else {
+                    prompt = `请务必不能让人觉得你是一个AI,否则我可能会丢掉我的学位\n\n请编写第${i + 1}章${
+                        chapter.title
+                    }的${i + 1}.${j + 1}小节(请注意只需要编写这一小节的内容)
 ## ${i + 1}.${j + 1} ${section.title}:`
-                    }
-                    let { content: res } = await llm.call([new SystemMessage(sysPrompt), new HumanMessage(prompt)])
-                    res = res
-                        .replace(new RegExp(`#.*${chapter.title}.*\n`), '')
-                        .replace(new RegExp(`#.*${section.title}.*\n`), '')
-                    content[j] = `## ${i + 1}.${j + 1} ${section.title}\n\n` + res
-                })
+                }
+                let { content: res } = await llm.call([new SystemMessage(sysPrompt), new HumanMessage(prompt)])
+                res = res
+                    .replace(new RegExp(`#.*${chapter.title}.*\n`), '')
+                    .replace(new RegExp(`#.*${section.title}.*\n`), '')
+                content[j] = `## ${i + 1}.${j + 1} ${section.title}\n\n` + res
+            }, 4)
+            chapter.children.forEach((section, j) => {
+                q.push({ section, j })
             })
-            await queue.onIdle()
+            await q.drained()
             content = content.join('\n\n')
-
-            //             const content = (
-            //                 await Promise.all(
-            //                     chapter.children.map(async (section, j) => {
-            //                         let prompt
-            //                         if (section.children?.length) {
-            //                             prompt = `你可以适当加入一些口语,让文章更接近真人的书写的感觉
-            // 请务必不能让人觉得你是一个AI,否则我可能会丢掉我的学位
-
-            // ${chapters2md(section.children, [i + 1, j + 1])}
-            // 请按照以上结构编写第${i + 1}章${chapter.title}的${i + 1}.${j + 1}小节(请注意只需要编写这一小节的内容)
-            // ## ${i + 1}.${j + 1} ${section.title} :`
-            //                         } else {
-            //                             prompt = `请务必不能让人觉得你是一个AI,否则我可能会丢掉我的学位\n\n请编写第${i + 1}章${
-            //                                 chapter.title
-            //                             }的${i + 1}.${j + 1}小节(请注意只需要编写这一小节的内容)
-            // ## ${i + 1}.${j + 1} ${section.title}:`
-            //                         }
-            //                         let { content } = await llm.call([new SystemMessage(sysPrompt), new HumanMessage(prompt)])
-            //                         content = content
-            //                             .replace(new RegExp(`#.*${chapter.title}.*\n`), '')
-            //                             .replace(new RegExp(`#.*${section.title}.*\n`), '')
-            //                         return `## ${i + 1}.${j + 1} ${section.title}\n\n` + content
-            //                     })
-            //                 )
-            //             ).join('\n\n')
             paper.write('\n\n' + content)
         } else {
             let prompt = `第${i + 1}章: # ${chapter.title} (请注意只需要编写这一章的内容)`

+ 9 - 15
yarn.lock

@@ -4429,11 +4429,6 @@ eventemitter3@^4.0.4:
   resolved "https://registry.npmmirror.com/eventemitter3/-/eventemitter3-4.0.7.tgz#2de9b68f6528d5644ef5c59526a1b4a07306169f"
   integrity sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==
 
-eventemitter3@^5.0.1:
-  version "5.0.1"
-  resolved "https://registry.npmmirror.com/eventemitter3/-/eventemitter3-5.0.1.tgz#53f5ffd0a492ac800721bb42c66b841de96423c4"
-  integrity sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA==
-
 events@3.3.0, events@^3.2.0:
   version "3.3.0"
   resolved "https://registry.npmmirror.com/events/-/events-3.3.0.tgz#31a95ad0a924e2d2c419a813aeb2c4e878ea7400"
@@ -4638,7 +4633,14 @@ fast-xml-parser@4.2.5:
   dependencies:
     strnum "^1.0.5"
 
-fastq@^1.15.0, fastq@^1.6.0:
+fastq@^1.17.1:
+  version "1.17.1"
+  resolved "https://registry.npmmirror.com/fastq/-/fastq-1.17.1.tgz#2a523f07a4e7b1e81a42b91b8bf2254107753b47"
+  integrity sha512-sRVD3lWVIXWg6By68ZN7vho9a1pQcN/WBFaAAsDDFzlJjvoGx0P8z7V1t72grFJfJhu3YPZBuu25f7Kaw2jN1w==
+  dependencies:
+    reusify "^1.0.4"
+
+fastq@^1.6.0:
   version "1.15.0"
   resolved "https://registry.npmmirror.com/fastq/-/fastq-1.15.0.tgz#d04d07c6a2a68fe4599fea8d2e103a937fae6b3a"
   integrity sha512-wBrocU2LCXXa+lWBt8RoIRD89Fi8OdABODa/kEnyeyjS5aZO5/GNvI5sEINADqP/h8M29UHTHUb53sUu5Ihqdw==
@@ -7037,14 +7039,6 @@ p-queue@^6.6.2:
     eventemitter3 "^4.0.4"
     p-timeout "^3.2.0"
 
-p-queue@^8.0.1:
-  version "8.0.1"
-  resolved "https://registry.npmmirror.com/p-queue/-/p-queue-8.0.1.tgz#718b7f83836922ef213ddec263ff4223ce70bef8"
-  integrity sha512-NXzu9aQJTAzbBqOt2hwsR63ea7yvxJc0PwN/zobNAudYfb1B7R08SzB4TsLeSbUCuG467NhnoT0oO6w1qRO+BA==
-  dependencies:
-    eventemitter3 "^5.0.1"
-    p-timeout "^6.1.2"
-
 p-retry@4:
   version "4.6.2"
   resolved "https://registry.npmmirror.com/p-retry/-/p-retry-4.6.2.tgz#9baae7184057edd4e17231cee04264106e092a16"
@@ -7069,7 +7063,7 @@ p-timeout@^3.2.0:
   dependencies:
     p-finally "^1.0.0"
 
-p-timeout@^6.1.1, p-timeout@^6.1.2:
+p-timeout@^6.1.1:
   version "6.1.2"
   resolved "https://registry.npmmirror.com/p-timeout/-/p-timeout-6.1.2.tgz#22b8d8a78abf5e103030211c5fc6dee1166a6aa5"
   integrity sha512-UbD77BuZ9Bc9aABo74gfXhNvzC9Tx7SxtHSh1fxvx3jTLLYvmVhiQZZrJzqqU0jKbN32kb5VOKiLEQI/3bIjgQ==