|
|
@@ -1,4 +1,3 @@
|
|
|
-const datastore = require('nedb-promise')
|
|
|
const fs = require('fs')
|
|
|
const path = require('path')
|
|
|
const app = require('electron').app || require('electron').remote.app
|
|
|
@@ -11,7 +10,10 @@ const { groupBy } = require('lodash')
|
|
|
const archiver = require('archiver')
|
|
|
const uuidv1 = require('uuid/v1')
|
|
|
const eventBus = require('./EventBus').default
|
|
|
-const queue = require('block-queue');
|
|
|
+const queue = require('block-queue')
|
|
|
+const Sequelize = require('sequelize')
|
|
|
+const Op = Sequelize.Op
|
|
|
+
|
|
|
if (!fs.existsSync(path.resolve(app.getPath('userData'), 'db'))) {
|
|
|
fs.mkdirSync(path.resolve(app.getPath('userData'), 'db'))
|
|
|
}
|
|
|
@@ -21,24 +23,95 @@ if (!fs.existsSync(archiveDir)) {
|
|
|
fs.mkdirSync(archiveDir)
|
|
|
}
|
|
|
|
|
|
-let db = {}
|
|
|
-
|
|
|
-let uploadQueue = queue(1, function (job, done) {
|
|
|
- job.start(done)
|
|
|
+const sequelize = new Sequelize({
|
|
|
+ dialect: 'sqlite',
|
|
|
+ storage: path.resolve(app.getPath('userData'), 'db', 'application.db'),
|
|
|
+ logging: false
|
|
|
})
|
|
|
|
|
|
-app.db = db
|
|
|
-db.orders = new datastore({
|
|
|
- filename: path.resolve(app.getPath('userData'), 'db', 'orders.db'),
|
|
|
- autoload: true
|
|
|
-})
|
|
|
-db.subOrders = new datastore({
|
|
|
- filename: path.resolve(app.getPath('userData'), 'db', 'subOrders.db'),
|
|
|
- autoload: true
|
|
|
-})
|
|
|
-db.files = new datastore({
|
|
|
- filename: path.resolve(app.getPath('userData'), 'db', 'files.db'),
|
|
|
- autoload: true
|
|
|
+const Model = Sequelize.Model;
|
|
|
+class Order extends Model { }
|
|
|
+Order.init({
|
|
|
+ orderId: {
|
|
|
+ type: Sequelize.INTEGER,
|
|
|
+ allowNull: false,
|
|
|
+ unique: true
|
|
|
+ },
|
|
|
+ path: {
|
|
|
+ type: Sequelize.STRING,
|
|
|
+ allowNull: false
|
|
|
+ },
|
|
|
+ name: {
|
|
|
+ type: Sequelize.STRING,
|
|
|
+ allowNull: false
|
|
|
+ }
|
|
|
+}, { sequelize })
|
|
|
+class SubOrder extends Model { }
|
|
|
+SubOrder.init({
|
|
|
+ orderId: {
|
|
|
+ type: Sequelize.INTEGER,
|
|
|
+ allowNull: false
|
|
|
+ },
|
|
|
+ parentId: {
|
|
|
+ type: Sequelize.INTEGER,
|
|
|
+ allowNull: false
|
|
|
+ },
|
|
|
+ path: {
|
|
|
+ type: Sequelize.STRING,
|
|
|
+ allowNull: false
|
|
|
+ },
|
|
|
+ name: {
|
|
|
+ type: Sequelize.STRING,
|
|
|
+ allowNull: false
|
|
|
+ }
|
|
|
+}, { sequelize })
|
|
|
+class File extends Model { }
|
|
|
+File.init({
|
|
|
+ orderId: {
|
|
|
+ type: Sequelize.INTEGER,
|
|
|
+ allowNull: false
|
|
|
+ },
|
|
|
+ subOrderId: {
|
|
|
+ type: Sequelize.INTEGER,
|
|
|
+ allowNull: false
|
|
|
+ },
|
|
|
+ src: {
|
|
|
+ type: Sequelize.STRING,
|
|
|
+ allowNull: false
|
|
|
+ },
|
|
|
+ dst: {
|
|
|
+ type: Sequelize.STRING,
|
|
|
+ allowNull: false
|
|
|
+ },
|
|
|
+ size: {
|
|
|
+ type: Sequelize.BIGINT(19),
|
|
|
+ defaultValue: 0,
|
|
|
+ },
|
|
|
+ uploaded: {
|
|
|
+ type: Sequelize.BIGINT(19),
|
|
|
+ defaultValue: 0,
|
|
|
+ },
|
|
|
+ progress: {
|
|
|
+ type: Sequelize.DOUBLE,
|
|
|
+ defaultValue: 0,
|
|
|
+ },
|
|
|
+ checkPoint: Sequelize.TEXT,
|
|
|
+ status: {
|
|
|
+ type: Sequelize.INTEGER,
|
|
|
+ defaultValue: 0,
|
|
|
+ }
|
|
|
+}, { sequelize })
|
|
|
+sequelize.sync()
|
|
|
+
|
|
|
+let db = {
|
|
|
+ Order,
|
|
|
+ SubOrder,
|
|
|
+ File
|
|
|
+}
|
|
|
+window.db = db
|
|
|
+
|
|
|
+let uploadQueue = queue(3, function (job, done) {
|
|
|
+ job.start(done)
|
|
|
})
|
|
|
|
|
|
function sleep(ms) {
|
|
|
@@ -59,38 +132,48 @@ class UploadJob {
|
|
|
this.subscription = eventBus.subscribe('pause', async arg => {
|
|
|
if (arg.orderId == this.fileInfo.orderId
|
|
|
|| arg.subOrderId == this.fileInfo.subOrderId
|
|
|
- || arg.fileId == this.fileInfo._id) {
|
|
|
- this.isPause = true
|
|
|
- await this.pause()
|
|
|
+ || arg.fileId == this.fileInfo.id) {
|
|
|
+ this.pause()
|
|
|
this.subscription.unsubscribe()
|
|
|
- if (this.done) {
|
|
|
- this.done()
|
|
|
- }
|
|
|
}
|
|
|
})
|
|
|
}
|
|
|
async pause() {
|
|
|
await this.oss.cancel()
|
|
|
- await db.files.update({ _id: this.fileInfo._id }, { $set: { uploadStatus: UploadStatus.PAUSE } })
|
|
|
+ if (this.fileInfo.status != UploadStatus.FINISHED) {
|
|
|
+ await db.File.update({ status: UploadStatus.PAUSE }, {
|
|
|
+ where: {
|
|
|
+ id: this.fileInfo.id
|
|
|
+ }
|
|
|
+ })
|
|
|
+ }
|
|
|
this.isPause = true
|
|
|
if (this.done) {
|
|
|
this.done()
|
|
|
+ this.done = null
|
|
|
}
|
|
|
+ console.log('upload paused')
|
|
|
}
|
|
|
start(done) {
|
|
|
this.done = done
|
|
|
if (this.isPause) {
|
|
|
done()
|
|
|
+ this.done = null
|
|
|
} else {
|
|
|
this.resume()
|
|
|
}
|
|
|
}
|
|
|
async resume() {
|
|
|
this.retry++
|
|
|
- if (this.retry > 2) {
|
|
|
- await db.files.update({ _id: this.fileInfo._id }, { $set: { uploadStatus: UploadStatus.ERROR, error: this.error } })
|
|
|
+ if (this.retry > 3) {
|
|
|
+ await db.File.update({ status: UploadStatus.ERROR, error: this.error }, {
|
|
|
+ where: {
|
|
|
+ id: this.fileInfo.id
|
|
|
+ }
|
|
|
+ })
|
|
|
if (this.done) {
|
|
|
this.done()
|
|
|
+ this.done = null
|
|
|
}
|
|
|
return
|
|
|
}
|
|
|
@@ -98,58 +181,62 @@ class UploadJob {
|
|
|
console.log('retry ' + (this.retry - 1))
|
|
|
}
|
|
|
|
|
|
- let doc = await db.files.findOne({ _id: this.fileInfo._id })
|
|
|
- if (doc) {
|
|
|
+ let res = await db.File.findOne({ where: { id: this.fileInfo.id } })
|
|
|
+ if (res) {
|
|
|
+ let doc = res.get()
|
|
|
console.log('resume upload job', this.fileInfo)
|
|
|
- await db.files.update({ _id: this.fileInfo._id }, { $set: { uploadStatus: UploadStatus.UPLOADING } })
|
|
|
+ await db.File.update({ status: UploadStatus.UPLOADING }, {
|
|
|
+ where: {
|
|
|
+ id: this.fileInfo.id
|
|
|
+ }
|
|
|
+ })
|
|
|
let stat = fs.statSync(this.fileInfo.src)
|
|
|
- if (stat.size > 50 * 1024 * 1024) {
|
|
|
- try {
|
|
|
+ try {
|
|
|
+ if (stat.size > 50 * 1024 * 1024) {
|
|
|
let self = this
|
|
|
let result = await this.oss.multipartUpload(this.fileInfo.dst, this.fileInfo.src, {
|
|
|
- checkpoint: doc.checkPoint,
|
|
|
- progress(p, cpt, res) {
|
|
|
- console.log('ppppp', p, cpt, res)
|
|
|
- db.files.update({ _id: self.fileInfo._id }, { $set: { checkPoint: cpt, progress: p } })
|
|
|
+ checkpoint: doc.checkPoint ? JSON.parse(doc.checkPoint) : null,
|
|
|
+ async progress(p, cpt, res) {
|
|
|
+ await db.File.update({ checkPoint: JSON.stringify(cpt), progress: p }, {
|
|
|
+ where: {
|
|
|
+ id: self.fileInfo.id
|
|
|
+ }
|
|
|
+ })
|
|
|
}
|
|
|
})
|
|
|
if (200 === result.res.status) {
|
|
|
console.log('upload success', result)
|
|
|
- this.fileInfo.uploadStatus = UploadStatus.FINISHED
|
|
|
- await db.files.update({ _id: this.fileInfo._id }, { $set: { uploadStatus: UploadStatus.FINISHED } })
|
|
|
+ this.fileInfo.status = UploadStatus.FINISHED
|
|
|
+ await db.File.update({ status: UploadStatus.FINISHED, progress: 1 }, { where: { id: this.fileInfo.id } })
|
|
|
if (this.done) {
|
|
|
this.done()
|
|
|
+ this.done = null
|
|
|
}
|
|
|
} else {
|
|
|
console.log('upload error', result)
|
|
|
await sleep(1000)
|
|
|
this.resume()
|
|
|
}
|
|
|
- } catch (e) {
|
|
|
- console.log(e)
|
|
|
- this.error = e
|
|
|
- await sleep(1000)
|
|
|
- this.resume()
|
|
|
- }
|
|
|
- } else {
|
|
|
- try {
|
|
|
+ } else {
|
|
|
let result = await this.oss.put(this.fileInfo.dst, this.fileInfo.src)
|
|
|
if (200 === result.res.status) {
|
|
|
console.log('upload success', result)
|
|
|
- this.fileInfo.uploadStatus = UploadStatus.FINISHED
|
|
|
- await db.files.update({ _id: this.fileInfo._id }, { $set: { uploadStatus: UploadStatus.FINISHED } })
|
|
|
+ this.fileInfo.status = UploadStatus.FINISHED
|
|
|
+ await db.File.update({ status: UploadStatus.FINISHED, progress: 1 }, { where: { id: this.fileInfo.id } })
|
|
|
if (this.done) {
|
|
|
this.done()
|
|
|
+ this.done = null
|
|
|
}
|
|
|
} else {
|
|
|
console.log('upload error', result)
|
|
|
await sleep(1000)
|
|
|
this.resume()
|
|
|
}
|
|
|
- } catch (e) {
|
|
|
+ }
|
|
|
+ } catch (e) {
|
|
|
+ if (!(e && 'cancel' == e.name)) {
|
|
|
+ console.log('upload error', e)
|
|
|
this.error = e
|
|
|
- console.log(e)
|
|
|
- console.log('>>>', this.fileInfo)
|
|
|
await sleep(1000)
|
|
|
this.resume()
|
|
|
}
|
|
|
@@ -163,13 +250,25 @@ const init = async () => {
|
|
|
if (initiated) {
|
|
|
return
|
|
|
}
|
|
|
- // let records = await db.files.find({ $not: { uploadStatus: UploadStatus.FINISHED } })
|
|
|
+ // let records = (await db.File.findAll({
|
|
|
+ // where: {
|
|
|
+ // status: {
|
|
|
+ // [Op.not]: UploadStatus.FINISHED
|
|
|
+ // }
|
|
|
+ // }
|
|
|
+ // })).map(i => i.get())
|
|
|
// records.forEach(i => {
|
|
|
// let job = new UploadJob(i)
|
|
|
// uploadQueue.push(job)
|
|
|
// })
|
|
|
|
|
|
- let record = await db.files.findOne({ $not: { uploadStatus: UploadStatus.FINISHED } })
|
|
|
+ let record = (await db.File.findOne({
|
|
|
+ where: {
|
|
|
+ status: {
|
|
|
+ [Op.not]: UploadStatus.FINISHED
|
|
|
+ }
|
|
|
+ }
|
|
|
+ })).get()
|
|
|
let job = new UploadJob(record)
|
|
|
uploadQueue.push(job)
|
|
|
|
|
|
@@ -193,11 +292,32 @@ const pause = async arg => {
|
|
|
const resume = async arg => {
|
|
|
let records = []
|
|
|
if (arg.orderId) {
|
|
|
- records = await db.orders.find({ orderId: arg.orderId, $not: { uploadStatus: UploadStatus.FINISHED } })
|
|
|
+ records = (await db.File.findAll({
|
|
|
+ where: {
|
|
|
+ orderId: arg.orderId,
|
|
|
+ status: {
|
|
|
+ [Op.not]: UploadStatus.FINISHED
|
|
|
+ }
|
|
|
+ }
|
|
|
+ })).map(i => i.get())
|
|
|
} else if (arg.subOrderId) {
|
|
|
- records = await db.orders.find({ subOrderId: arg.subOrderId, $not: { uploadStatus: UploadStatus.FINISHED } })
|
|
|
+ records = (await db.File.findAll({
|
|
|
+ where: {
|
|
|
+ subOrderId: arg.subOrderId,
|
|
|
+ status: {
|
|
|
+ [Op.not]: UploadStatus.FINISHED
|
|
|
+ }
|
|
|
+ }
|
|
|
+ })).map(i => i.get())
|
|
|
} else if (arg.fileId) {
|
|
|
- records = await db.orders.find({ _id: arg.fileId, $not: { uploadStatus: UploadStatus.FINISHED } })
|
|
|
+ records = (await db.File.findAll({
|
|
|
+ where: {
|
|
|
+ id: arg.fileId,
|
|
|
+ status: {
|
|
|
+ [Op.not]: UploadStatus.FINISHED
|
|
|
+ }
|
|
|
+ }
|
|
|
+ })).map(i => i.get())
|
|
|
}
|
|
|
records.forEach(i => {
|
|
|
let job = new UploadJob(i)
|
|
|
@@ -206,25 +326,65 @@ const resume = async arg => {
|
|
|
}
|
|
|
|
|
|
const queryUnfinishedOrder = async () => {
|
|
|
- let records = await db.files.find({})
|
|
|
- let groups = groupBy(records, 'orderId')
|
|
|
- let result = {}
|
|
|
- for (let orderId in groups) {
|
|
|
- orderId = Number(orderId)
|
|
|
- let orderInfo = await db.orders.find({ orderId: Number(orderId) })
|
|
|
- let unfinishedFiles = await db.files.find({ orderId: orderId, $not: { uploadStatus: UploadStatus.FINISHED } })
|
|
|
- console.log(unfinishedFiles)
|
|
|
- }
|
|
|
+ let res = (await sequelize.query(
|
|
|
+ `SELECT
|
|
|
+ o.*,
|
|
|
+ fileNum,
|
|
|
+ totalSize
|
|
|
+ FROM
|
|
|
+ Orders o
|
|
|
+ JOIN ( SELECT SUM( size ) AS totalSize, COUNT( * ) AS fileNum, orderId FROM Files GROUP BY orderId ) t ON o.orderId = t.orderId
|
|
|
+ WHERE
|
|
|
+ o.orderId IN (
|
|
|
+ SELECT
|
|
|
+ orderId
|
|
|
+ FROM
|
|
|
+ Files
|
|
|
+ WHERE
|
|
|
+ status != ${UploadStatus.FINISHED}
|
|
|
+ GROUP BY
|
|
|
+ orderId)`, { type: sequelize.QueryTypes.SELECT }))
|
|
|
+ return res
|
|
|
}
|
|
|
+
|
|
|
+const queryDetail = async (orderId) => {
|
|
|
+ let res = await sequelize.query(
|
|
|
+ `SELECT
|
|
|
+ s.*,
|
|
|
+ COUNT( * ) AS fileNum,
|
|
|
+ SUM( size ) AS totalSize,
|
|
|
+ (
|
|
|
+ SUM( progress * size ) / SUM( size )) AS progress,
|
|
|
+ (
|
|
|
+ SELECT
|
|
|
+ a = b
|
|
|
+ FROM
|
|
|
+ (
|
|
|
+ ( SELECT COUNT( * ) AS a FROM Files ta WHERE ta.subOrderId = f.subOrderId )
|
|
|
+ JOIN ( SELECT COUNT( * ) AS b FROM Files tb WHERE tb.subOrderId = f.subOrderId AND tb.status = ${UploadStatus.PAUSE} )
|
|
|
+ )
|
|
|
+ ) AS paused
|
|
|
+ FROM
|
|
|
+ Files f
|
|
|
+ JOIN SubOrders s ON f.subOrderId = s.id
|
|
|
+ WHERE
|
|
|
+ f.orderId = ${orderId}
|
|
|
+ GROUP BY
|
|
|
+ f.subOrderId`, { type: sequelize.QueryTypes.SELECT })
|
|
|
+ console.log(res)
|
|
|
+ return res
|
|
|
+}
|
|
|
+
|
|
|
const transferFilename = (filename) => {
|
|
|
return filename.replace(/\.[0-9a-z]+$/i, '') + '.jpg'
|
|
|
}
|
|
|
const uploadOrder = async (topDir, orderId, name) => {
|
|
|
- let mainOrder = await db.orders.insert({
|
|
|
+ let mainOrder = (await db.Order.create({
|
|
|
orderId: orderId,
|
|
|
path: topDir,
|
|
|
name: name
|
|
|
- })
|
|
|
+ })).get()
|
|
|
+
|
|
|
let hasFile = false
|
|
|
let hasSubDir = false
|
|
|
fs.readdirSync(topDir).forEach(child => {
|
|
|
@@ -244,50 +404,50 @@ const uploadOrder = async (topDir, orderId, name) => {
|
|
|
throw ('空文件夹')
|
|
|
}
|
|
|
if (!hasSubDir) {
|
|
|
- let subOrder = await db.subOrders.insert({
|
|
|
- orderId: orderId,
|
|
|
- parentId: mainOrder._id,
|
|
|
+ let subOrder = (await db.SubOrder.create({
|
|
|
+ orderId,
|
|
|
+ parentId: mainOrder.id,
|
|
|
path: topDir,
|
|
|
- name: name
|
|
|
- })
|
|
|
- await readDir(topDir, orderId, subOrder._id)
|
|
|
+ name
|
|
|
+ })).get()
|
|
|
+ await readDir(topDir, orderId, subOrder.id)
|
|
|
let archiveFile = await archive(topDir)
|
|
|
let info = fs.statSync(archiveFile)
|
|
|
- let fileinfo = await db.files.insert({
|
|
|
- orderId: orderId,
|
|
|
- subOrderId: subOrder._id,
|
|
|
+ let fileinfo = (await db.File.create({
|
|
|
+ orderId,
|
|
|
+ subOrderId: subOrder.id,
|
|
|
src: archiveFile,
|
|
|
dst: orderId + "/" + path.basename(archiveFile),
|
|
|
size: info.size,
|
|
|
uploaded: 0,
|
|
|
progress: 0,
|
|
|
- uploadStatus: UploadStatus.WAITING
|
|
|
- })
|
|
|
+ status: UploadStatus.WAITING
|
|
|
+ })).get()
|
|
|
uploadQueue.push(new UploadJob(fileinfo))
|
|
|
} else {
|
|
|
for await (const child of fs.readdirSync(topDir)) {
|
|
|
if (!child.startsWith('.')) {
|
|
|
let info = fs.statSync(path.resolve(topDir, child))
|
|
|
if (info.isDirectory()) {
|
|
|
- let subOrder = await db.subOrders.insert({
|
|
|
- orderId: orderId,
|
|
|
- parentId: mainOrder._id,
|
|
|
+ let subOrder = (await db.SubOrder.create({
|
|
|
+ orderId,
|
|
|
+ parentId: mainOrder.id,
|
|
|
path: path.resolve(topDir, child),
|
|
|
name: child
|
|
|
- })
|
|
|
- await readDir(path.resolve(topDir, child), orderId, subOrder._id, child)
|
|
|
+ })).get()
|
|
|
+ await readDir(path.resolve(topDir, child), orderId, subOrder.id, child)
|
|
|
let archiveFile = await archive(path.resolve(topDir, child))
|
|
|
let info = fs.statSync(archiveFile)
|
|
|
- let fileinfo = await db.files.insert({
|
|
|
- orderId: orderId,
|
|
|
- subOrderId: subOrder._id,
|
|
|
+ let fileinfo = (await db.File.create({
|
|
|
+ orderId,
|
|
|
+ subOrderId: subOrder.id,
|
|
|
src: archiveFile,
|
|
|
dst: orderId + "/" + path.basename(archiveFile),
|
|
|
size: info.size,
|
|
|
uploaded: 0,
|
|
|
progress: 0,
|
|
|
- uploadStatus: UploadStatus.WAITING
|
|
|
- })
|
|
|
+ status: UploadStatus.WAITING
|
|
|
+ })).get()
|
|
|
uploadQueue.push(new UploadJob(fileinfo))
|
|
|
}
|
|
|
}
|
|
|
@@ -309,20 +469,21 @@ const uploadOrder = async (topDir, orderId, name) => {
|
|
|
if (type.mime.startsWith('image')) {
|
|
|
src = await ImageUtils.makeThumbnail(filepath)
|
|
|
dst = orderId + "/" + relative + '/' + transferFilename(path.basename(currRelative))
|
|
|
+ info = fs.statSync(src)
|
|
|
console.log('thumbnail:' + src)
|
|
|
+ let fileinfo = (await db.File.create({
|
|
|
+ orderId,
|
|
|
+ subOrderId,
|
|
|
+ src,
|
|
|
+ dst,
|
|
|
+ size: info.size,
|
|
|
+ uploaded: 0,
|
|
|
+ progress: 0,
|
|
|
+ status: UploadStatus.WAITING
|
|
|
+ })).get()
|
|
|
+ console.log(fileinfo)
|
|
|
+ uploadQueue.push(new UploadJob(fileinfo))
|
|
|
}
|
|
|
- let fileinfo = await db.files.insert({
|
|
|
- orderId: orderId,
|
|
|
- subOrderId: subOrderId,
|
|
|
- src: src,
|
|
|
- dst: dst,
|
|
|
- size: info.size,
|
|
|
- uploaded: 0,
|
|
|
- progress: 0,
|
|
|
- uploadStatus: UploadStatus.WAITING
|
|
|
- })
|
|
|
- console.log(fileinfo)
|
|
|
- uploadQueue.push(new UploadJob(fileinfo))
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
@@ -377,9 +538,9 @@ const archive = async (dir) => {
|
|
|
}
|
|
|
|
|
|
const clearDB = async () => {
|
|
|
- await db.orders.remove({}, { multi: true })
|
|
|
- await db.subOrders.remove({}, { multi: true })
|
|
|
- await db.files.remove({}, { multi: true })
|
|
|
+ await db.Order.destroy({ where: {} })
|
|
|
+ await db.SubOrder.destroy({ where: {} })
|
|
|
+ await db.File.destroy({ where: {} })
|
|
|
require('child_process').execSync('rm -rf *', {
|
|
|
cwd: path.resolve(app.getPath('userData'), 'thumb')
|
|
|
})
|
|
|
@@ -396,5 +557,6 @@ export default {
|
|
|
uploadOrder,
|
|
|
queryUnfinishedOrder,
|
|
|
clearDB,
|
|
|
- archive
|
|
|
+ archive,
|
|
|
+ queryDetail
|
|
|
}
|