删除dump storage 和 restore storage

o2server/configSample/dumpRestoreData.json

@@ -2,11 +2,15 @@
   "enable": false,
   "includes": [],
   "excludes": [],
-  "batchSize": 1000.0,
   "mode": "lite",
+  "parallel": true,
+  "redistribute": true,
+  "exceptionInvalidStorage": true,
   "###enable": "是否启用.###",
   "###includes": "导出导入包含对象,可以使用通配符*.###",
   "###excludes": "导出导入排除对象,可以使用通配符*.###",
-  "###batchSize": "批量对象数量.###",
-  "###mode": "导出数据模式,lite|full,默认使用lite###"
+  "###mode": "导出数据模式,lite|full,默认使用lite###",
+  "###parallel": "使用并行导出,默认true###",
+  "###redistribute": "是否进行重新分布.###",
+  "###exceptionInvalidStorage": "无法获取storage是否升起错误.###"
 }

o2server/configSample/node_127.0.0.1.json

@@ -138,16 +138,6 @@
     "###size": "最大保留份数,超过将自动删除最久的数据.###",
     "###path": "备份路径###"
   },
-  "dumpStorage": {
-    "enable": true,
-    "cron": "",
-    "size": 7.0,
-    "path": "",
-    "###enable": "是否启用,默认每天凌晨4点进行备份.###",
-    "###cron": "定时任务cron表达式###",
-    "###size": "最大保留份数,超过将自动删除最久的数据.###",
-    "###path": "备份路径###"
-  },
   "restoreData": {
     "enable": false,
     "cron": "",
@@ -156,14 +146,6 @@
     "###cron": "定时任务cron表达式###",
     "###path": "恢复路径###"
   },
-  "restoreStorage": {
-    "enable": false,
-    "cron": "",
-    "path": "",
-    "###enable": "是否启用.###",
-    "###cron": "定时任务cron表达式###",
-    "###path": "恢复路径###"
-  },
   "nodeAgentEnable": true,
   "nodeAgentPort": 20010.0,
   "nodeAgentEncrypt": true,
@@ -178,9 +160,7 @@
   "###storage": "Storage服务器配置###",
   "###logLevel": "日志级别,默认当前节点的slf4j日志级别,通过系统变量\"org.slf4j.simpleLogger.defaultLogLevel\"设置到当前jvm中.###",
   "###dumpData": "定时数据导出配置###",
-  "###dumpStorage": "定时存储文件导出配置###",
   "###restoreData": "定时数据导入配置###",
-  "###restoreStorage": "定时存储文件导入配置###",
   "###logSize": "日志文件保留天数.###",
   "###auditLogSize": "审计日志文件保留天数.###",
   "###nodeAgentEnable": "是否启用节点代理###",

o2server/pom.xml

@@ -494,7 +494,7 @@
 			<dependency>
 				<groupId>commons-io</groupId>
 				<artifactId>commons-io</artifactId>
-				<version>2.6</version>
+				<version>2.7</version>
 			</dependency>
 			<dependency>
 				<groupId>org.apache.commons</groupId>

o2server/x_base_core_project/src/main/java/com/x/base/core/project/build/CheckCore.java

@@ -19,6 +19,8 @@ import org.apache.commons.lang3.reflect.FieldUtils;
 import org.apache.openjpa.persistence.jdbc.ContainerTable;
 
 import com.x.base.core.entity.JpaObject;
+import com.x.base.core.entity.JsonProperties;
+import com.x.base.core.entity.StorageObject;
 import com.x.base.core.entity.annotation.ContainerEntity;
 import com.x.base.core.project.annotation.FieldDescribe;
 import com.x.base.core.project.annotation.Module;
@@ -44,6 +46,7 @@ public class CheckCore {
 			checkTableNameUniqueConstraintName(classes);
 			checkIdCreateTimeUpdateTimeSequenceIndex(classes);
 			checkEnum(classes);
+			checkDumpSize(classes);
 		}
 	}
 
@@ -215,17 +218,33 @@ public class CheckCore {
 		for (Class<?> cls : classes) {
 			List<Field> fields = FieldUtils.getFieldsListWithAnnotation(cls, Column.class);
 			for (Field field : fields) {
-				if ((!String.class.isAssignableFrom(field.getType())) && (!field.getType().isEnum())) {
-					Column column = field.getAnnotation(Column.class);
-					if (column.length() != 255) {
-						System.err.println(String.format("checkColumnLength error: class: %s, field: %s.",
-								cls.getName(), field.getName()));
+				if ((!String.class.isAssignableFrom(field.getType())) && (!field.getType().isEnum())
+						&& (!JsonProperties.class.isAssignableFrom(field.getType()))) {
+					Lob lob = cls.getAnnotation(Lob.class);
+					if (null != lob) {
+						Column column = field.getAnnotation(Column.class);
+						if (column.length() > 255) {
+							System.err.println(String.format("checkColumnLength error: class: %s, field: %s.",
+									cls.getName(), field.getName()));
+						}
 					}
 				}
 			}
 		}
 	}
 
+	/* 检查StorageObject的dumpSize是否设置正确 */
+	public static void checkDumpSize(List<Class<?>> classes) throws Exception {
+		for (Class<?> cls : classes) {
+			if (StorageObject.class.isAssignableFrom(cls)) {
+				ContainerEntity containerEntity = cls.getAnnotation(ContainerEntity.class);
+				if (containerEntity.dumpSize() > 10) {
+					System.err.println(String.format("checkDumpSize error: class: %s.", cls.getName()));
+				}
+			}
+		}
+	}
+
 	public static void checkIdUnique(List<Class<?>> classes) throws Exception {
 		for (Class<?> cls : classes) {
 			Field idField = FieldUtils.getField(cls, JpaObject.id_FIELDNAME, true);

o2server/x_base_core_project/src/main/java/com/x/base/core/project/config/DumpRestoreData.java

@@ -11,22 +11,27 @@ import com.x.base.core.project.tools.ListTools;
 
 public class DumpRestoreData extends ConfigObject {
 
-	public static String TYPE_FULL = "full";
-	public static String TYPE_LITE = "lite";
+	private static final long serialVersionUID = 8910820385137391619L;
 
 	public static DumpRestoreData defaultInstance() {
 		return new DumpRestoreData();
 	}
 
-	public static final int default_batchSize = 1000;
-	public static final String default_type = TYPE_LITE;
+	public static final String TYPE_FULL = "full";
+	public static final String TYPE_LITE = "lite";
+	public static final String DEFAULT_TYPE = TYPE_LITE;
+	public static final Boolean DEFAULT_PARALLEL = true;
+	public static final Boolean DEFAULT_REDISTRIBUTE = true;
+	public static final Boolean DEFAULT_EXCEPTIONINVALIDSTORAGE = true;
 
 	public DumpRestoreData() {
 		this.enable = false;
-		this.includes = new ArrayList<String>();
-		this.excludes = new ArrayList<String>();
-		this.batchSize = default_batchSize;
-		this.mode = default_type;
+		this.includes = new ArrayList<>();
+		this.excludes = new ArrayList<>();
+		this.mode = DEFAULT_TYPE;
+		this.parallel = DEFAULT_PARALLEL;
+		this.redistribute = DEFAULT_REDISTRIBUTE;
+		this.exceptionInvalidStorage = DEFAULT_EXCEPTIONINVALIDSTORAGE;
 	}
 
 	@FieldDescribe("是否启用.")
@@ -38,12 +43,30 @@ public class DumpRestoreData extends ConfigObject {
 	@FieldDescribe("导出导入排除对象,可以使用通配符*.")
 	private List<String> excludes;
 
-	@FieldDescribe("批量对象数量.")
-	private Integer batchSize;
-
 	@FieldDescribe("导出数据模式,lite|full,默认使用lite")
 	private String mode;
 
+	@FieldDescribe("使用并行导出,默认true")
+	private Boolean parallel;
+
+	@FieldDescribe("是否进行重新分布.")
+	private Boolean redistribute;
+
+	@FieldDescribe("无法获取storage是否升起错误.")
+	private Boolean exceptionInvalidStorage;
+
+	public Boolean getRedistribute() {
+		return BooleanUtils.isNotFalse(redistribute);
+	}
+
+	public Boolean getExceptionInvalidStorage() {
+		return BooleanUtils.isNotFalse(exceptionInvalidStorage);
+	}
+
+	public Boolean getParallel() {
+		return BooleanUtils.isNotFalse(parallel);
+	}
+
 	public String getMode() {
 		return StringUtils.equals(TYPE_FULL, mode) ? TYPE_FULL : TYPE_LITE;
 	}
@@ -68,13 +91,6 @@ public class DumpRestoreData extends ConfigObject {
 		return list;
 	}
 
-	public Integer getBatchSize() {
-		if ((null == this.batchSize) || (this.batchSize < 1)) {
-			return default_batchSize;
-		}
-		return this.batchSize;
-	}
-
 	public void setIncludes(List<String> includes) {
 		this.includes = includes;
 	}
@@ -83,8 +99,16 @@ public class DumpRestoreData extends ConfigObject {
 		this.excludes = excludes;
 	}
 
-	public void setBatchSize(Integer batchSize) {
-		this.batchSize = batchSize;
+	public void setParallel(Boolean parallel) {
+		this.parallel = parallel;
+	}
+
+	public void setRedistribute(Boolean redistribute) {
+		this.redistribute = redistribute;
+	}
+
+	public void setExceptionInvalidStorage(Boolean exceptionInvalidStorage) {
+		this.exceptionInvalidStorage = exceptionInvalidStorage;
 	}
 
 	public void setEnable(Boolean enable) {

o2server/x_base_core_project/src/main/java/com/x/base/core/project/config/Node.java

@@ -1,16 +1,16 @@
 package com.x.base.core.project.config;
 
-import com.x.base.core.project.annotation.FieldDescribe;
-import com.x.base.core.project.tools.DateTools;
-
 import org.apache.commons.lang3.BooleanUtils;
 import org.apache.commons.lang3.StringUtils;
 
+import com.x.base.core.project.annotation.FieldDescribe;
+import com.x.base.core.project.tools.DateTools;
+
 public class Node extends ConfigObject {
 
-	public static final Integer default_nodeAgentPort = 20010;
-	public static final String default_banner = "O2OA";
-	public static final Integer default_logSize = 14;
+	public static final Integer DEFAULT_NODEAGENTPORT = 20010;
+	public static final String DEFAULT_BANNER = "O2OA";
+	public static final Integer DEFAULT_LOGSIZE = 14;
 
 	public static Node defaultInstance() {
 		Node o = new Node();
@@ -23,12 +23,10 @@ public class Node extends ConfigObject {
 		o.storage = StorageServer.defaultInstance();
 		o.logLevel = "warn";
 		o.dumpData = new ScheduleDumpData();
-		o.dumpStorage = new ScheduleDumpStorage();
 		o.restoreData = new ScheduleRestoreData();
-		o.restoreStorage = new ScheduleRestoreStorage();
 		o.nodeAgentEnable = true;
 		o.nodeAgentEncrypt = true;
-		o.nodeAgentPort = default_nodeAgentPort;
+		o.nodeAgentPort = DEFAULT_NODEAGENTPORT;
 		o.quickStartWebApp = false;
 		o.autoStart = true;
 		return o;
@@ -52,12 +50,8 @@ public class Node extends ConfigObject {
 	private String logLevel;
 	@FieldDescribe("定时数据导出配置")
 	private ScheduleDumpData dumpData;
-	@FieldDescribe("定时存储文件导出配置")
-	private ScheduleDumpStorage dumpStorage;
 	@FieldDescribe("定时数据导入配置")
 	private ScheduleRestoreData restoreData;
-	@FieldDescribe("定时存储文件导入配置")
-	private ScheduleRestoreStorage restoreStorage;
 	@FieldDescribe("日志文件保留天数.")
 	private Integer logSize;
 	@FieldDescribe("审计日志文件保留天数.")
@@ -83,7 +77,6 @@ public class Node extends ConfigObject {
 	}
 	/* 20191009兼容centerServer end */
 
- 
 	public Boolean getEraseContentEnable() {
 		return BooleanUtils.isNotFalse(eraseContentEnable);
 	}
@@ -136,14 +129,14 @@ public class Node extends ConfigObject {
 	}
 
 	public String getBanner() {
-		return StringUtils.isBlank(this.banner) ? default_banner : this.banner;
+		return StringUtils.isBlank(this.banner) ? DEFAULT_BANNER : this.banner;
 	}
 
 	public Integer logSize() {
 		if ((this.logSize != null) && (this.logSize > 0)) {
 			return this.logSize;
 		}
-		return default_logSize;
+		return DEFAULT_LOGSIZE;
 	}
 
 	public Boolean getQuickStartWebApp() {
@@ -152,7 +145,7 @@ public class Node extends ConfigObject {
 
 	public Integer nodeAgentPort() {
 		if (null == this.nodeAgentPort || this.nodeAgentPort < 0) {
-			return default_nodeAgentPort;
+			return DEFAULT_NODEAGENTPORT;
 		}
 		return this.nodeAgentPort;
 	}
@@ -173,18 +166,10 @@ public class Node extends ConfigObject {
 		return (dumpData == null) ? new ScheduleDumpData() : this.dumpData;
 	}
 
-	public ScheduleDumpStorage dumpStorage() {
-		return (dumpStorage == null) ? new ScheduleDumpStorage() : this.dumpStorage;
-	}
-
 	public ScheduleRestoreData restoreData() {
 		return (restoreData == null) ? new ScheduleRestoreData() : this.restoreData;
 	}
 
-	public ScheduleRestoreStorage restoreStorage() {
-		return (restoreStorage == null) ? new ScheduleRestoreStorage() : this.restoreStorage;
-	}
-
 	public static class ScheduleDumpData extends ConfigObject {
 
 		public static ScheduleDumpData defaultInstance() {
@@ -225,46 +210,6 @@ public class Node extends ConfigObject {
 
 	}
 
-	public static class ScheduleDumpStorage extends ConfigObject {
-
-		public static ScheduleDumpStorage defaultInstance() {
-			return new ScheduleDumpStorage();
-		}
-
-		public boolean available() {
-			return DateTools.cronAvailable(this.cron());
-		}
-
-		@FieldDescribe("是否启用,默认每天凌晨4点进行备份.")
-		private Boolean enable = true;
-
-		@FieldDescribe("定时任务cron表达式")
-		private String cron = "";
-
-		@FieldDescribe("最大保留份数,超过将自动删除最久的数据.")
-		private Integer size = 7;
-
-		@FieldDescribe("备份路径")
-		private String path = "";
-
-		public Boolean enable() {
-			return (BooleanUtils.isTrue(this.enable)) ? true : false;
-		}
-
-		public String cron() {
-			return (null == cron) ? "5 0 4 * * ?" : this.cron;
-		}
-
-		public Integer size() {
-			return (null == size) ? 14 : this.size;
-		}
-
-		public String path() {
-			return StringUtils.trim(path);
-		}
-
-	}
-
 	public static class ScheduleRestoreData extends ConfigObject {
 
 		public static ScheduleRestoreData defaultInstance() {
@@ -298,37 +243,4 @@ public class Node extends ConfigObject {
 
 	}
 
-	public static class ScheduleRestoreStorage extends ConfigObject {
-
-		public static ScheduleRestoreStorage defaultInstance() {
-			return new ScheduleRestoreStorage();
-		}
-
-		public boolean available() {
-			return DateTools.cronAvailable(this.cron) && StringUtils.isNotEmpty(this.path);
-		}
-
-		@FieldDescribe("是否启用.")
-		private Boolean enable = false;
-
-		@FieldDescribe("定时任务cron表达式")
-		private String cron = "";
-
-		@FieldDescribe("恢复路径")
-		private String path = "";
-
-		public Boolean enable() {
-			return (BooleanUtils.isTrue(this.enable)) ? true : false;
-		}
-
-		public String cron() {
-			return (null == cron) ? "" : this.cron;
-		}
-
-		public String path() {
-			return StringUtils.trim(path);
-		}
-
-	}
-
 }

o2server/x_base_core_project/src/main/java/com/x/base/core/project/tools/StringTools.java

@@ -35,7 +35,26 @@ public class StringTools {
 	/** 中文,英文,数字,-,.· 【】（） */
 	public static final Pattern SIMPLY_REGEX = Pattern
 			.compile("^[\u4e00-\u9fa5a-zA-Z0-9\\_\\(\\)\\-\\ \\.\\ \\·\\【\\】\\（\\）]*$");
-	public static final Pattern FILENAME_REGEX = Pattern.compile("[^/\\\\<>*?|\"]+(\\.?)[^/\\\\<>*?|\"]+");
+	/**
+	 * MSDN
+	 * https://docs.microsoft.com/zh-cn/windows/win32/fileio/naming-a-file?redirectedfrom=MSDN#file_and_directory_names
+	 */
+	public static final Pattern FILENAME_REGEX = Pattern.compile(
+			"# Match a valid Windows filename (unspecified file system).          \n"
+					+ "^                                # Anchor to start of string.        \n"
+					+ "(?!                              # Assert filename is not: CON, PRN, \n"
+					+ "  (?:                            # AUX, NUL, COM1, COM2, COM3, COM4, \n"
+					+ "    CON|PRN|AUX|NUL|             # COM5, COM6, COM7, COM8, COM9,     \n"
+					+ "    COM[1-9]|LPT[1-9]            # LPT1, LPT2, LPT3, LPT4, LPT5,     \n"
+					+ "  )                              # LPT6, LPT7, LPT8, and LPT9...     \n"
+					+ "  (?:\\.[^.]*)?                  # followed by optional extension    \n"
+					+ "  $                              # and end of string                 \n"
+					+ ")                                # End negative lookahead assertion. \n"
+					+ "[^<>:\"/\\\\|?*\\x00-\\x1F]*     # Zero or more valid filename chars.\n"
+					+ "[^<>:\"/\\\\|?*\\x00-\\x1F\\ .]  # Last char is not a space or dot.  \n"
+					+ "$                                # Anchor to end of string.            ",
+			Pattern.CASE_INSENSITIVE | Pattern.UNICODE_CASE | Pattern.COMMENTS);
+
 	/**
 	 * RFC822 compliant regex adapted for Java
 	 * http://stackoverflow.com/questions/8204680/java-regex-email
@@ -480,37 +499,37 @@ public class StringTools {
 		while (tok.hasMoreTokens()) {
 			String nextTok = tok.nextToken();
 			switch (state) {
-				case inQuote:
-					if ("\'".equals(nextTok)) {
-						lastTokenHasBeenQuoted = true;
-						state = normal;
-					} else {
-						current.append(nextTok);
-					}
-					break;
-				case inDoubleQuote:
-					if ("\"".equals(nextTok)) {
-						lastTokenHasBeenQuoted = true;
-						state = normal;
-					} else {
-						current.append(nextTok);
-					}
-					break;
-				default:
-					if ("\'".equals(nextTok)) {
-						state = inQuote;
-					} else if ("\"".equals(nextTok)) {
-						state = inDoubleQuote;
-					} else if (" ".equals(nextTok)) {
-						if (lastTokenHasBeenQuoted || current.length() > 0) {
-							result.add(current.toString());
-							current.setLength(0);
-						}
-					} else {
-						current.append(nextTok);
+			case inQuote:
+				if ("\'".equals(nextTok)) {
+					lastTokenHasBeenQuoted = true;
+					state = normal;
+				} else {
+					current.append(nextTok);
+				}
+				break;
+			case inDoubleQuote:
+				if ("\"".equals(nextTok)) {
+					lastTokenHasBeenQuoted = true;
+					state = normal;
+				} else {
+					current.append(nextTok);
+				}
+				break;
+			default:
+				if ("\'".equals(nextTok)) {
+					state = inQuote;
+				} else if ("\"".equals(nextTok)) {
+					state = inDoubleQuote;
+				} else if (" ".equals(nextTok)) {
+					if (lastTokenHasBeenQuoted || current.length() > 0) {
+						result.add(current.toString());
+						current.setLength(0);
 					}
-					lastTokenHasBeenQuoted = false;
-					break;
+				} else {
+					current.append(nextTok);
+				}
+				lastTokenHasBeenQuoted = false;
+				break;
 			}
 		}
 		if (lastTokenHasBeenQuoted || current.length() > 0) {

o2server/x_bbs_core_entity/src/main/java/com/x/bbs/entity/BBSSubjectAttachment.java

@@ -29,7 +29,7 @@ import com.x.base.core.project.tools.DateTools;
 /**
  * 附件信息管理表
  */
-@ContainerEntity(dumpSize = 1000, type = ContainerEntity.Type.content, reference = ContainerEntity.Reference.strong)
+@ContainerEntity(dumpSize = 10, type = ContainerEntity.Type.content, reference = ContainerEntity.Reference.strong)
 @Entity
 @Table(name = PersistenceProperties.BBSSubjectAttachment.table, uniqueConstraints = {
 		@UniqueConstraint(name = PersistenceProperties.BBSSubjectAttachment.table + JpaObject.IndexNameMiddle

o2server/x_cms_core_entity/src/main/java/com/x/cms/core/entity/FileInfo.java

@@ -39,7 +39,7 @@ import com.x.base.core.project.tools.DateTools;
  * 内容管理应用目录分类信息
  *
  */
-@ContainerEntity(dumpSize = 1000, type = ContainerEntity.Type.content, reference = ContainerEntity.Reference.strong)
+@ContainerEntity(dumpSize = 5, type = ContainerEntity.Type.content, reference = ContainerEntity.Reference.strong)
 @Entity
 @Table(name = PersistenceProperties.FileInfo.table, uniqueConstraints = {
 		@UniqueConstraint(name = PersistenceProperties.FileInfo.table + JpaObject.IndexNameMiddle

o2server/x_console/src/main/java/com/x/server/console/DumpStorageTask.java

@@ -1,56 +1,56 @@
-package com.x.server.console;
-
-import java.io.File;
-import java.util.ArrayList;
-import java.util.Comparator;
-import java.util.List;
-import java.util.stream.Collectors;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.io.filefilter.FalseFileFilter;
-import org.apache.commons.io.filefilter.RegexFileFilter;
-import org.quartz.Job;
-import org.quartz.JobExecutionContext;
-import org.quartz.JobExecutionException;
-
-import com.x.base.core.project.config.Config;
-import com.x.base.core.project.logger.Logger;
-import com.x.base.core.project.logger.LoggerFactory;
-import com.x.server.console.action.DumpStorage;
-
-public class DumpStorageTask implements Job {
-
-	private static Logger logger = LoggerFactory.getLogger(DumpStorageTask.class);
-
-	@Override
-	public void execute(JobExecutionContext arg0) throws JobExecutionException {
-		try {
-			logger.print("schedule dump storage task start.");
-			DumpStorage action = new DumpStorage();
-			action.execute(Config.currentNode().dumpStorage().path());
-			if (Config.currentNode().dumpStorage().size() > 0) {
-				File dir = new File(Config.base(), "local/dump");
-				List<File> list = new ArrayList<>();
-				if (dir.exists() && dir.isDirectory()) {
-					for (File f : FileUtils.listFilesAndDirs(dir, FalseFileFilter.FALSE, new RegexFileFilter(
-							"^dumpStorage_[1,2][0,9][0-9][0-9][0,1][0-9][0-3][0-9][0-5][0-9][0-5][0-9][0-5][0-9]$"))) {
-						if (dir != f) {
-							list.add(f);
-						}
-					}
-					list = list.stream().sorted(Comparator.comparing(File::getName).reversed())
-							.collect(Collectors.toList());
-					if (list.size() > Config.currentNode().dumpStorage().size()) {
-						for (int i = Config.currentNode().dumpStorage().size(); i < list.size(); i++) {
-							File file = list.get(i);
-							logger.print("dumpStorageTask delete{}.", file.getAbsolutePath());
-							FileUtils.forceDelete(file);
-						}
-					}
-				}
-			}
-		} catch (Exception e) {
-			e.printStackTrace();
-		}
-	}
-}
+//package com.x.server.console;
+//
+//import java.io.File;
+//import java.util.ArrayList;
+//import java.util.Comparator;
+//import java.util.List;
+//import java.util.stream.Collectors;
+//
+//import org.apache.commons.io.FileUtils;
+//import org.apache.commons.io.filefilter.FalseFileFilter;
+//import org.apache.commons.io.filefilter.RegexFileFilter;
+//import org.quartz.Job;
+//import org.quartz.JobExecutionContext;
+//import org.quartz.JobExecutionException;
+//
+//import com.x.base.core.project.config.Config;
+//import com.x.base.core.project.logger.Logger;
+//import com.x.base.core.project.logger.LoggerFactory;
+//import com.x.server.console.action.DumpStorage;
+//
+//public class DumpStorageTask implements Job {
+//
+//	private static Logger logger = LoggerFactory.getLogger(DumpStorageTask.class);
+//
+//	@Override
+//	public void execute(JobExecutionContext arg0) throws JobExecutionException {
+//		try {
+//			logger.print("schedule dump storage task start.");
+//			DumpStorage action = new DumpStorage();
+//			action.execute(Config.currentNode().dumpStorage().path());
+//			if (Config.currentNode().dumpStorage().size() > 0) {
+//				File dir = new File(Config.base(), "local/dump");
+//				List<File> list = new ArrayList<>();
+//				if (dir.exists() && dir.isDirectory()) {
+//					for (File f : FileUtils.listFilesAndDirs(dir, FalseFileFilter.FALSE, new RegexFileFilter(
+//							"^dumpStorage_[1,2][0,9][0-9][0-9][0,1][0-9][0-3][0-9][0-5][0-9][0-5][0-9][0-5][0-9]$"))) {
+//						if (dir != f) {
+//							list.add(f);
+//						}
+//					}
+//					list = list.stream().sorted(Comparator.comparing(File::getName).reversed())
+//							.collect(Collectors.toList());
+//					if (list.size() > Config.currentNode().dumpStorage().size()) {
+//						for (int i = Config.currentNode().dumpStorage().size(); i < list.size(); i++) {
+//							File file = list.get(i);
+//							logger.print("dumpStorageTask delete{}.", file.getAbsolutePath());
+//							FileUtils.forceDelete(file);
+//						}
+//					}
+//				}
+//			}
+//		} catch (Exception e) {
+//			e.printStackTrace();
+//		}
+//	}
+//}

o2server/x_console/src/main/java/com/x/server/console/Main.java

@@ -3,8 +3,11 @@ package com.x.server.console;
 import java.io.BufferedReader;
 import java.io.File;
 import java.io.FileReader;
+import java.io.IOException;
 import java.io.InputStreamReader;
 import java.io.RandomAccessFile;
+import java.lang.management.ManagementFactory;
+import java.lang.management.RuntimeMXBean;
 import java.lang.reflect.Method;
 import java.net.URL;
 import java.net.URLClassLoader;
@@ -12,12 +15,22 @@ import java.nio.MappedByteBuffer;
 import java.nio.channels.FileChannel;
 import java.nio.channels.FileChannel.MapMode;
 import java.nio.channels.FileLock;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 import java.util.concurrent.LinkedBlockingQueue;
 import java.util.regex.Matcher;
 
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.FilenameUtils;
+import org.apache.commons.lang3.BooleanUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.quartz.Scheduler;
+
 import com.x.base.core.project.config.ApplicationServer;
 import com.x.base.core.project.config.CenterServer;
 import com.x.base.core.project.config.Config;
@@ -34,14 +47,6 @@ import com.x.server.console.action.ActionVersion;
 import com.x.server.console.log.LogTools;
 import com.x.server.console.server.Servers;
 
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.io.FilenameUtils;
-import org.apache.commons.lang3.BooleanUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.eclipse.jetty.deploy.App;
-import org.eclipse.jetty.deploy.DeploymentManager;
-import org.quartz.Scheduler;
-
 public class Main {
 
 	private static final String MANIFEST_FILENAME = "manifest.cfg";
@@ -50,6 +55,7 @@ public class Main {
 
 	public static void main(String[] args) throws Exception {
 		String base = getBasePath();
+		pid(base);
 		scanWar(base);
 		loadJars(base);
 		/* getVersion需要FileUtils在后面运行 */
@@ -620,4 +626,12 @@ public class Main {
 		return false;
 	}
 
+	private static void pid(String base) throws IOException {
+		RuntimeMXBean runtimeBean = ManagementFactory.getRuntimeMXBean();
+		String jvmName = runtimeBean.getName();
+		long pid = Long.parseLong(jvmName.split("@")[0]);
+		Path path = Paths.get(base, "pid.log");
+		Files.write(path, Long.toString(pid).getBytes(), StandardOpenOption.CREATE,
+				StandardOpenOption.TRUNCATE_EXISTING);
+	}
 }

o2server/x_console/src/main/java/com/x/server/console/RestoreStorageTask.java

@@ -1,29 +1,29 @@
-package com.x.server.console;
-
-import com.x.base.core.project.config.Config;
-import com.x.base.core.project.logger.Logger;
-import com.x.base.core.project.logger.LoggerFactory;
-import com.x.server.console.action.RestoreStorage;
-
-import org.quartz.Job;
-import org.quartz.JobExecutionContext;
-import org.quartz.JobExecutionException;
-
-public class RestoreStorageTask implements Job {
-
-	private static Logger logger = LoggerFactory.getLogger(RestoreStorageTask.class);
-
-	@Override
-	public void execute(JobExecutionContext arg0) throws JobExecutionException {
-		try {
-			logger.print("schedule restore storage task start, restore from:{}.",
-					Config.currentNode().restoreData().path());
-			RestoreStorage action = new RestoreStorage();
-			action.execute(Config.currentNode().restoreStorage().path());
-		} catch (Exception e) {
-			throw new JobExecutionException(e);
-		}
-
-	}
-
-}
+//package com.x.server.console;
+//
+//import com.x.base.core.project.config.Config;
+//import com.x.base.core.project.logger.Logger;
+//import com.x.base.core.project.logger.LoggerFactory;
+//import com.x.server.console.action.RestoreStorage;
+//
+//import org.quartz.Job;
+//import org.quartz.JobExecutionContext;
+//import org.quartz.JobExecutionException;
+//
+//public class RestoreStorageTask implements Job {
+//
+//	private static Logger logger = LoggerFactory.getLogger(RestoreStorageTask.class);
+//
+//	@Override
+//	public void execute(JobExecutionContext arg0) throws JobExecutionException {
+//		try {
+//			logger.print("schedule restore storage task start, restore from:{}.",
+//					Config.currentNode().restoreData().path());
+//			RestoreStorage action = new RestoreStorage();
+//			action.execute(Config.currentNode().restoreStorage().path());
+//		} catch (Exception e) {
+//			throw new JobExecutionException(e);
+//		}
+//
+//	}
+//
+//}

o2server/x_console/src/main/java/com/x/server/console/SchedulerBuilder.java

@@ -28,14 +28,14 @@ public class SchedulerBuilder {
 					.withSchedule(CronScheduleBuilder.cronSchedule(Config.currentNode().dumpData().cron())).build();
 			scheduler.scheduleJob(jobDetail, trigger);
 		}
-		if (Config.currentNode().dumpStorage().enable() && Config.currentNode().dumpStorage().available()) {
-			JobDetail jobDetail = JobBuilder.newJob(DumpStorageTask.class)
-					.withIdentity(DumpStorageTask.class.getName(), scheduleGroup).withDescription(Config.node())
-					.build();
-			Trigger trigger = TriggerBuilder.newTrigger().withIdentity(DumpStorageTask.class.getName(), scheduleGroup)
-					.withSchedule(CronScheduleBuilder.cronSchedule(Config.currentNode().dumpStorage().cron())).build();
-			scheduler.scheduleJob(jobDetail, trigger);
-		}
+//		if (Config.currentNode().dumpStorage().enable() && Config.currentNode().dumpStorage().available()) {
+//			JobDetail jobDetail = JobBuilder.newJob(DumpStorageTask.class)
+//					.withIdentity(DumpStorageTask.class.getName(), scheduleGroup).withDescription(Config.node())
+//					.build();
+//			Trigger trigger = TriggerBuilder.newTrigger().withIdentity(DumpStorageTask.class.getName(), scheduleGroup)
+//					.withSchedule(CronScheduleBuilder.cronSchedule(Config.currentNode().dumpStorage().cron())).build();
+//			scheduler.scheduleJob(jobDetail, trigger);
+//		}
 		if (Config.currentNode().restoreData().enable() && Config.currentNode().restoreData().available()) {
 			JobDetail jobDetail = JobBuilder.newJob(RestoreDataTask.class)
 					.withIdentity(RestoreDataTask.class.getName(), scheduleGroup).withDescription(Config.node())
@@ -44,16 +44,16 @@ public class SchedulerBuilder {
 					.withSchedule(CronScheduleBuilder.cronSchedule(Config.currentNode().restoreData().cron())).build();
 			scheduler.scheduleJob(jobDetail, trigger);
 		}
-		if (Config.currentNode().restoreStorage().enable() && Config.currentNode().restoreStorage().available()) {
-			JobDetail jobDetail = JobBuilder.newJob(RestoreStorageTask.class)
-					.withIdentity(RestoreStorageTask.class.getName(), scheduleGroup).withDescription(Config.node())
-					.build();
-			Trigger trigger = TriggerBuilder.newTrigger()
-					.withIdentity(RestoreStorageTask.class.getName(), scheduleGroup)
-					.withSchedule(CronScheduleBuilder.cronSchedule(Config.currentNode().restoreStorage().cron()))
-					.build();
-			scheduler.scheduleJob(jobDetail, trigger);
-		}
+//		if (Config.currentNode().restoreStorage().enable() && Config.currentNode().restoreStorage().available()) {
+//			JobDetail jobDetail = JobBuilder.newJob(RestoreStorageTask.class)
+//					.withIdentity(RestoreStorageTask.class.getName(), scheduleGroup).withDescription(Config.node())
+//					.build();
+//			Trigger trigger = TriggerBuilder.newTrigger()
+//					.withIdentity(RestoreStorageTask.class.getName(), scheduleGroup)
+//					.withSchedule(CronScheduleBuilder.cronSchedule(Config.currentNode().restoreStorage().cron()))
+//					.build();
+//			scheduler.scheduleJob(jobDetail, trigger);
+//		}
 		this.registApplicationsAndVoteCenterTask(scheduler, scheduleGroup);
 		return scheduler;
 

o2server/x_console/src/main/java/com/x/server/console/action/ActionControl.java

@@ -33,9 +33,9 @@ public class ActionControl extends ActionBase {
 	private static final String CMD_TD = "td";
 	private static final String CMD_EC = "ec";
 	private static final String CMD_DD = "dd";
-	private static final String CMD_DS = "ds";
+	// private static final String CMD_DS = "ds";
 	private static final String CMD_RD = "rd";
-	private static final String CMD_RS = "rs";
+	// private static final String CMD_RS = "rs";
 	private static final String CMD_CLH2 = "clh2";
 	private static final String CMD_UF = "uf";
 	private static final String CMD_DDL = "ddl";
@@ -63,12 +63,12 @@ public class ActionControl extends ActionBase {
 				ec(cmd);
 			} else if (cmd.hasOption(CMD_DD)) {
 				dd(cmd);
-			} else if (cmd.hasOption(CMD_DS)) {
-				ds(cmd);
+//			} else if (cmd.hasOption(CMD_DS)) {
+//				ds(cmd);
 			} else if (cmd.hasOption(CMD_RD)) {
 				rd(cmd);
-			} else if (cmd.hasOption(CMD_RS)) {
-				rs(cmd);
+//			} else if (cmd.hasOption(CMD_RS)) {
+//				rs(cmd);
 			} else if (cmd.hasOption(CMD_CLH2)) {
 				clh2(cmd);
 			} else if (cmd.hasOption(CMD_UF)) {
@@ -96,9 +96,9 @@ public class ActionControl extends ActionBase {
 		options.addOption(tdOption());
 		options.addOption(ecOption());
 		options.addOption(ddOption());
-		options.addOption(dsOption());
+		// options.addOption(dsOption());
 		options.addOption(rdOption());
-		options.addOption(rsOption());
+		// options.addOption(rsOption());
 		options.addOption(clh2Option());
 		options.addOption(ufOption());
 		options.addOption(ddlOption());
@@ -145,20 +145,20 @@ public class ActionControl extends ActionBase {
 				.desc("导出数据库服务器的数据转换成json格式保存到本地文件.").build();
 	}
 
-	private static Option dsOption() {
-		return Option.builder("ds").longOpt("dumpStorage").argName("path").hasArg().optionalArg(true)
-				.desc("导出存储服务器的文件数据转换成json格式保存到本地文件.").build();
-	}
+//	private static Option dsOption() {
+//		return Option.builder("ds").longOpt("dumpStorage").argName("path").hasArg().optionalArg(true)
+//				.desc("导出存储服务器的文件数据转换成json格式保存到本地文件.").build();
+//	}
 
 	private static Option rdOption() {
 		return Option.builder("rd").longOpt("restoreData").argName("path or date").hasArg()
 				.desc("将导出的json格式数据恢复到数据库服务器.").build();
 	}
 
-	private static Option rsOption() {
-		return Option.builder("rs").longOpt("restoreStorage").argName("path or date").hasArg()
-				.desc("将导出的json格式文件数据恢复到存储服务器.").build();
-	}
+//	private static Option rsOption() {
+//		return Option.builder("rs").longOpt("restoreStorage").argName("path or date").hasArg()
+//				.desc("将导出的json格式文件数据恢复到存储服务器.").build();
+//	}
 
 	private static Option ufOption() {
 		return Option.builder("uf").longOpt("updateFile").argName("path").hasArg().desc("升级服务器,升级前请注意备份.").build();
@@ -229,11 +229,11 @@ public class ActionControl extends ActionBase {
 		dumpData.execute(path);
 	}
 
-	private void ds(CommandLine cmd) throws Exception {
-		String path = Objects.toString(cmd.getOptionValue(CMD_DS), "");
-		DumpStorage dumpStorage = new DumpStorage();
-		dumpStorage.execute(path);
-	}
+//	private void ds(CommandLine cmd) throws Exception {
+//		String path = Objects.toString(cmd.getOptionValue(CMD_DS), "");
+//		DumpStorage dumpStorage = new DumpStorage();
+//		dumpStorage.execute(path);
+//	}
 
 	private void rd(CommandLine cmd) throws Exception {
 		String path = Objects.toString(cmd.getOptionValue(CMD_RD), "");
@@ -241,11 +241,11 @@ public class ActionControl extends ActionBase {
 		restoreData.execute(path);
 	}
 
-	private void rs(CommandLine cmd) throws Exception {
-		String path = Objects.toString(cmd.getOptionValue(CMD_RS), "");
-		RestoreStorage restoreStorage = new RestoreStorage();
-		restoreStorage.execute(path);
-	}
+//	private void rs(CommandLine cmd) throws Exception {
+//		String path = Objects.toString(cmd.getOptionValue(CMD_RS), "");
+//		RestoreStorage restoreStorage = new RestoreStorage();
+//		restoreStorage.execute(path);
+//	}
 
 	private void hs(CommandLine cmd) {
 		final Integer repeat = this.getArgInteger(cmd, CMD_HS, 1);

o2server/x_console/src/main/java/com/x/server/console/action/DumpData.java

@@ -1,12 +1,16 @@
 package com.x.server.console.action;
 
-import java.io.File;
-import java.io.IOException;
-import java.lang.reflect.InvocationTargetException;
+import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.Date;
 import java.util.List;
 import java.util.Objects;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Stream;
 
 import javax.persistence.EntityManager;
 import javax.persistence.EntityManagerFactory;
@@ -16,152 +20,198 @@ import javax.persistence.criteria.Predicate;
 import javax.persistence.criteria.Root;
 
 import org.apache.commons.beanutils.BeanUtils;
-import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.FilenameUtils;
+import org.apache.commons.io.file.PathUtils;
+import org.apache.commons.lang3.BooleanUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.openjpa.persistence.OpenJPAPersistence;
 
 import com.google.gson.Gson;
 import com.x.base.core.container.factory.PersistenceXmlHelper;
 import com.x.base.core.entity.JpaObject;
+import com.x.base.core.entity.StorageObject;
 import com.x.base.core.entity.annotation.ContainerEntity;
 import com.x.base.core.entity.annotation.ContainerEntity.Reference;
 import com.x.base.core.project.config.Config;
 import com.x.base.core.project.config.DumpRestoreData;
+import com.x.base.core.project.config.StorageMapping;
+import com.x.base.core.project.config.StorageMappings;
 import com.x.base.core.project.gson.XGsonBuilder;
 import com.x.base.core.project.logger.Logger;
 import com.x.base.core.project.logger.LoggerFactory;
 import com.x.base.core.project.tools.DateTools;
-import com.x.base.core.project.tools.DefaultCharset;
 import com.x.base.core.project.tools.ListTools;
 
 public class DumpData {
 
 	private static Logger logger = LoggerFactory.getLogger(DumpData.class);
 
-	private Date start = new Date();
-
-	private File dir;
-
-	private DumpRestoreDataCatalog catalog;
-
-	private Gson pureGsonDateFormated = XGsonBuilder.instance();
-
-	@SuppressWarnings("unchecked")
 	public boolean execute(String path) throws Exception {
+		Path dir = null;
+		Date start = new Date();
 		if (StringUtils.isEmpty(path)) {
-			this.dir = new File(Config.base(), "local/dump/dumpData_" + DateTools.compact(this.start));
+			dir = Paths.get(Config.base(), "local", "dump", "dumpData_" + DateTools.compact(start));
 		} else {
-			this.dir = new File(path);
-			if (dir.getAbsolutePath().startsWith(Config.base())) {
-				logger.print("path can not in base directory.");
+			dir = Paths.get(path);
+			if (dir.startsWith(Paths.get(Config.base()))) {
+				logger.warn("path can not in base directory.");
 				return false;
 			}
 		}
-		FileUtils.forceMkdir(this.dir);
-		this.catalog = new DumpRestoreDataCatalog();
-
-		/* 初始化完成 */
-
-		List<String> containerEntityNames = this.entities();
-
-		/** 过滤include exclude 条件 */
-		List<String> classNames = ListTools.includesExcludesWildcard(containerEntityNames,
-				Config.dumpRestoreData().getIncludes(), Config.dumpRestoreData().getExcludes());
-		logger.print("dump data find {} data to dump, start at {}.", classNames.size(), DateTools.format(start));
-		File persistence = new File(Config.dir_local_temp_classes(), DateTools.compact(this.start) + "_dump.xml");
-		PersistenceXmlHelper.write(persistence.getAbsolutePath(), classNames);
-		for (int i = 0; i < classNames.size(); i++) {
-			Class<JpaObject> cls = (Class<JpaObject>) Class.forName(classNames.get(i));
-			EntityManagerFactory emf = OpenJPAPersistence.createEntityManagerFactory(cls.getName(),
-					persistence.getName(), PersistenceXmlHelper.properties(cls.getName(), Config.slice().getEnable()));
-			EntityManager em = emf.createEntityManager();
-			try {
-				long estimateCount = this.estimateCount(em, cls);
-				logger.print("dump data({}/{}): {}, count: {}.", (i + 1), classNames.size(), cls.getName(),
-						estimateCount);
-				this.dump(cls, em, estimateCount);
-			} finally {
-				em.close();
-				emf.close();
-			}
-		}
-		FileUtils.write(new File(dir, "catalog.json"), pureGsonDateFormated.toJson(this.catalog),
-				DefaultCharset.charset);
-		logger.print("dump data completed, directory: {}, count: {}, elapsed: {} minutes.", dir.getAbsolutePath(),
-				this.count(), (System.currentTimeMillis() - start.getTime()) / 1000 / 60);
+		Files.createDirectories(dir);
+		Thread thread = new Thread(new RunnableImpl(dir, start));
+		thread.start();
 		return true;
 	}
 
-	private <T extends JpaObject> long estimateCount(EntityManager em, Class<T> cls) {
-		CriteriaBuilder cb = em.getCriteriaBuilder();
-		CriteriaQuery<Long> cq = cb.createQuery(Long.class);
-		Root<T> root = cq.from(cls);
-		cq.select(cb.count(root));
-		return em.createQuery(cq).getSingleResult();
-	}
+	public class RunnableImpl implements Runnable {
 
-	private Integer count() {
-		return this.catalog.values().stream().mapToInt(Integer::intValue).sum();
-	}
+		private Path dir;
+		private Date start;
+		private DumpRestoreDataCatalog catalog;
+		private Gson pureGsonDateFormated;
+
+		public RunnableImpl(Path dir, Date start) {
+			this.dir = dir;
+			this.start = start;
+			this.catalog = new DumpRestoreDataCatalog();
+			this.pureGsonDateFormated = XGsonBuilder.instance();
+		}
 
-	private <T> void dump(Class<T> cls, EntityManager em, long total)
-			throws IOException, IllegalAccessException, InvocationTargetException, NoSuchMethodException {
-		/** 创建最终存储文件的目录 */
-		File directory = new File(dir, cls.getName());
-		FileUtils.forceMkdir(directory);
-		FileUtils.cleanDirectory(directory);
-		ContainerEntity containerEntity = cls.getAnnotation(ContainerEntity.class);
-		String id = "";
-		List<T> list = null;
-		int count = 0;
-		int loop = 1;
-		int btach = (int) ((total + 0.0) / containerEntity.dumpSize());
-		do {
-			list = this.list(em, cls, id, containerEntity.dumpSize());
-			if (ListTools.isNotEmpty(list)) {
-				count = count + list.size();
-				id = BeanUtils.getProperty(list.get(list.size() - 1), JpaObject.id_FIELDNAME);
-				File file = new File(directory, count + ".json");
-				FileUtils.write(file, pureGsonDateFormated.toJson(list), DefaultCharset.charset);
-				logger.print("dumping {}/{} part of data:{}.", loop++, btach, cls.getName());
+		public void run() {
+			try {
+				List<String> classNames = entities();
+				logger.print("find {} data to dump, start at {}.", classNames.size(), DateTools.format(start));
+				Path xml = Paths.get(Config.dir_local_temp_classes().getAbsolutePath(),
+						DateTools.compact(start) + "_dump.xml");
+				PersistenceXmlHelper.write(xml.toString(), classNames);
+				StorageMappings storageMappings = Config.storageMappings();
+				Stream<String> stream = BooleanUtils.isTrue(Config.dumpRestoreData().getParallel())
+						? classNames.parallelStream()
+						: classNames.stream();
+				AtomicInteger idx = new AtomicInteger(1);
+				stream.forEach(className -> {
+					String nameOfThread = Thread.currentThread().getName();
+					EntityManagerFactory emf = null;
+					EntityManager em = null;
+					try {
+						Thread.currentThread().setName(DumpData.class.getName() + ":" + className);
+						@SuppressWarnings("unchecked")
+						Class<JpaObject> cls = (Class<JpaObject>) Class.forName(className);
+						emf = OpenJPAPersistence.createEntityManagerFactory(cls.getName(), xml.getFileName().toString(),
+								PersistenceXmlHelper.properties(cls.getName(), Config.slice().getEnable()));
+						em = emf.createEntityManager();
+						long estimateCount = estimateCount(em, cls);
+						logger.print("dump data({}/{}): {}, count: {}.", idx.getAndAdd(1), classNames.size(),
+								cls.getName(), estimateCount);
+						dump(cls, em, storageMappings, estimateCount);
+					} catch (Exception e) {
+						logger.error(new Exception(String.format("dump:%s error.", className), e));
+					} finally {
+						Thread.currentThread().setName(nameOfThread);
+						em.close();
+						emf.close();
+					}
+				});
+				Files.write(dir.resolve("catalog.json"),
+						pureGsonDateFormated.toJson(catalog).getBytes(StandardCharsets.UTF_8));
+				logger.print("dump data completed, directory: {}, count: {}, elapsed: {} minutes.", dir.toString(),
+						count(), (System.currentTimeMillis() - start.getTime()) / 1000 / 60);
+			} catch (Exception e) {
+				e.printStackTrace();
 			}
-			em.clear();
-		} while (ListTools.isNotEmpty(list));
-		this.catalog.put(cls.getName(), count);
-	}
+		}
+
+		@SuppressWarnings("unchecked")
+		private List<String> entities() throws Exception {
+			List<String> list = new ArrayList<>();
+			if (StringUtils.equals(Config.dumpRestoreData().getMode(), DumpRestoreData.TYPE_FULL)) {
+				list.addAll((List<String>) Config.resource(Config.RESOURCE_CONTAINERENTITYNAMES));
+				return list;
+			}
+			for (String str : (List<String>) Config.resource(Config.RESOURCE_CONTAINERENTITYNAMES)) {
+				Class<?> cls = Class.forName(str);
+				ContainerEntity containerEntity = cls.getAnnotation(ContainerEntity.class);
+				if (Objects.equals(containerEntity.reference(), Reference.strong)) {
+					list.add(str);
+				}
+			}
+			return ListTools.includesExcludesWildcard(list, Config.dumpRestoreData().getIncludes(),
+					Config.dumpRestoreData().getExcludes());
+		}
 
-	private <T> List<T> list(EntityManager em, Class<T> cls, String id, Integer size) {
-		CriteriaBuilder cb = em.getCriteriaBuilder();
-		CriteriaQuery<T> cq = cb.createQuery(cls);
-		Root<T> root = cq.from(cls);
-		Predicate p = cb.conjunction();
-		if (StringUtils.isNotEmpty(id)) {
-			p = cb.greaterThan(root.get(JpaObject.id_FIELDNAME), id);
+		private <T extends JpaObject> long estimateCount(EntityManager em, Class<T> cls) {
+			CriteriaBuilder cb = em.getCriteriaBuilder();
+			CriteriaQuery<Long> cq = cb.createQuery(Long.class);
+			Root<T> root = cq.from(cls);
+			cq.select(cb.count(root));
+			return em.createQuery(cq).getSingleResult();
 		}
-		cq.select(root).where(p).orderBy(cb.asc(root.get(JpaObject.id_FIELDNAME)));
-		return em.createQuery(cq).setMaxResults(size).getResultList();
-	}
 
-	/**
-	 * 根据设置的模式不同输出需要dump的entity className
-	 * 
-	 * @return
-	 * @throws Exception
-	 */
-	@SuppressWarnings("unchecked")
-	private List<String> entities() throws Exception {
-		List<String> list = new ArrayList<>();
-		if (StringUtils.equals(Config.dumpRestoreData().getMode(), DumpRestoreData.TYPE_FULL)) {
-			list.addAll((List<String>) Config.resource(Config.RESOURCE_CONTAINERENTITYNAMES));
-			return list;
+		private Integer count() {
+			return catalog.values().stream().mapToInt(Integer::intValue).sum();
 		}
-		for (String str : (List<String>) Config.resource(Config.RESOURCE_CONTAINERENTITYNAMES)) {
-			Class<?> cls = Class.forName(str);
+
+		private <T> List<T> list(EntityManager em, Class<T> cls, String id, Integer size) {
+			CriteriaBuilder cb = em.getCriteriaBuilder();
+			CriteriaQuery<T> cq = cb.createQuery(cls);
+			Root<T> root = cq.from(cls);
+			Predicate p = cb.conjunction();
+			if (StringUtils.isNotEmpty(id)) {
+				p = cb.greaterThan(root.get(JpaObject.id_FIELDNAME), id);
+			}
+			cq.select(root).where(p).orderBy(cb.asc(root.get(JpaObject.id_FIELDNAME)));
+			return em.createQuery(cq).setMaxResults(size).getResultList();
+		}
+
+		private <T> void dump(Class<T> cls, EntityManager em, StorageMappings storageMappings, long total)
+				throws Exception {
+			// 创建最终存储文件的目录
+			Path directory = dir.resolve(cls.getName());
+			Files.createDirectories(directory);
+			PathUtils.cleanDirectory(directory);
 			ContainerEntity containerEntity = cls.getAnnotation(ContainerEntity.class);
-			if (Objects.equals(containerEntity.reference(), Reference.strong)) {
-				list.add(str);
+			String id = "";
+			List<T> list = null;
+			int count = 0;
+			int loop = 1;
+			int btach = (int) Math.ceil((total + 0.0) / containerEntity.dumpSize());
+			do {
+				list = list(em, cls, id, containerEntity.dumpSize());
+				if (ListTools.isNotEmpty(list)) {
+					count = count + list.size();
+					id = BeanUtils.getProperty(list.get(list.size() - 1), JpaObject.id_FIELDNAME);
+					if (StorageObject.class.isAssignableFrom(cls)) {
+						Path sub = directory.resolve(count + "");
+						Files.createDirectories(sub);
+						binary(list, sub, storageMappings);
+					}
+					Files.write(directory.resolve(count + ".json"),
+							pureGsonDateFormated.toJson(list).getBytes(StandardCharsets.UTF_8));
+					logger.print("dump data {}/{} part of data:{}.", loop++, btach, cls.getName());
+				}
+				em.clear();
+			} while (ListTools.isNotEmpty(list));
+			catalog.put(cls.getName(), count);
+		}
+
+		private <T> void binary(List<T> list, Path sub, StorageMappings storageMappings) throws Exception {
+			for (T t : list) {
+				StorageObject s = (StorageObject) t;
+				String name = s.getStorage();
+				StorageMapping mapping = storageMappings.get(s.getClass(), name);
+				if (null == mapping && Config.dumpRestoreStorage().getExceptionInvalidStorage()) {
+					throw new ExceptionInvalidStorage(s);
+				}
+				if (null != mapping) {
+					Path p = sub.resolve(FilenameUtils.getName(s.path()));
+					if (s.existContent(mapping)) {
+						try (OutputStream out = Files.newOutputStream(p)) {
+							out.write(s.readContent(mapping));
+						}
+					}
+				}
 			}
 		}
-		return list;
 	}
 }

o2server/x_console/src/main/java/com/x/server/console/action/DumpStorage.java

@@ -1,261 +0,0 @@
-package com.x.server.console.action;
-
-import java.io.File;
-import java.io.FileOutputStream;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Objects;
-
-import javax.persistence.EntityManager;
-import javax.persistence.EntityManagerFactory;
-import javax.persistence.criteria.CriteriaBuilder;
-import javax.persistence.criteria.CriteriaQuery;
-import javax.persistence.criteria.Predicate;
-import javax.persistence.criteria.Root;
-
-import org.apache.commons.beanutils.BeanUtils;
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.io.FilenameUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.openjpa.persistence.OpenJPAPersistence;
-
-import com.google.gson.Gson;
-import com.x.base.core.container.factory.PersistenceXmlHelper;
-import com.x.base.core.entity.JpaObject;
-import com.x.base.core.entity.StorageObject;
-import com.x.base.core.entity.annotation.ContainerEntity;
-import com.x.base.core.entity.annotation.ContainerEntity.Reference;
-import com.x.base.core.project.config.Config;
-import com.x.base.core.project.config.DumpRestoreData;
-import com.x.base.core.project.config.StorageMapping;
-import com.x.base.core.project.config.StorageMappings;
-import com.x.base.core.project.gson.XGsonBuilder;
-import com.x.base.core.project.logger.Logger;
-import com.x.base.core.project.logger.LoggerFactory;
-import com.x.base.core.project.tools.DateTools;
-import com.x.base.core.project.tools.DefaultCharset;
-import com.x.base.core.project.tools.ListTools;
-
-public class DumpStorage {
-
-	private static Logger logger = LoggerFactory.getLogger(DumpStorage.class);
-
-	private Date start = new Date();
-
-	private File dir;
-
-	private DumpRestoreStorageCatalog catalog;
-
-	private Gson pureGsonDateFormated = XGsonBuilder.instance();
-
-	public boolean execute(String path) throws Exception {
-		if (StringUtils.isEmpty(path)) {
-			this.dir = new File(Config.base(), "local/dump/dumpStorage_" + DateTools.compact(this.start));
-		} else {
-			this.dir = new File(path);
-			if (dir.getAbsolutePath().startsWith(Config.base())) {
-				logger.print("path can not in base directory.");
-				return false;
-			}
-		}
-		FileUtils.forceMkdir(this.dir);
-		this.catalog = new DumpRestoreStorageCatalog();
-
-		List<String> storageContainerEntityNames = this.entities();
-		List<String> classNames = ListTools.includesExcludesWildcard(storageContainerEntityNames,
-				Config.dumpRestoreStorage().getIncludes(), Config.dumpRestoreStorage().getExcludes());
-		logger.print("dump storage find {} data to dump, start at {}.", classNames.size(), DateTools.format(start));
-		StorageMappings storageMappings = Config.storageMappings();
-		File persistence = new File(Config.dir_local_temp_classes(), DateTools.compact(this.start) + "_dump.xml");
-		PersistenceXmlHelper.write(persistence.getAbsolutePath(), classNames);
-		for (int i = 0; i < classNames.size(); i++) {
-			Class<StorageObject> cls = (Class<StorageObject>) Class.forName(classNames.get(i));
-			EntityManagerFactory emf = OpenJPAPersistence.createEntityManagerFactory(cls.getName(),
-					persistence.getName(), PersistenceXmlHelper.properties(cls.getName(), Config.slice().getEnable()));
-			EntityManager em = emf.createEntityManager();
-			try {
-				logger.print("dump storage({}/{}): {}, estimate count: {}, estimate size: {}M.", (i + 1),
-						classNames.size(), cls.getName(), this.estimateCount(em, cls),
-						(this.estimateSize(em, cls) / 1024 / 1024));
-				this.dump(cls, em, storageMappings);
-			} finally {
-				em.close();
-				emf.close();
-			}
-		}
-		FileUtils.write(new File(dir, "catalog.json"), XGsonBuilder.instance().toJson(this.catalog),
-				DefaultCharset.charset);
-		logger.print(
-				"dump storage completed, directory: {}, count: {}, normal: {}, empty: {}, invalidStorage: {}, size: {}M, elapsed: {} minutes.",
-				dir.getAbsolutePath(), this.count(), this.normal(), this.empty(), this.invalidStorage(),
-				(this.size() / 1024 / 1024), (System.currentTimeMillis() - start.getTime()) / 1000 / 60);
-		return true;
-	}
-
-	private Integer count() {
-		return this.catalog.values().stream().mapToInt(DumpRestoreStorageCatalogItem::getCount).sum();
-	}
-
-	private Long size() {
-		return this.catalog.values().stream().mapToLong(DumpRestoreStorageCatalogItem::getSize).sum();
-	}
-
-	private Long normal() {
-		return this.catalog.values().stream().mapToLong(DumpRestoreStorageCatalogItem::getNormal).sum();
-	}
-
-	private Long empty() {
-		return this.catalog.values().stream().mapToLong(DumpRestoreStorageCatalogItem::getEmpty).sum();
-	}
-
-	private Long invalidStorage() {
-		return this.catalog.values().stream().mapToLong(DumpRestoreStorageCatalogItem::getInvalidStorage).sum();
-	}
-
-	private <T> long estimateCount(EntityManager em, Class<T> cls) {
-		CriteriaBuilder cb = em.getCriteriaBuilder();
-		CriteriaQuery<Long> cq = cb.createQuery(Long.class);
-		Root<T> root = cq.from(cls);
-		cq.select(cb.count(root));
-		return em.createQuery(cq).getSingleResult();
-	}
-
-	private <T> long estimateSize(EntityManager em, Class<T> cls) {
-		CriteriaBuilder cb = em.getCriteriaBuilder();
-		CriteriaQuery<Long> cq = cb.createQuery(Long.class);
-		Root<T> root = cq.from(cls);
-		cq.select(root.get("length"));
-		List<Long> list = em.createQuery(cq).getResultList();
-		if (!list.isEmpty()) {
-			/** 上面的语句有可能返回的是null值,所以要先过滤 */
-			return list.stream().filter(o -> null != o).mapToLong(Long::longValue).sum();
-		} else {
-			return 0L;
-		}
-	}
-
-	private <T extends StorageObject> void dump(Class<T> cls, EntityManager em, StorageMappings storageMappings)
-			throws Exception {
-		/** 创建最终存储文件的目录 */
-		File classDirectory = new File(dir, cls.getName());
-		FileUtils.forceMkdir(classDirectory);
-		FileUtils.cleanDirectory(classDirectory);
-		int count = 0;
-		long size = 0L;
-		int normal = 0;
-		int invalidStorage = 0;
-		int empty = 0;
-		String id = "";
-		String name = "";
-		List<T> list = null;
-		File directory = null;
-		StorageMapping mapping = null;
-		List<T> normalList = null;
-		List<T> emptyList = null;
-		List<T> invalidStorageList = null;
-		ContainerEntity containerEntity = cls.getAnnotation(ContainerEntity.class);
-		do {
-			list = this.list(em, cls, id, containerEntity.dumpSize());
-			if (ListTools.isNotEmpty(list)) {
-				count += list.size();
-				directory = new File(classDirectory, Integer.toString(count));
-				FileUtils.forceMkdir(directory);
-				FileUtils.cleanDirectory(directory);
-				normalList = new ArrayList<T>();
-				emptyList = new ArrayList<T>();
-				invalidStorageList = new ArrayList<T>();
-				for (T t : list) {
-					name = t.getStorage();
-					mapping = storageMappings.get(cls, name);
-					if (StringUtils.isNotEmpty(name)) {
-						if (null == mapping && Config.dumpRestoreStorage().getExceptionInvalidStorage()) {
-							throw new Exception("can not find storageMapping class: " + cls.getName() + ", storage: "
-									+ name + ", id: " + t.getId() + ", name: " + t.getName()
-									+ ", set exceptionInvalidStorage to false will ignore item.");
-						}
-					}
-					if (null != mapping) {
-						File file = new File(directory, FilenameUtils.getName(t.path()));
-						if (t.existContent(mapping)) {
-							try (FileOutputStream fos = new FileOutputStream(file)) {
-								size += t.readContent(mapping, fos);
-								normalList.add(t);
-								normal++;
-							}
-						} else {
-							emptyList.add(t);
-							empty++;
-						}
-					} else {
-						invalidStorageList.add(t);
-						invalidStorage++;
-					}
-				}
-				id = BeanUtils.getProperty(list.get(list.size() - 1), JpaObject.id_FIELDNAME);
-				File file = new File(classDirectory, count + ".json");
-				this.dumpWrite(file, normalList, emptyList, invalidStorageList);
-			}
-			em.clear();
-		} while (ListTools.isNotEmpty(list));
-		DumpRestoreStorageCatalogItem item = new DumpRestoreStorageCatalogItem();
-		item.setCount(count);
-		item.setNormal(normal);
-		item.setEmpty(empty);
-		item.setSize(size);
-		item.setInvalidStorage(invalidStorage);
-		this.catalog.put(cls.getName(), item);
-		logger.print(
-				"dumped storage: " + cls.getName() + ", count: " + count + ", normal: " + normal + ", invalidStorage: "
-						+ invalidStorage + ", empty: " + empty + ", size: " + (size / 1024 / 1024) + "M.");
-	}
-
-	private <T extends StorageObject> void dumpWrite(File file, List<T> normalList, List<T> emptyList,
-			List<T> invalidStorageList) throws Exception {
-		LinkedHashMap<String, List<T>> o = new LinkedHashMap<>();
-		o.put("normals", normalList);
-		o.put("emptys", emptyList);
-		o.put("invalidStorages", invalidStorageList);
-		FileUtils.write(file, pureGsonDateFormated.toJson(o), DefaultCharset.charset);
-	}
-
-	private <T extends StorageObject> List<T> list(EntityManager em, Class<T> cls, String id, Integer size) {
-		CriteriaBuilder cb = em.getCriteriaBuilder();
-		CriteriaQuery<T> cq = cb.createQuery(cls);
-		Root<T> root = cq.from(cls);
-		Predicate p = cb.conjunction();
-		if (StringUtils.isNotEmpty(id)) {
-			p = cb.greaterThan(root.get("id"), id);
-		}
-		cq.select(root).where(p).orderBy(cb.asc(root.get("id")));
-		return em.createQuery(cq).setMaxResults(size).getResultList();
-	}
-
-	/**
-	 * 根据设置的模式不同输出需要dump的entity className
-	 * 
-	 * @return
-	 * @throws Exception
-	 */
-	@SuppressWarnings("unchecked")
-	private List<String> entities() throws Exception {
-		List<String> list = new ArrayList<>();
-		if (StringUtils.equals(Config.dumpRestoreData().getMode(), DumpRestoreData.TYPE_FULL)) {
-			list.addAll((List<String>) Config.resource(Config.RESOURCE_STORAGECONTAINERENTITYNAMES));
-			return list;
-		}
-		for (String str : (List<String>) Config.resource(Config.RESOURCE_STORAGECONTAINERENTITYNAMES)) {
-			Class<?> cls = Class.forName(str);
-			ContainerEntity containerEntity = cls.getAnnotation(ContainerEntity.class);
-			if (Objects.equals(containerEntity.reference(), Reference.strong)) {
-				list.add(str);
-			}
-		}
-		return list;
-	}
-
-	public static class Item {
-
-	}
-}

o2server/x_console/src/main/java/com/x/server/console/action/ExceptionDirectoryNotExist.java

@@ -0,0 +1,15 @@
+package com.x.server.console.action;
+
+import java.nio.file.Path;
+
+import com.x.base.core.project.exception.PromptException;
+
+class ExceptionDirectoryNotExist extends PromptException {
+
+	private static final long serialVersionUID = -5515077418025884395L;
+
+	ExceptionDirectoryNotExist(Path path) {
+		super("directory not exist: {}.", path.toString());
+	}
+
+}

o2server/x_console/src/main/java/com/x/server/console/action/ExceptionFileNotExist.java

@@ -0,0 +1,16 @@
+package com.x.server.console.action;
+
+
+import java.nio.file.Path;
+
+import com.x.base.core.project.exception.PromptException;
+
+class ExceptionFileNotExist extends PromptException {
+
+	private static final long serialVersionUID = -5515077418025884395L;
+
+	ExceptionFileNotExist(Path path) {
+		super("file not exist: {}.", path.toString());
+	}
+
+}

o2server/x_console/src/main/java/com/x/server/console/action/ExceptionInvalidStorage.java

@@ -0,0 +1,16 @@
+package com.x.server.console.action;
+
+import com.x.base.core.entity.StorageObject;
+import com.x.base.core.project.exception.PromptException;
+
+class ExceptionInvalidStorage extends PromptException {
+
+	private static final long serialVersionUID = -5515077418025884395L;
+
+	ExceptionInvalidStorage(StorageObject storageObject) {
+		super("can not find storageMapping class: " + storageObject.getClass().getName() + ", storage: "
+				+ storageObject.getStorage() + ", id: " + storageObject.getId() + ", name: " + storageObject.getName()
+				+ ", set exceptionInvalidStorage to false will ignore item.");
+	}
+
+}

o2server/x_console/src/main/java/com/x/server/console/action/ExceptionMappingNotExist.java

@@ -0,0 +1,13 @@
+package com.x.server.console.action;
+
+import com.x.base.core.project.exception.PromptException;
+
+class ExceptionMappingNotExist extends PromptException {
+
+	private static final long serialVersionUID = -5515077418025884395L;
+
+	ExceptionMappingNotExist() {
+		super("can not find storageMapping");
+	}
+
+}

o2server/x_console/src/main/java/com/x/server/console/action/RestoreData.java

@@ -1,176 +1,249 @@
 package com.x.server.console.action;
 
-import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
 import java.util.Date;
 import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.stream.Stream;
 
 import javax.persistence.EntityManager;
 import javax.persistence.EntityManagerFactory;
-import javax.persistence.FlushModeType;
 import javax.persistence.criteria.CriteriaBuilder;
 import javax.persistence.criteria.CriteriaQuery;
 import javax.persistence.criteria.Root;
 
+import org.apache.commons.io.FilenameUtils;
+import org.apache.commons.lang3.BooleanUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.openjpa.persistence.OpenJPAPersistence;
+
 import com.google.gson.Gson;
 import com.google.gson.JsonArray;
 import com.google.gson.JsonElement;
 import com.x.base.core.container.factory.PersistenceXmlHelper;
 import com.x.base.core.entity.JpaObject;
+import com.x.base.core.entity.StorageObject;
+import com.x.base.core.entity.annotation.ContainerEntity;
 import com.x.base.core.project.config.Config;
+import com.x.base.core.project.config.StorageMapping;
+import com.x.base.core.project.config.StorageMappings;
 import com.x.base.core.project.gson.XGsonBuilder;
 import com.x.base.core.project.logger.Logger;
 import com.x.base.core.project.logger.LoggerFactory;
-import com.x.base.core.project.tools.BaseTools;
 import com.x.base.core.project.tools.DateTools;
-import com.x.base.core.project.tools.DefaultCharset;
 import com.x.base.core.project.tools.ListTools;
 
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.io.FilenameUtils;
-import org.apache.commons.lang3.BooleanUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.openjpa.persistence.OpenJPAPersistence;
-
-/**
- * @author zhourui
- */
 public class RestoreData {
 
 	private static Logger logger = LoggerFactory.getLogger(RestoreData.class);
 
-	private Date start = new Date();
-
-	private File dir;
-
-	private DumpRestoreDataCatalog catalog;
-
-	private Gson pureGsonDateFormated = XGsonBuilder.instance();
-
 	public boolean execute(String path) throws Exception {
+		Date start = new Date();
+		Path dir;
 		if (StringUtils.isEmpty(path)) {
-			logger.print("path is empty.");
+			logger.warn("path is empty.");
 		}
 		if (BooleanUtils.isTrue(DateTools.isCompactDateTime(path))) {
-			this.dir = new File(Config.base(), "local/dump/dumpData_" + path);
-			this.catalog = BaseTools.readConfigObject("local/dump/dumpData_" + path + "/catalog.json",
-					DumpRestoreDataCatalog.class);
+			dir = Paths.get(Config.base(), "local", "dump", "dumpData_" + path);
 		} else {
-			this.dir = new File(path);
-			if (!(this.dir.exists() && this.dir.isDirectory())) {
-				logger.print("dir not exist: {}.", path);
+			dir = Paths.get(path);
+			if ((!Files.exists(dir)) || (!Files.isDirectory(dir))) {
+				logger.warn("directory not exist: {}.", path);
 				return false;
-			} else if (StringUtils.startsWith(dir.getAbsolutePath(), Config.base())) {
-				logger.print("path can not in base directory.");
+			} else if (dir.startsWith(Paths.get(Config.base()))) {
+				logger.warn("path can not in base directory.");
 				return false;
 			}
 		}
-		this.catalog = XGsonBuilder.instance().fromJson(
-				FileUtils.readFileToString(new File(dir, "catalog.json"), DefaultCharset.charset_utf_8),
-				DumpRestoreDataCatalog.class);
-		return this.execute();
+		Thread thread = new Thread(new RunnableImpl(dir, start));
+		thread.start();
+		return true;
 	}
 
-	@SuppressWarnings("unchecked")
-	public boolean execute() throws Exception {
-		List<String> containerEntityNames = new ArrayList<>();
-		containerEntityNames.addAll((List<String>) Config.resource(Config.RESOURCE_CONTAINERENTITYNAMES));
-		List<String> classNames = new ArrayList<>();
-		classNames.addAll(this.catalog.keySet());
-		classNames = ListTools.includesExcludesWildcard(classNames, Config.dumpRestoreData().getIncludes(),
-				Config.dumpRestoreData().getExcludes());
-		classNames = ListTools.includesExcludesWildcard(containerEntityNames, classNames, null);
-
-		logger.print("find: {} data to restore, path: {}.", classNames.size(), this.dir.getAbsolutePath());
-		File persistence = new File(Config.dir_local_temp_classes(), DateTools.compact(this.start) + "_dump.xml");
-		PersistenceXmlHelper.write(persistence.getAbsolutePath(), classNames);
-		long count = 0;
-		for (int i = 0; i < classNames.size(); i++) {
-			Class<JpaObject> cls = (Class<JpaObject>) Class.forName(classNames.get(i));
+	public class RunnableImpl implements Runnable {
+
+		private Path dir;
+		private Date start;
+		private DumpRestoreDataCatalog catalog;
+		private Gson gson;
+
+		public RunnableImpl(Path dir, Date start) throws IOException {
+			this.dir = dir;
+			this.start = start;
+			this.catalog = new DumpRestoreDataCatalog();
+			this.gson = XGsonBuilder.instance();
+			Path path = dir.resolve("catalog.json");
+			this.catalog = XGsonBuilder.instance().fromJson(
+					new String(Files.readAllBytes(path), StandardCharsets.UTF_8), DumpRestoreDataCatalog.class);
+		}
+
+		@Override
+		public void run() {
+			try {
+				List<String> classNames = this.entities();
+				logger.print("find: {} data to restore, path: {}.", classNames.size(), this.dir.toString());
+				Path xml = Paths.get(Config.dir_local_temp_classes().getAbsolutePath(),
+						DateTools.compact(start) + "_restore.xml");
+				PersistenceXmlHelper.write(xml.toString(), classNames);
+				Stream<String> stream = BooleanUtils.isTrue(Config.dumpRestoreData().getParallel())
+						? classNames.parallelStream()
+						: classNames.stream();
+				AtomicInteger idx = new AtomicInteger(1);
+				AtomicLong total = new AtomicLong(0);
+				stream.forEach(className -> {
+					String nameOfThread = Thread.currentThread().getName();
+					try {
+						Thread.currentThread().setName(RestoreData.class.getName() + ":" + className);
+						@SuppressWarnings("unchecked")
+						Class<JpaObject> cls = (Class<JpaObject>) Class.forName(className);
+						logger.print("restore data({}/{}): {}.", idx.getAndAdd(1), classNames.size(), cls.getName());
+						long size = restore(cls, xml);
+						total.getAndAdd(size);
+					} catch (Exception e) {
+						logger.error(new Exception(String.format("restore:%s error.", className), e));
+					} finally {
+						Thread.currentThread().setName(nameOfThread);
+					}
+				});
+				logger.print("restore data completed, directory: {}, count: {}, total: {}, elapsed: {} minutes.",
+						dir.toString(), idx.get(), total.longValue(),
+						(System.currentTimeMillis() - start.getTime()) / 1000 / 60);
+			} catch (Exception e) {
+				logger.error(e);
+			}
+		}
+
+		@SuppressWarnings("unchecked")
+		private List<String> entities() throws Exception {
+			List<String> containerEntityNames = new ArrayList<>();
+			containerEntityNames.addAll((List<String>) Config.resource(Config.RESOURCE_CONTAINERENTITYNAMES));
+			List<String> classNames = new ArrayList<>();
+			classNames.addAll(this.catalog.keySet());
+			classNames = ListTools.includesExcludesWildcard(classNames, Config.dumpRestoreData().getIncludes(),
+					Config.dumpRestoreData().getExcludes());
+			return ListTools.includesExcludesWildcard(containerEntityNames, classNames, null);
+		}
+
+		private long restore(Class<?> cls, Path xml) throws Exception {
 			EntityManagerFactory emf = OpenJPAPersistence.createEntityManagerFactory(cls.getName(),
-					persistence.getName(), PersistenceXmlHelper.properties(cls.getName(), Config.slice().getEnable()));
-			if (emf != null) {
-				EntityManager em = emf.createEntityManager();
-				em.setFlushMode(FlushModeType.COMMIT);
-				try {
-					logger.print("restore data({}/{}): {}, count: {}.", (i + 1), classNames.size(), cls.getName(),
-							catalog.get(cls.getName()));
-					count = count + this.store(cls, em);
-				} finally {
-					em.close();
-					emf.close();
+					xml.getFileName().toString(),
+					PersistenceXmlHelper.properties(cls.getName(), Config.slice().getEnable()));
+			AtomicLong count = new AtomicLong(0);
+			AtomicInteger batch = new AtomicInteger(1);
+			try {
+				Path directory = dir.resolve(cls.getName());
+				if ((!Files.exists(directory)) || (!Files.isDirectory(directory))) {
+					throw new ExceptionDirectoryNotExist(directory);
 				}
-			} else {
-				logger.warn("can not create 'EntityManagerFactory' for Entity:[" + cls.getName() + "]");
+				StorageMappings storageMappings = Config.storageMappings();
+				this.clean(cls, emf, storageMappings, cls.getAnnotation(ContainerEntity.class));
+				List<Path> paths = this.list(directory);
+				paths.stream().forEachOrdered(o -> {
+					logger.print("restore {}/{} part of data:{}.", batch.getAndAdd(1), paths.size(), cls.getName());
+					EntityManager em = null;
+					try {
+						em = emf.createEntityManager();
+						em.getTransaction().begin();
+						JsonArray raws = this.convert(o);
+						for (JsonElement json : raws) {
+							Object t = gson.fromJson(json, cls);
+							if (StorageObject.class.isAssignableFrom(cls)) {
+								Path sub = o.resolveSibling(FilenameUtils.getBaseName(o.getFileName().toString()));
+								this.binary(t, cls, sub, storageMappings);
+							}
+							em.persist(t);
+							count.getAndAdd(1);
+						}
+						em.getTransaction().commit();
+						em.clear();
+					} catch (Exception e) {
+						logger.error(new Exception(String.format("restore error with file:%s.", o.toString()), e));
+					} finally {
+						em.close();
+					}
+				});
+				logger.print("restore data: {} completed, count: {}.", cls.getName(), count.intValue());
+			} catch (Exception e) {
+				logger.error(e);
+			} finally {
+				emf.close();
 			}
+			return count.longValue();
 		}
-		logger.print("restore data completed, total count: {}, elapsed: {} minutes.", count,
-				(System.currentTimeMillis() - start.getTime()) / 1000 / 60);
-		return true;
-	}
 
-	private <T> long store(Class<T> cls, EntityManager em) throws Exception {
-		File directory = new File(this.dir, cls.getName());
-		if ((!directory.exists()) || (!directory.isDirectory())) {
-			throw new Exception("can not find directory: " + directory.getAbsolutePath() + ".");
+		private List<Path> list(Path directory) throws IOException {
+			List<Path> list = new ArrayList<>();
+			try (Stream<Path> stream = Files.list(directory)) {
+				stream.filter(p -> StringUtils.endsWithIgnoreCase(p.getFileName().toString(), ".json"))
+						.sorted((Path p1, Path p2) -> {
+							Integer i1 = Integer.parseInt(FilenameUtils.getBaseName(p1.getFileName().toString()));
+							Integer i2 = Integer.parseInt(FilenameUtils.getBaseName(p2.getFileName().toString()));
+							return i1.compareTo(i2);
+						}).forEach(list::add);
+			}
+			return list;
 		}
-		long count = 0;
-		List<File> files = new ArrayList<>(FileUtils.listFiles(directory, new String[] { "json" }, false));
-		/** 对文件进行排序,和dump的时候的顺序保持一直 */
-		Collections.sort(files, new Comparator<File>() {
-			public int compare(File o1, File o2) {
-				String n1 = FilenameUtils.getBaseName(o1.getName());
-				String n2 = FilenameUtils.getBaseName(o2.getName());
-				Integer i1 = Integer.parseInt(n1);
-				Integer i2 = Integer.parseInt(n2);
-				return i1.compareTo(i2);
+
+		@SuppressWarnings("unchecked")
+		private void binary(Object o, Class<?> cls, Path sub, StorageMappings storageMappings) throws Exception {
+			StorageObject so = (StorageObject) o;
+			StorageMapping mapping = null;
+			if (BooleanUtils.isTrue(Config.dumpRestoreData().getRedistribute())) {
+				mapping = storageMappings.random((Class<StorageObject>) cls);
+			} else {
+				mapping = storageMappings.get((Class<StorageObject>) cls, so.getStorage());
+			}
+			if (null == mapping) {
+				throw new ExceptionMappingNotExist();
+			}
+			Path path = sub.resolve(Paths.get(so.path()).getFileName());
+			if (!Files.exists(path)) {
+				throw new ExceptionFileNotExist(path);
 			}
-		});
-		/** 尽量在最后进行清空操作 */
-		this.clean(cls, em);
-		File file = null;
-		for (int i = 0; i < files.size(); i++) {
-			file = files.get(i);
-			logger.print("restoring {}/{} part of data:{}.", (i + 1), files.size(), cls.getName());
-			JsonArray raws = this.convert(file);
-			em.getTransaction().begin();
-			for (JsonElement o : raws) {
-				T t = pureGsonDateFormated.fromJson(o, cls);
-				em.persist(t);
-				count++;
+			try (InputStream input = Files.newInputStream(path)) {
+				so.saveContent(mapping, input, so.getName());
 			}
-			em.getTransaction().commit();
-			em.clear();
 		}
-		System.out.println("restore data: " + cls.getName() + " completed, count: " + count + ".");
-		return count;
-
-	}
 
-	private JsonArray convert(File file) throws Exception {
-		/* 必须先转换成 jsonElement 不能直接转成泛型T,如果直接转会有类型不匹配比如Integer变成了Double */
-		String json = FileUtils.readFileToString(file, DefaultCharset.charset);
-		JsonElement jsonElement = pureGsonDateFormated.fromJson(json, JsonElement.class);
-		return jsonElement.getAsJsonArray();
-	}
+		private JsonArray convert(Path path) throws IOException {
+			// 必须先转换成 jsonElement 不能直接转成泛型T,如果直接转会有类型不匹配比如Integer变成了Double
+			String json = new String(Files.readAllBytes(path), StandardCharsets.UTF_8);
+			JsonElement jsonElement = gson.fromJson(json, JsonElement.class);
+			return jsonElement.getAsJsonArray();
+		}
 
-	private <T> void clean(Class<T> cls, EntityManager em) throws Exception {
-		List<T> list = null;
-		do {
-			if (ListTools.isNotEmpty(list)) {
-				em.getTransaction().begin();
-				for (T t : list) {
-					em.remove(t);
+		private <T> void clean(Class<T> cls, EntityManagerFactory emf, StorageMappings storageMappings,
+				ContainerEntity containerEntity) throws Exception {
+			EntityManager em = emf.createEntityManager();
+			List<T> list = null;
+			do {
+				if (ListTools.isNotEmpty(list)) {
+					em.getTransaction().begin();
+					for (T t : list) {
+						em.remove(t);
+						if (StorageObject.class.isAssignableFrom(cls)) {
+							StorageObject so = (StorageObject) t;
+							@SuppressWarnings("unchecked")
+							StorageMapping mapping = storageMappings.get((Class<StorageObject>) cls, so.getStorage());
+							if (null != mapping) {
+								so.deleteContent(mapping);
+							}
+						}
+					}
+					em.getTransaction().commit();
 				}
-				em.getTransaction().commit();
-			}
-			CriteriaBuilder cb = em.getCriteriaBuilder();
-			CriteriaQuery<T> cq = cb.createQuery(cls);
-			Root<T> root = cq.from(cls);
-			cq.select(root);
-			list = em.createQuery(cq).setMaxResults(Config.dumpRestoreData().getBatchSize()).getResultList();
-		} while (ListTools.isNotEmpty(list));
+				CriteriaBuilder cb = em.getCriteriaBuilder();
+				CriteriaQuery<T> cq = cb.createQuery(cls);
+				Root<T> root = cq.from(cls);
+				list = em.createQuery(cq.select(root)).setMaxResults(containerEntity.dumpSize()).getResultList();
+			} while (ListTools.isNotEmpty(list));
+		}
 	}
 }

o2server/x_console/src/main/java/com/x/server/console/action/RestoreStorage.java

@@ -1,206 +0,0 @@
-package com.x.server.console.action;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.Date;
-import java.util.List;
-
-import javax.persistence.EntityManager;
-import javax.persistence.EntityManagerFactory;
-import javax.persistence.FlushModeType;
-import javax.persistence.criteria.CriteriaBuilder;
-import javax.persistence.criteria.CriteriaQuery;
-import javax.persistence.criteria.Root;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.io.FilenameUtils;
-import org.apache.commons.lang3.BooleanUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.openjpa.persistence.OpenJPAPersistence;
-
-import com.google.gson.Gson;
-import com.google.gson.JsonArray;
-import com.google.gson.JsonElement;
-import com.google.gson.JsonObject;
-import com.x.base.core.container.factory.PersistenceXmlHelper;
-import com.x.base.core.entity.StorageObject;
-import com.x.base.core.project.config.Config;
-import com.x.base.core.project.config.StorageMapping;
-import com.x.base.core.project.config.StorageMappings;
-import com.x.base.core.project.gson.XGsonBuilder;
-import com.x.base.core.project.logger.Logger;
-import com.x.base.core.project.logger.LoggerFactory;
-import com.x.base.core.project.tools.BaseTools;
-import com.x.base.core.project.tools.DateTools;
-import com.x.base.core.project.tools.DefaultCharset;
-import com.x.base.core.project.tools.ListTools;
-
-/**
- * @author zhourui
- */
-
-public class RestoreStorage {
-
-	private static Logger logger = LoggerFactory.getLogger(RestoreStorage.class);
-
-	private Date start = new Date();
-
-	private File dir;
-
-	private DumpRestoreStorageCatalog catalog;
-
-	private Gson pureGsonDateFormated = XGsonBuilder.instance();
-
-	public boolean execute(final String path) throws Exception {
-		if (StringUtils.isEmpty(path)) {
-			logger.print("path is empty.");
-		}
-		if (BooleanUtils.isTrue(DateTools.isCompactDateTime(path))) {
-			this.dir = new File(Config.base(), "local/dump/dumpStorage_" + path);
-			this.catalog = BaseTools.readConfigObject("local/dump/dumpStorage_" + path + "/catalog.json",
-					DumpRestoreStorageCatalog.class);
-		} else {
-			this.dir = new File(path);
-			if (!(this.dir.exists() && this.dir.isDirectory())) {
-				logger.print("dir not exist: {}.", path);
-				return false;
-			} else if (StringUtils.startsWith(dir.getAbsolutePath(), Config.base())) {
-				logger.print("path can not in base directory.");
-				return false;
-			}
-			this.catalog = XGsonBuilder.instance().fromJson(FileUtils
-					.readFileToString(new File(dir.getAbsolutePath(), "catalog.json"), DefaultCharset.charset_utf_8),
-					DumpRestoreStorageCatalog.class);
-		}
-		return this.execute();
-	}
-
-	private boolean execute() throws Exception {
-		final List<String> storageContainerEntityNames = new ArrayList<>();
-		storageContainerEntityNames.addAll((List<String>) Config.resource(Config.RESOURCE_STORAGECONTAINERENTITYNAMES));
-		List<String> classNames = new ArrayList<>();
-		classNames.addAll(this.catalog.keySet());
-		classNames = ListTools.includesExcludesWildcard(classNames, Config.dumpRestoreStorage().getIncludes(),
-				Config.dumpRestoreStorage().getExcludes());
-		logger.print("restore storage find {} to restore.", classNames.size());
-		final File persistence = new File(Config.dir_local_temp_classes(), DateTools.compact(this.start) + "_dump.xml");
-		PersistenceXmlHelper.write(persistence.getAbsolutePath(), classNames);
-		final StorageMappings storageMappings = Config.storageMappings();
-		int count = 0;
-		for (int i = 0; i < classNames.size(); i++) {
-			final Class<StorageObject> cls = (Class<StorageObject>) Class.forName(classNames.get(i));
-			final EntityManagerFactory emf = OpenJPAPersistence.createEntityManagerFactory(cls.getName(),
-					persistence.getName(), PersistenceXmlHelper.properties(cls.getName(), Config.slice().getEnable()));
-			final EntityManager em = emf.createEntityManager();
-			em.setFlushMode(FlushModeType.COMMIT);
-			try {
-				final DumpRestoreStorageCatalogItem item = this.catalog.get(cls.getName());
-				logger.print(
-						"restore storage({}/{}): {}, count: {}, normal: {} will be restore, invalidStorage: {} and empty: {} will be ignore, size: {}M.",
-						(i + 1), classNames.size(), cls.getName(), item.getCount(), item.getNormal(),
-						item.getInvalidStorage(), item.getEmpty(), (item.getSize() / 1024 / 1024));
-				count += this.store(cls, em, storageMappings);
-			} finally {
-				em.close();
-				emf.close();
-			}
-			logger.print("restore storage completed, total count: {}, elapsed: {} minutes.", count,
-					(System.currentTimeMillis() - start.getTime()) / 1000 / 60);
-		}
-		return false;
-	}
-
-	private <T extends StorageObject> long store(final Class<T> cls, final EntityManager em,
-			final StorageMappings storageMappings) throws Exception {
-		final File classDirectory = new File(this.dir, cls.getName());
-		if ((!classDirectory.exists()) || (!classDirectory.isDirectory())) {
-			throw new Exception("can not find directory: " + classDirectory.getAbsolutePath() + ".");
-		}
-		long count = 0;
-		final List<File> files = new ArrayList<File>(
-				FileUtils.listFiles(classDirectory, new String[] { "json" }, false));
-		/** 对文件进行排序,和dump的时候的顺序保持一直 */
-		Collections.sort(files, new Comparator<File>() {
-			public int compare(final File o1, final File o2) {
-				final String n1 = FilenameUtils.getBaseName(o1.getName());
-				final String n2 = FilenameUtils.getBaseName(o2.getName());
-				final Integer i1 = Integer.parseInt(n1);
-				final Integer i2 = Integer.parseInt(n2);
-				return i1.compareTo(i2);
-			}
-		});
-		/** 尽量将删除的工作放在后面 */
-		this.clean(cls, em, storageMappings);
-		StorageMapping mapping = null;
-		File file = null;
-		for (int i = 0; i < files.size(); i++) {
-			file = files.get(i);
-			/** 必须先转换成 jsonElement 不能直接转成泛型T,如果直接转会有类型不匹配比如Integer变成了Double */
-			logger.print("restoring " + (i + 1) + "/" + files.size() + " part of storage: " + cls.getName() + ".");
-			final JsonArray raws = this.convert(file);
-			if (null != raws) {
-				em.getTransaction().begin();
-				for (final JsonElement o : raws) {
-					final T t = pureGsonDateFormated.fromJson(o, cls);
-					if (Config.dumpRestoreStorage().getRedistribute()) {
-						mapping = storageMappings.random(cls);
-					} else {
-						mapping = storageMappings.get(cls, t.getStorage());
-					}
-					if (null == mapping) {
-						throw new Exception(
-								"can not find storageMapping class: " + cls.getName() + ", name:" + t.getName());
-					}
-					final File source = new File(classDirectory, FilenameUtils.getBaseName(file.getName())
-							+ StorageObject.PATHSEPARATOR + FilenameUtils.getName(t.path()));
-					try (FileInputStream input = new FileInputStream(source)) {
-						t.saveContent(mapping, input, t.getName());
-					}
-					em.persist(t);
-					count++;
-				}
-				em.getTransaction().commit();
-				em.clear();
-				Runtime.getRuntime().gc();
-			}
-		}
-		return count;
-	}
-
-	private JsonArray convert(final File file) throws IOException {
-		/** 这里不进行判断,因为格式是严格约定的,出现意外应该先报错停止 */
-		final String json = FileUtils.readFileToString(file, DefaultCharset.charset);
-		final JsonElement jsonElement = pureGsonDateFormated.fromJson(json, JsonElement.class);
-		final JsonObject jsonObject = jsonElement.getAsJsonObject();
-		return jsonObject.get("normals").getAsJsonArray();
-	}
-
-	private <T extends StorageObject> void clean(final Class<T> cls, final EntityManager em,
-			final StorageMappings storageMappings) throws Exception {
-		List<T> list = null;
-		StorageMapping mapping = null;
-		do {
-			if (ListTools.isNotEmpty(list)) {
-				em.getTransaction().begin();
-				for (final T t : list) {
-					mapping = storageMappings.get(cls, t.getStorage());
-					if (null != mapping) {
-						t.deleteContent(mapping);
-					}
-					em.remove(t);
-				}
-				em.getTransaction().commit();
-			}
-			final CriteriaBuilder cb = em.getCriteriaBuilder();
-			final CriteriaQuery<T> cq = cb.createQuery(cls);
-			final Root<T> root = cq.from(cls);
-			cq.select(root);
-			list = em.createQuery(cq).setMaxResults(Config.dumpRestoreData().getBatchSize()).getResultList();
-		} while (ListTools.isNotEmpty(list));
-	}
-
-}

o2server/x_file_core_entity/src/main/java/com/x/file/core/entity/personal/Attachment.java

@@ -36,7 +36,7 @@ import com.x.base.core.project.annotation.FieldDescribe;
 import com.x.base.core.project.tools.DateTools;
 import com.x.file.core.entity.PersistenceProperties;
 
-@ContainerEntity(dumpSize = 1000, type = ContainerEntity.Type.content, reference = ContainerEntity.Reference.strong)
+@ContainerEntity(dumpSize = 5, type = ContainerEntity.Type.content, reference = ContainerEntity.Reference.strong)
 @Entity
 @Table(name = PersistenceProperties.Personal.Attachment.table, uniqueConstraints = {
 		@UniqueConstraint(name = PersistenceProperties.Personal.Attachment.table + JpaObject.IndexNameMiddle

o2server/x_meeting_core_entity/src/main/java/com/x/meeting/core/entity/Attachment.java

@@ -23,7 +23,7 @@ import com.x.base.core.entity.annotation.CheckPersist;
 import com.x.base.core.entity.annotation.ContainerEntity;
 import com.x.base.core.project.annotation.FieldDescribe;
 
-@ContainerEntity(dumpSize = 1000, type = ContainerEntity.Type.content, reference = ContainerEntity.Reference.strong)
+@ContainerEntity(dumpSize = 5, type = ContainerEntity.Type.content, reference = ContainerEntity.Reference.strong)
 @Entity
 @Table(name = PersistenceProperties.Attachment.table, uniqueConstraints = {
 		@UniqueConstraint(name = PersistenceProperties.Attachment.table + JpaObject.IndexNameMiddle

o2server/x_message_core_entity/src/main/java/com/x/message/core/entity/IMMsgFile.java

@@ -20,7 +20,7 @@ import java.util.Date;
  */
 
 
-@ContainerEntity(dumpSize = 1000, type = ContainerEntity.Type.content, reference = ContainerEntity.Reference.strong)
+@ContainerEntity(dumpSize = 5, type = ContainerEntity.Type.content, reference = ContainerEntity.Reference.strong)
 @Entity
 @Table(name = PersistenceProperties.IMMsgFile.table, uniqueConstraints = {
         @UniqueConstraint(name = PersistenceProperties.IMMsgFile.table + JpaObject.IndexNameMiddle

o2server/x_okr_core_entity/src/main/java/com/x/okr/entity/OkrAttachmentFileInfo.java

@@ -30,7 +30,7 @@ import com.x.base.core.project.tools.DateTools;
  * 附件文件信息管理表
  *
  */
-@ContainerEntity(dumpSize = 1000, type = ContainerEntity.Type.content, reference = ContainerEntity.Reference.strong)
+@ContainerEntity(dumpSize = 5, type = ContainerEntity.Type.content, reference = ContainerEntity.Reference.strong)
 @Entity
 @Table(name = PersistenceProperties.OkrAttachmentFileInfo.table, uniqueConstraints = {
 		@UniqueConstraint(name = PersistenceProperties.OkrAttachmentFileInfo.table + JpaObject.IndexNameMiddle

o2server/x_processplatform_core_entity/src/main/java/com/x/processplatform/core/entity/content/Attachment.java

@@ -38,7 +38,7 @@ import com.x.base.core.project.tools.DateTools;
 import com.x.processplatform.core.entity.PersistenceProperties;
 import com.x.processplatform.core.entity.element.ActivityType;
 
-@ContainerEntity(dumpSize = 1000, type = ContainerEntity.Type.content, reference = ContainerEntity.Reference.strong)
+@ContainerEntity(dumpSize = 5, type = ContainerEntity.Type.content, reference = ContainerEntity.Reference.strong)
 @Entity
 @Table(name = PersistenceProperties.Content.Attachment.table, uniqueConstraints = {
 		@UniqueConstraint(name = PersistenceProperties.Content.Attachment.table + JpaObject.IndexNameMiddle

o2server/x_program_center_core_entity/src/main/java/com/x/program/center/core/entity/Structure.java

@@ -25,7 +25,7 @@ import com.x.base.core.project.annotation.FieldDescribe;
 import com.x.base.core.project.tools.StringTools;
 
 @Entity
-@ContainerEntity(dumpSize = 1000, type = ContainerEntity.Type.content, reference = ContainerEntity.Reference.strong)
+@ContainerEntity(dumpSize = 10, type = ContainerEntity.Type.content, reference = ContainerEntity.Reference.strong)
 @Table(name = PersistenceProperties.Structure.table, uniqueConstraints = {
 		@UniqueConstraint(name = PersistenceProperties.Structure.table + JpaObject.IndexNameMiddle
 				+ JpaObject.DefaultUniqueConstraintSuffix, columnNames = { JpaObject.IDCOLUMN,