Commit f3008c19 by qiuchaofei

1.修改分页获取的数目*2

parent 63cbbd76
......@@ -70,15 +70,22 @@ public class AsyncDataFromMongoToNeo4j {
}
});
//先删除同个环境的数据
long time0 = System.currentTimeMillis();
metadataService.deleteNodeByCatalogName(catalogName);
long time1 = System.currentTimeMillis();
logger.info("删除"+catalogName+"的数据完成,耗时:"+(time1-time0));
//先同步元数据,纵向关系,root开始,系统--schema--table--column,系统--etl作业--etl脚本--etlsql。
metadataService.syschroMetadataFromMongo(catalogName,mongoDataList);
//再同步元数据的横向关系
metadataService.syschroTable2EtlJobRelations(catalogName);
//在同步表与表之间的横向关系
metadataService.syschroTable2TableRelations(catalogName);
//同步1104报表
// metadataService.syschro1104Relations(catalogName);
// metadataService.syschro1104Relations(catalogName);
// metadataService.syschroRelationFromMongo(catalogName);
// // 同步其他数据(模型,标准,质量等)
//
......
......@@ -1232,9 +1232,7 @@ public class MetadataServiceImpl implements IMetadataService {
long columnTime1 = System.currentTimeMillis();
//专门做字段解析,批量获取字段,进行组装。
saveAllColumnByBatch(catalogName,compositionRelationList,neo4jTableIdMap);
long columnTime2 = System.currentTimeMillis();
logger.info("获取字段用时:"+(columnTime2-columnTime1));
logger.info("创建的system数量:"+systemDataMap.size());
logger.info("创建的schema:"+(objectCountMap.get(SchemaCountString)-1));
// logger.info("创建的table数量:"+(objectCountMap.get(TableCountString)-1));
......@@ -1306,13 +1304,17 @@ public class MetadataServiceImpl implements IMetadataService {
schem2Table.setEnd(neo4jTable);
schem2Table.setName("Schema--Table");
compositionRelationList.add(schem2Table);
tableCount++;
neo4jTableIdMap.put(neo4jTable.getMetadataId(),neo4jTable);
if(tableCount % batchSize ==0){
neo4jTableRepository.saveAll(neo4jTableList);
logger.info("同步了:"+tableCount+"个表。"+tableId);
neo4jTableList.clear();
}
tableCount++;
}
if(tableCursor != null){
tableCursor.close();
}
if(neo4jTableList.size()!=0){
......@@ -1383,9 +1385,12 @@ public class MetadataServiceImpl implements IMetadataService {
private void saveAllColumnByBatch(String catalogName,List<CompositionRelation> compositionRelationList,
Map<String,Neo4jTable> neo4jTableIdMap) {
long columnTime1 = System.currentTimeMillis();
int columnCount = 0;// objectCountMap.get(ColumnCountString);
String ColumnPrefix = "Catalog,Database,Schema,Table,Column";
int batchSize = 2000;
int batchSize = 5000;
BasicDBObject query = new BasicDBObject();
query.put("_class", ColumnPrefix);
MongoCursor<Document> columnCursor = this.mongoTemplate.getCollection(PREFIX_METADATA_NODE + catalogName).
......@@ -1424,8 +1429,8 @@ public class MetadataServiceImpl implements IMetadataService {
neo4jColumnRepository.saveAll(neo4jColumnList);
neo4jColumnList.clear();
}
logger.info("创建的column数量:"+columnCount);
long columnTime2 = System.currentTimeMillis();
logger.info("创建的column数量:"+columnCount+",用时:"+(columnTime2-columnTime1));
}
private void saveTableDocument(List<Document> tableDocumentList, Neo4jSchema neo4jSchema, Map<String, Integer> objectCountMap,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment