Skip to content

N
neo4jRelation
Commit 96980932 by qiuchaofei
Options

1.添加etljob，etlscript模型，2修改table的查找，3修改同步流向关系的方法

parent a7b76387
Showing with 491 additions and 251 deletions
src/main/java/com/keymobile/metadata/metadataRelation/controller/RelationalGraphController.java
...@@ -145,9 +145,6 @@ public class RelationalGraphController { ...@@ -145,9 +145,6 @@ public class RelationalGraphController {
@RequestMapping(path = "/getTableBySchemaIdAndKeyWord", method = RequestMethod.GET) @RequestMapping(path = "/getTableBySchemaIdAndKeyWord", method = RequestMethod.GET)
public Map<String, List<ReturnNode>> getTableBySchemaIdAndKeyWord(String schemaId,String keyWord){ public Map<String, List<ReturnNode>> getTableBySchemaIdAndKeyWord(String schemaId,String keyWord){
//传入一个系统名称/id，返回系统下的所有schema，注意分层 //传入一个系统名称/id，返回系统下的所有schema，注意分层
Map<String, List<ReturnNode>> stringListMap = schemaService.getTablesBySchemaId(schemaId);
return tableService.autoMatchBySchemaIdAndInputWord(schemaId,keyWord); return tableService.autoMatchBySchemaIdAndInputWord(schemaId,keyWord);
......
src/main/java/com/keymobile/metadata/metadataRelation/pojo/metadata/Neo4jJob.java → src/main/java/com/keymobile/metadata/metadataRelation/pojo/metadata/Neo4jETLJob.java
...@@ -3,6 +3,7 @@ package com.keymobile.metadata.metadataRelation.pojo.metadata; ...@@ -3,6 +3,7 @@ package com.keymobile.metadata.metadataRelation.pojo.metadata;
import com.keymobile.metadata.metadataRelation.pojo.BaseNode; import com.keymobile.metadata.metadataRelation.pojo.BaseNode;
import org.neo4j.ogm.annotation.NodeEntity; import org.neo4j.ogm.annotation.NodeEntity;
@NodeEntity(label="Neo4jJob") @NodeEntity(label="Neo4jETLJob")
public class Neo4jJob extends BaseNode { public class Neo4jETLJob extends BaseNode {
} }
src/main/java/com/keymobile/metadata/metadataRelation/pojo/metadata/Neo4jETLScript.java 0 → 100644
package com.keymobile.metadata.metadataRelation.pojo.metadata;
import com.keymobile.metadata.metadataRelation.pojo.BaseNode;
import org.neo4j.ogm.annotation.NodeEntity;
@NodeEntity(label="Neo4jETLScript")
public class Neo4jETLScript extends BaseNode {
}
src/main/java/com/keymobile/metadata/metadataRelation/respository/metadata/Neo4jETLJobRepository.java 0 → 100644
package com.keymobile.metadata.metadataRelation.respository.metadata;
import com.keymobile.metadata.metadataRelation.pojo.metadata.Neo4jETLJob;
import com.keymobile.metadata.metadataRelation.pojo.metadata.Neo4jFunction;
import org.springframework.data.neo4j.repository.Neo4jRepository;
import org.springframework.stereotype.Repository;
import java.util.List;
@Repository
public interface Neo4jETLJobRepository extends Neo4jRepository<Neo4jETLJob, Long> {
List<Neo4jETLJob> findNeo4jETLJobByMetadataId(String metadataId);
}
src/main/java/com/keymobile/metadata/metadataRelation/respository/metadata/Neo4jETLScriptRepository.java 0 → 100644
package com.keymobile.metadata.metadataRelation.respository.metadata;
import com.keymobile.metadata.metadataRelation.pojo.metadata.Neo4jETLScript;
import org.springframework.data.neo4j.repository.Neo4jRepository;
import org.springframework.stereotype.Repository;
import java.util.List;
@Repository
public interface Neo4jETLScriptRepository extends Neo4jRepository<Neo4jETLScript, Long> {
List<Neo4jETLScript> findNeo4jETLScriptByMetadataId(String metadataId);
}
src/main/java/com/keymobile/metadata/metadataRelation/respository/metadata/Neo4jTableRepository.java
...@@ -13,10 +13,19 @@ import java.util.List; ...@@ -13,10 +13,19 @@ import java.util.List;
public interface Neo4jTableRepository extends Neo4jRepository<Neo4jTable,Long> { public interface Neo4jTableRepository extends Neo4jRepository<Neo4jTable,Long> {
List<Neo4jTable> findNeo4jTableByMetadataId(String metadataId); List<Neo4jTable> findNeo4jTableByMetadataId(String metadataId);
@Query("match (n{metadataId:{tableId}} )<-[r:流向]-(m) return m ") @Query("match (n{metadataId:{tableId}} )<-[r:流向]-(m1)<-[r1:流向]-(m) return m ")
List<BaseNode> getSourceTable(@Param("tableId") String tableId); List<BaseNode> getSourceTable(@Param("tableId") String tableId);
@Query("match (n{metadataId:{tableId}} )-[r:流向]->(m) return m ") @Query("match (n{metadataId:{tableId}} )-[r1:流向]->(m1)-[r:流向]->(m) return m ")
List<BaseNode> getTargetTable(@Param("tableId") String tableId); List<BaseNode> getTargetTable(@Param("tableId") String tableId);
@Query("match (n{metadataId:{tableId}} )<-[r1:流向]-(m1) <-[r:Composition]-(m) return m ")
List<BaseNode> getSourceEtlJob(@Param("tableId") String tableId);
@Query("match (n{metadataId:{tableId}} )-[r1:流向]->(m1) <-[r:Composition]-(m) return m ")
List<BaseNode> getTargetEtlJob(@Param("tableId") String tableId);
} }
src/main/java/com/keymobile/metadata/metadataRelation/service/AsyncDataFromMongoToNeo4j.java
...@@ -32,13 +32,10 @@ public class AsyncDataFromMongoToNeo4j { ...@@ -32,13 +32,10 @@ public class AsyncDataFromMongoToNeo4j {
@Autowired @Autowired
private JdbcTemplate jdbcTemplate; private JdbcTemplate jdbcTemplate;
@Async @Async
public void asyncDataFromMongoToNeo4j(String catalogName) { public void asyncDataFromMongoToNeo4j(String catalogName) {
long start = System.currentTimeMillis(); long start = System.currentTimeMillis();
String sql = "select scope_id, scope_name from auth_scope"; String sql = "select scope_id, scope_name from auth_scope where domain_id ="+ catalogName;
List<MongoData> mongoDataList = jdbcTemplate.query(sql, new RowMapper<MongoData>() { List<MongoData> mongoDataList = jdbcTemplate.query(sql, new RowMapper<MongoData>() {
@Override @Override
public MongoData mapRow(ResultSet resultSet, int i) throws SQLException { public MongoData mapRow(ResultSet resultSet, int i) throws SQLException {
......
src/main/java/com/keymobile/metadata/metadataRelation/service/impl/MetadataServiceImpl.java
...@@ -87,6 +87,11 @@ public class MetadataServiceImpl implements IMetadataService { ...@@ -87,6 +87,11 @@ public class MetadataServiceImpl implements IMetadataService {
@Autowired @Autowired
private Neo4jProcedureRepository neo4jProcedureRepository; private Neo4jProcedureRepository neo4jProcedureRepository;
@Autowired
private Neo4jETLJobRepository neo4jETLJobRepository;
@Autowired
private Neo4jETLScriptRepository neo4jETLScriptRepository;
@Override @Override
public List<MetaData> findNodeByName(String dataName) { public List<MetaData> findNodeByName(String dataName) {
return metadataRepository.findMetaData(dataName, 3); return metadataRepository.findMetaData(dataName, 3);
...@@ -1281,203 +1286,273 @@ public class MetadataServiceImpl implements IMetadataService { ...@@ -1281,203 +1286,273 @@ public class MetadataServiceImpl implements IMetadataService {
} }
systemDataMap.put(systemData.get_id(),neo4jSystem); systemDataMap.put(systemData.get_id(),neo4jSystem);
} }
List<CompositionRelation> compositionRelationList = new ArrayList<>(); List<CompositionRelation> compositionRelationList = new ArrayList<>();
int etlJobCount = 1;
int etlScriptCount = 1;
for(MongoData catalogData:catalogDataList){ for(MongoData catalogData:catalogDataList){
List<MongoData> dataBaseDataList= mongoDbServiceImpl.findDataByparentId(catalogData.get_id(),collectionName); List<MongoData> dataBaseDataList= mongoDbServiceImpl.findDataByparentId(catalogData.get_id(),collectionName);
for(MongoData dataBaseData:dataBaseDataList){ for(MongoData dataBaseData:dataBaseDataList){
List<MongoData> schemaDataList= mongoDbServiceImpl.findDataByparentId(dataBaseData.get_id(),collectionName); String mongoId = dataBaseData.get_id();
for(MongoData schemaData:schemaDataList){ if(mongoId.startsWith("Server=")){
//系统与schema的关系 //如果是server，就把etljob找出来
Neo4jSchema neo4jSchema = new Neo4jSchema();
neo4jSchema.setMetadataId(schemaData.get_id()); List<MongoData> etlJobDataList= mongoDbServiceImpl.findDataByparentId(dataBaseData.get_id(),collectionName);
neo4jSchema.setName(schemaData.getName());
neo4jSchema.setCnName(schemaData.getCnName()); List<Neo4jETLJob> neo4jETLJobList = new ArrayList<>();
Neo4jSystem neo4jSystem = null; for(MongoData etlJobData:etlJobDataList){
List<String> systemIdList = dataBaseData.getSysList(); if(etlJobCount%200 == 0){
for(String systemId:systemIdList){ logger.info("同步了"+etlJobCount);
if(systemDataMap.containsKey(systemId)){ neo4jETLJobRepository.saveAll(neo4jETLJobList);
neo4jSystem = systemDataMap.get(systemId); neo4jETLJobList.clear();
} }
Neo4jETLJob neo4jETLJob = new Neo4jETLJob();
neo4jETLJob.setMetadataId(etlJobData.get_id());
neo4jETLJob.setName(etlJobData.getName());
neo4jETLJob.setCnName(etlJobData.getCnName());
//如果存在，就不用创建
List<Neo4jETLJob> neo4jETLJobList1 = neo4jETLJobRepository.findNeo4jETLJobByMetadataId(neo4jETLJob.getMetadataId());
if(neo4jETLJobList1==null || neo4jETLJobList1.size()==0){
neo4jETLJobList.add(neo4jETLJob);
etlJobCount++;
}
List<Neo4jETLScript> neo4jETLScriptList = new ArrayList<>();
List<MongoData> etlScriptMongoList = mongoDbServiceImpl.findDataByparentId(etlJobData.get_id(),collectionName);
for(MongoData etlScriptMongo:etlScriptMongoList){
if(etlScriptCount%200==0){
neo4jETLScriptRepository.saveAll(neo4jETLScriptList);
neo4jETLScriptList.clear();
}
Neo4jETLScript neo4jETLScript = new Neo4jETLScript();
neo4jETLScript.setMetadataId(etlScriptMongo.get_id());
neo4jETLScript.setName(etlScriptMongo.getName());
neo4jETLScript.setCnName(etlScriptMongo.getCnName());
//如果存在，就不用创建
List<Neo4jETLScript> neo4jScriptList = neo4jETLScriptRepository.findNeo4jETLScriptByMetadataId(neo4jETLScript.getMetadataId());
if(neo4jScriptList==null || neo4jScriptList.size()==0){
neo4jETLScriptList.add(neo4jETLScript);
etlScriptCount++;
CompositionRelation compositionRelation = new CompositionRelation();
compositionRelation.setStart(neo4jETLJob);
compositionRelation.setEnd(neo4jETLScript);
compositionRelation.setName("etlJob--Script");
compositionRelationList.add(compositionRelation);
}
}
if(neo4jETLScriptList !=null && neo4jETLScriptList.size()!=0 ){
neo4jETLScriptRepository.saveAll(neo4jETLScriptList);
}
} }
String schemaPath = neo4jSystem.getDataPath()+";"+schemaData.getName(); if(neo4jETLJobList !=null && neo4jETLJobList.size()!=0 ){
neo4jSchema.setDataPath(schemaPath);
int ran1 = r.nextInt(6); neo4jETLJobRepository.saveAll(neo4jETLJobList);
if(ran1>6){
ran1 = 6;
}
if(neo4jSchema.getLabel()==null || neo4jSchema.getLabel().equals("")){
String label = layer.get(ran1);
neo4jSchema.setLabel(label);
}
//如果存在，就不用创建
List<Neo4jSchema> neo4jSchemaList = neo4jSchemaRepository.findNeo4jSchemaByMetadataId(neo4jSchema.getMetadataId());
if(neo4jSchemaList==null || neo4jSchemaList.size()==0){
neo4jSchemaRepository.save(neo4jSchema);
} }
CompositionRelation compositionRelation = new CompositionRelation();
compositionRelation.setStart(neo4jSystem); }else if(mongoId.startsWith("Database=")){
compositionRelation.setEnd(neo4jSchema); List<MongoData> schemaDataList= mongoDbServiceImpl.findDataByparentId(dataBaseData.get_id(),collectionName);
compositionRelation.setName("System--Schema"); for(MongoData schemaData:schemaDataList){
compositionRelationList.add(compositionRelation); //系统与schema的关系
Neo4jSchema neo4jSchema = new Neo4jSchema();
neo4jSchema.setMetadataId(schemaData.get_id());
neo4jSchema.setName(schemaData.getName());
neo4jSchema.setCnName(schemaData.getCnName());
Neo4jSystem neo4jSystem = null;
List<String> systemIdList = dataBaseData.getSysList();
for(String systemId:systemIdList){
if(systemDataMap.containsKey(systemId)){
neo4jSystem = systemDataMap.get(systemId);
}
}
String schemaPath = neo4jSystem.getDataPath()+";"+schemaData.getName();
neo4jSchema.setDataPath(schemaPath);
int ran1 = r.nextInt(6);
if(ran1>6){
ran1 = 6;
}
if(neo4jSchema.getLabel()==null || neo4jSchema.getLabel().equals("")){
String label = layer.get(ran1);
neo4jSchema.setLabel(label);
}
//如果存在，就不用创建
List<Neo4jSchema> neo4jSchemaList = neo4jSchemaRepository.findNeo4jSchemaByMetadataId(neo4jSchema.getMetadataId());
if(neo4jSchemaList==null || neo4jSchemaList.size()==0){
neo4jSchemaRepository.save(neo4jSchema);
schemaCount++;
}
CompositionRelation compositionRelation = new CompositionRelation();
compositionRelation.setStart(neo4jSystem);
compositionRelation.setEnd(neo4jSchema);
compositionRelation.setName("System--Schema");
compositionRelationList.add(compositionRelation);
// relationshipService.saveRelation(neo4jSystem.getMetadataId(),neo4jSchema.getMetadataId(),"Composition"); // relationshipService.saveRelation(neo4jSystem.getMetadataId(),neo4jSchema.getMetadataId(),"Composition");
//根据schema获取表级数据，再按照类型分类：表，视图，作业，函数等 //根据schema获取表级数据，再按照类型分类：表，视图，作业，函数等
List<MongoData> tableClassDataList = mongoDbServiceImpl.findDataByparentId(schemaData.get_id(),collectionName); List<MongoData> tableClassDataList = mongoDbServiceImpl.findDataByparentId(schemaData.get_id(),collectionName);
int count=1;//获取前100个表测试 int count=1;//获取前100个表测试
List<Neo4jTable> neo4jTableList = new ArrayList<>(); List<Neo4jTable> neo4jTableList = new ArrayList<>();
List<Neo4jView> viewTableList = new ArrayList<>(); List<Neo4jView> viewTableList = new ArrayList<>();
List<Neo4jFunction> functionTableList = new ArrayList<>(); List<Neo4jFunction> functionTableList = new ArrayList<>();
List<Neo4jProcedure> procedureTableList = new ArrayList<>(); List<Neo4jProcedure> procedureTableList = new ArrayList<>();
for(MongoData tableClassData:tableClassDataList){ for(MongoData tableClassData:tableClassDataList){
String metadataId = tableClassData.get_id(); String metadataId = tableClassData.get_id();
String endId = tableClassData.get_id(); String endId = tableClassData.get_id();
String tablePath = schemaPath+";"+tableClassData.getName(); String tablePath = schemaPath+";"+tableClassData.getName();
neo4jSchema.setDataPath(schemaPath); neo4jSchema.setDataPath(schemaPath);
if(metadataId.startsWith("Table=")){ if(metadataId.startsWith("Table=")){
if(tableCount %100==0){ if(tableCount %100==0){
// break; // break;
logger.info("同步了："+tableCount+"个表。"); logger.info("同步了："+tableCount+"个表。");
neo4jTableRepository.saveAll(neo4jTableList); neo4jTableRepository.saveAll(neo4jTableList);
neo4jTableList.clear(); neo4jTableList.clear();
} }
Neo4jTable neo4jTable = new Neo4jTable(); List<Neo4jTable> neo4jTableListExist = neo4jTableRepository.findNeo4jTableByMetadataId(metadataId);
neo4jTable.setMetadataId(tableClassData.get_id()); if(neo4jTableListExist == null || neo4jTableListExist.size() == 0){
neo4jTable.setName(tableClassData.getName());
neo4jTable.setCnName(tableClassData.getCnName()); Neo4jTable neo4jTable = new Neo4jTable();
neo4jTable.setDataPath(tablePath); neo4jTable.setMetadataId(tableClassData.get_id());
List<Neo4jTable> neo4jTableListExist = neo4jTableRepository.findNeo4jTableByMetadataId(metadataId); neo4jTable.setName(tableClassData.getName());
if(neo4jTableListExist == null || neo4jTableListExist.size() == 0){ neo4jTable.setCnName(tableClassData.getCnName());
neo4jTableList.add(neo4jTable); neo4jTable.setDataPath(tablePath);
CompositionRelation schem2Table = new CompositionRelation();
schem2Table.setStart(neo4jSchema);
schem2Table.setEnd(neo4jTable);
schem2Table.setName("System--Schema");
compositionRelationList.add(schem2Table);
tableCount++;
}
neo4jTableList.add(neo4jTable);
CompositionRelation schem2Table = new CompositionRelation();
schem2Table.setStart(neo4jSchema);
schem2Table.setEnd(neo4jTable);
schem2Table.setName("System--Schema");
compositionRelationList.add(schem2Table);
tableCount++;
}
}else if(metadataId.startsWith("View=")){
if(viewCount %100==0){ }else if(metadataId.startsWith("View=")){
if(viewCount %100==0){
// break; // break;
logger.info("同步了："+viewCount+"个表。"); logger.info("同步了："+viewCount+"个表。");
neo4jViewRepository.saveAll(viewTableList); neo4jViewRepository.saveAll(viewTableList);
viewTableList.clear(); viewTableList.clear();
} }
Neo4jView neo4jView = new Neo4jView(); Neo4jView neo4jView = new Neo4jView();
neo4jView.setMetadataId(tableClassData.get_id()); neo4jView.setMetadataId(tableClassData.get_id());
neo4jView.setName(tableClassData.getName()); neo4jView.setName(tableClassData.getName());
neo4jView.setCnName(tableClassData.getCnName()); neo4jView.setCnName(tableClassData.getCnName());
neo4jView.setDataPath(tablePath); neo4jView.setDataPath(tablePath);
List<Neo4jView> neo4jViewList = neo4jViewRepository.findNeo4jViewByMetadataId(metadataId); List<Neo4jView> neo4jViewList = neo4jViewRepository.findNeo4jViewByMetadataId(metadataId);
if(neo4jViewList==null || neo4jViewList.size()==0){ if(neo4jViewList==null || neo4jViewList.size()==0){
neo4jViewList.add(neo4jView); neo4jViewList.add(neo4jView);
CompositionRelation schem2View = new CompositionRelation(); CompositionRelation schem2View = new CompositionRelation();
schem2View.setStart(neo4jSchema); schem2View.setStart(neo4jSchema);
schem2View.setEnd(neo4jView); schem2View.setEnd(neo4jView);
schem2View.setName("Schema--View"); schem2View.setName("Schema--View");
compositionRelationList.add(schem2View); compositionRelationList.add(schem2View);
} }
}else if(metadataId.startsWith("Function=")){ }else if(metadataId.startsWith("Function=")){
if(functionCount %100==0){ if(functionCount %100==0){
// break; // break;
logger.info("同步了："+functionCount+"个函数。"); logger.info("同步了："+functionCount+"个函数。");
neo4jFunctionRepository.saveAll(functionTableList); neo4jFunctionRepository.saveAll(functionTableList);
functionTableList.clear(); functionTableList.clear();
} }
Neo4jFunction neo4jFunction = new Neo4jFunction(); Neo4jFunction neo4jFunction = new Neo4jFunction();
neo4jFunction.setMetadataId(tableClassData.get_id()); neo4jFunction.setMetadataId(tableClassData.get_id());
neo4jFunction.setName(tableClassData.getName()); neo4jFunction.setName(tableClassData.getName());
neo4jFunction.setCnName(tableClassData.getCnName()); neo4jFunction.setCnName(tableClassData.getCnName());
neo4jFunction.setDataPath(tablePath); neo4jFunction.setDataPath(tablePath);
List<Neo4jFunction> neo4jFunctionList = neo4jFunctionRepository.findNeo4jFunctionByMetadataId(metadataId); List<Neo4jFunction> neo4jFunctionList = neo4jFunctionRepository.findNeo4jFunctionByMetadataId(metadataId);
if(neo4jFunctionList==null || neo4jFunctionList.size()==0){ if(neo4jFunctionList==null || neo4jFunctionList.size()==0){
functionTableList.add(neo4jFunction); functionTableList.add(neo4jFunction);
// neo4jFunctionRepository.save(neo4jFunction); // neo4jFunctionRepository.save(neo4jFunction);
CompositionRelation schem2Function = new CompositionRelation(); CompositionRelation schem2Function = new CompositionRelation();
schem2Function.setStart(neo4jSchema); schem2Function.setStart(neo4jSchema);
schem2Function.setEnd(neo4jFunction); schem2Function.setEnd(neo4jFunction);
schem2Function.setName("Schema--Function"); schem2Function.setName("Schema--Function");
compositionRelationList.add(schem2Function); compositionRelationList.add(schem2Function);
functionCount++; functionCount++;
} }
}else if(metadataId.startsWith("Procedure=")){ }else if(metadataId.startsWith("Procedure=")){
if(procedureCount %100==0){ if(procedureCount %100==0){
// break; // break;
logger.info("同步了："+procedureCount+"个函数。"); logger.info("同步了："+procedureCount+"个函数。");
neo4jProcedureRepository.saveAll(procedureTableList); neo4jProcedureRepository.saveAll(procedureTableList);
procedureTableList.clear(); procedureTableList.clear();
} }
Neo4jProcedure neo4jProcedure = new Neo4jProcedure(); Neo4jProcedure neo4jProcedure = new Neo4jProcedure();
neo4jProcedure.setMetadataId(tableClassData.get_id()); neo4jProcedure.setMetadataId(tableClassData.get_id());
neo4jProcedure.setName(tableClassData.getName()); neo4jProcedure.setName(tableClassData.getName());
neo4jProcedure.setCnName(tableClassData.getCnName()); neo4jProcedure.setCnName(tableClassData.getCnName());
neo4jProcedure.setDataPath(tablePath); neo4jProcedure.setDataPath(tablePath);
List<Neo4jProcedure> neo4jProcedureList = neo4jProcedureRepository.findNeo4jProcedureByMetadataId(metadataId); List<Neo4jProcedure> neo4jProcedureList = neo4jProcedureRepository.findNeo4jProcedureByMetadataId(metadataId);
if(neo4jProcedureList==null || neo4jProcedureList.size()==0){ if(neo4jProcedureList==null || neo4jProcedureList.size()==0){
neo4jProcedureList.add(neo4jProcedure); neo4jProcedureList.add(neo4jProcedure);
// neo4jProcedureRepository.save(neo4jProcedure); // neo4jProcedureRepository.save(neo4jProcedure);
CompositionRelation schem2Procedure = new CompositionRelation(); CompositionRelation schem2Procedure = new CompositionRelation();
schem2Procedure.setStart(neo4jSchema); schem2Procedure.setStart(neo4jSchema);
schem2Procedure.setEnd(neo4jProcedure); schem2Procedure.setEnd(neo4jProcedure);
schem2Procedure.setName("Schema--Procedure"); schem2Procedure.setName("Schema--Procedure");
compositionRelationList.add(schem2Procedure); compositionRelationList.add(schem2Procedure);
procedureCount++; procedureCount++;
} }
}else{ }else{
} }
// relationshipService.saveRelation(neo4jSchema.getMetadataId(),endId,"Composition"); // relationshipService.saveRelation(neo4jSchema.getMetadataId(),endId,"Composition");
} }
if(neo4jTableList!=null && neo4jTableList.size()!=0){ if(neo4jTableList!=null && neo4jTableList.size()!=0){
neo4jTableRepository.saveAll(neo4jTableList); neo4jTableRepository.saveAll(neo4jTableList);
} }
if(viewTableList!=null && viewTableList.size()!=0){ if(viewTableList!=null && viewTableList.size()!=0){
neo4jViewRepository.saveAll(viewTableList); neo4jViewRepository.saveAll(viewTableList);
} }
if(functionTableList!=null && functionTableList.size()!=0){ if(functionTableList!=null && functionTableList.size()!=0){
neo4jFunctionRepository.saveAll(functionTableList); neo4jFunctionRepository.saveAll(functionTableList);
} }
if(procedureTableList != null && procedureTableList.size()!=0){ if(procedureTableList != null && procedureTableList.size()!=0){
neo4jProcedureRepository.saveAll(procedureTableList); neo4jProcedureRepository.saveAll(procedureTableList);
}
} }
} }
} }
...@@ -1495,94 +1570,194 @@ public class MetadataServiceImpl implements IMetadataService { ...@@ -1495,94 +1570,194 @@ public class MetadataServiceImpl implements IMetadataService {
if(newCompositionList.size()!=0){ if(newCompositionList.size()!=0){
compositionRelationRespository.saveAll(newCompositionList); compositionRelationRespository.saveAll(newCompositionList);
} }
logger.info("创建的system数量："+systemDataMap.size());
logger.info("创建的schema："+schemaCount);
logger.info("创建的table数量："+tableCount);
logger.info("创建的作业数量："+etlJobCount);
logger.info("创建的脚本数量："+etlScriptCount);
} }
@Override @Override
public void syschroTable2EtlJobRelations(String catalogName) { public void syschroTable2EtlJobRelations(String catalogName) {
Map<String, String> relationMap = new HashMap<>(); Map<String, String> relationMap = new HashMap<>();
List<RelationMongo> relationMongoList = mongoDbServiceImpl.findAllRelationByCatalog(catalogName);
int size = 0; int page =0,pageSize = 300;
for (RelationMongo relationMongo : relationMongoList) { long totalElement = mongoDbServiceImpl.countRelation(PageRequest.of(page,pageSize),catalogName);
String sourceId = relationMongo.getSource(); for(; page< totalElement;page+= pageSize){
String targetId = relationMongo.getTarget();
Map<String, Object> sourceData = metadataRepoRemoteService.getMetadata(sourceId); List<Document> relationList = mongoDbServiceImpl.findRelationByPage(PageRequest.of(page,pageSize,Sort.by("_id")),catalogName);
Map<String, Object> targetData = metadataRepoRemoteService.getMetadata(targetId); for(Document relation :relationList){
if(sourceData==null || targetData==null){
continue; String type = relation.getString("type");
}
String startId = ""; if(!(type.equals("Input") || type.equals("Output"))){
String endId = "";
//如果是字段，上升到表，如果是sql，上升到etljob，其他的模型站不处理
if(sourceId.contains("Column=") && targetId.contains("Column=") ){
Map<String, Object> sourceParent = metadataRepoRemoteService.getParent(sourceId);
Map<String, Object> targetParent = metadataRepoRemoteService.getParent(targetId);
if (sourceParent == null ) {
logger.info("没有找到id：" + sourceId + "的父节点元数据。");
continue;
}
if (targetParent==null) {
logger.info("没有找到id：" + targetId + "的父节点元数据。");
continue;
}
startId = (String )sourceParent.get("_id");
endId = (String )targetParent.get("_id");
} else if(sourceId.contains("Column=") && targetId.contains("SQL=") ){
Map<String, Object> sourceParent = metadataRepoRemoteService.getParent(sourceId);
Map<String, Object> targetParent = metadataRepoRemoteService.getParent(targetId);
if (sourceParent == null ) {
logger.info("没有找到id：" + sourceId + "的父节点元数据。");
continue;
}
if (targetParent==null) {
logger.info("没有找到id：" + targetId + "的父节点元数据。");
continue; continue;
} }
String targetParentId = (String) targetParent.get("_id");
Map<String, Object> targetParentParent = metadataRepoRemoteService.getParent(targetParentId);
if (targetParentParent==null) { String sourceId =relation.getString("source"); //relationMongo.getSource();
logger.info("没有找到id：" + targetId + "的祖父节点元数据。"); String targetId =relation.getString("target"); // relationMongo.getTarget();
Map<String, Object> sourceData = metadataRepoRemoteService.getMetadata(sourceId);
Map<String, Object> targetData = metadataRepoRemoteService.getMetadata(targetId);
if(sourceData==null || targetData==null){
continue; continue;
} }
String startId = "";
String endId = "";
//如果是字段，上升到表，如果是sql，上升到etljob，其他的模型站不处理
if(sourceId.contains("Column=") && targetId.contains("Column=") ){
Map<String, Object> sourceParent = metadataRepoRemoteService.getParent(sourceId);
Map<String, Object> targetParent = metadataRepoRemoteService.getParent(targetId);
if (sourceParent == null ) {
logger.info("没有找到id：" + sourceId + "的父节点元数据。");
continue;
}
if (targetParent==null) {
logger.info("没有找到id：" + targetId + "的父节点元数据。");
continue;
}
startId = (String )sourceParent.get("_id");
endId = (String )targetParent.get("_id");
} else if(sourceId.contains("Column=") && targetId.contains("SQL=") ){
Map<String, Object> sourceParent = metadataRepoRemoteService.getParent(sourceId);
Map<String, Object> targetParent = metadataRepoRemoteService.getParent(targetId);
if (sourceParent == null ) {
logger.info("没有找到id：" + sourceId + "的父节点元数据。");
continue;
}
if (targetParent==null) {
logger.info("没有找到id：" + targetId + "的父节点元数据。");
continue;
}
// String targetParentId = (String) targetParent.get("_id");
// Map<String, Object> targetParentParent = metadataRepoRemoteService.getParent(targetParentId);
// if (targetParentParent==null) {
// logger.info("没有找到id：" + targetId + "的祖父节点元数据。");
// continue;
// }
startId = (String )sourceParent.get("_id"); startId = (String )sourceParent.get("_id");
endId = (String )targetParentParent.get("_id"); endId = (String )targetParent.get("_id");
}else if(sourceId.contains("SQL=") && targetId.contains("Column=")){ }else if(sourceId.contains("SQL=") && targetId.contains("Column=")){
Map<String, Object> targetParent = metadataRepoRemoteService.getParent(sourceId); Map<String, Object> sourceParent = metadataRepoRemoteService.getParent(sourceId);
Map<String, Object> sourceParent = metadataRepoRemoteService.getParent(targetId); Map<String, Object> targetParent = metadataRepoRemoteService.getParent(targetId);
if (sourceParent == null ) { if (sourceParent == null ) {
logger.info("没有找到id：" + sourceId + "的父节点元数据。"); logger.info("没有找到id：" + sourceId + "的父节点元数据。");
continue; continue;
} }
if (targetParent==null) { if (targetParent==null) {
logger.info("没有找到id：" + targetId + "的父节点元数据。"); logger.info("没有找到id：" + targetId + "的父节点元数据。");
continue; continue;
}
// String sourceParenttId = (String) sourceParent.get("_id");
// Map<String, Object> sourceParentParent = metadataRepoRemoteService.getParent(sourceParenttId);
// if (sourceParentParent==null) {
// logger.info("没有找到id：" + sourceId + "的祖父节点元数据。");
// continue;
// }
startId = (String )sourceParent.get("_id");
endId = (String )targetParent.get("_id");
} }
String sourceParenttId = (String) sourceParent.get("_id");
Map<String, Object> sourceParentParent = metadataRepoRemoteService.getParent(sourceParenttId); String relationId = startId+"_"+endId;
if (sourceParentParent==null) { if(!relationMap.containsKey(relationId)){
logger.info("没有找到id：" + sourceId + "的祖父节点元数据。"); if(relationMap.size()%200 == 0){
continue; logger.info("创建第"+relationMap.size()+"多少关系：");
}
relationMap.put(relationId,"");
relationshipService.saveRelation(startId,endId,"流向");
} }
startId = (String )sourceParentParent.get("_id");
endId = (String )targetParent.get("_id");
} }
}
String type = relationMongo.getType();
String relationId = startId+"_"+endId;
if(!relationMap.containsKey(relationId)){
if(relationMap.size()%200 == 0){
logger.info("创建第"+relationMap.size()+"多少关系：");
}
relationMap.put(relationId,"");
relationshipService.saveRelation(startId,endId,"流向");
}
} // List<RelationMongo> relationMongoList = mongoDbServiceImpl.findAllRelationByCatalog(catalogName);
// int size = 0;
// for (RelationMongo relationMongo : relationMongoList) {
// String sourceId = relationMongo.getSource();
// String targetId = relationMongo.getTarget();
// Map<String, Object> sourceData = metadataRepoRemoteService.getMetadata(sourceId);
// Map<String, Object> targetData = metadataRepoRemoteService.getMetadata(targetId);
// if(sourceData==null || targetData==null){
// continue;
// }
// String startId = "";
// String endId = "";
// //如果是字段，上升到表，如果是sql，上升到etljob，其他的模型站不处理
// if(sourceId.contains("Column=") && targetId.contains("Column=") ){
//
// Map<String, Object> sourceParent = metadataRepoRemoteService.getParent(sourceId);
// Map<String, Object> targetParent = metadataRepoRemoteService.getParent(targetId);
// if (sourceParent == null ) {
// logger.info("没有找到id：" + sourceId + "的父节点元数据。");
// continue;
// }
// if (targetParent==null) {
// logger.info("没有找到id：" + targetId + "的父节点元数据。");
// continue;
// }
// startId = (String )sourceParent.get("_id");
// endId = (String )targetParent.get("_id");
// } else if(sourceId.contains("Column=") && targetId.contains("SQL=") ){
// Map<String, Object> sourceParent = metadataRepoRemoteService.getParent(sourceId);
// Map<String, Object> targetParent = metadataRepoRemoteService.getParent(targetId);
// if (sourceParent == null ) {
// logger.info("没有找到id：" + sourceId + "的父节点元数据。");
// continue;
// }
// if (targetParent==null) {
// logger.info("没有找到id：" + targetId + "的父节点元数据。");
// continue;
// }
// String targetParentId = (String) targetParent.get("_id");
// Map<String, Object> targetParentParent = metadataRepoRemoteService.getParent(targetParentId);
// if (targetParentParent==null) {
// logger.info("没有找到id：" + targetId + "的祖父节点元数据。");
// continue;
// }
//
// startId = (String )sourceParent.get("_id");
// endId = (String )targetParentParent.get("_id");
// }else if(sourceId.contains("SQL=") && targetId.contains("Column=")){
// Map<String, Object> sourceParent = metadataRepoRemoteService.getParent(sourceId);
//
// Map<String, Object> targetParent = metadataRepoRemoteService.getParent(targetId);
//
// if (sourceParent == null ) {
// logger.info("没有找到id：" + sourceId + "的父节点元数据。");
// continue;
// }
// if (targetParent==null) {
// logger.info("没有找到id：" + targetId + "的父节点元数据。");
// continue;
// }
// String sourceParenttId = (String) sourceParent.get("_id");
// Map<String, Object> sourceParentParent = metadataRepoRemoteService.getParent(sourceParenttId);
// if (sourceParentParent==null) {
// logger.info("没有找到id：" + sourceId + "的祖父节点元数据。");
// continue;
// }
//
// startId = (String )sourceParentParent.get("_id");
// endId = (String )targetParent.get("_id");
// }
//
// String type = relationMongo.getType();
// String relationId = startId+"_"+endId;
// if(!relationMap.containsKey(relationId)){
// if(relationMap.size()%200 == 0){
// logger.info("创建第"+relationMap.size()+1+"多少关系：");
// }
// relationMap.put(relationId,"");
// relationshipService.saveRelation(startId,endId,"流向");
// }
//
// }
logger.info("创建了多少关系："+relationMap.size()); logger.info("创建了多少关系："+relationMap.size());
} }
......
src/main/java/com/keymobile/metadata/metadataRelation/service/impl/MongoDbServiceImpl.java
...@@ -23,6 +23,7 @@ public class MongoDbServiceImpl { ...@@ -23,6 +23,7 @@ public class MongoDbServiceImpl {
@Autowired @Autowired
private MongoTemplate mongoTemplate; private MongoTemplate mongoTemplate;
private static final String PREFIX_MD_RELATION = "md_relation_";
...@@ -125,4 +126,16 @@ public class MongoDbServiceImpl { ...@@ -125,4 +126,16 @@ public class MongoDbServiceImpl {
query.with(pageable); query.with(pageable);
return mongoTemplate.count(query, Document.class, PREFIX_MD_RELATION_TEMP_NODE + catalogName); return mongoTemplate.count(query, Document.class, PREFIX_MD_RELATION_TEMP_NODE + catalogName);
} }
public List<Document> findRelationByPage(Pageable pageable, String catalogName) {
Query query = new Query(new Criteria().orOperator(Criteria.where("type").is("Input"),Criteria.where("type").is("Output")));
query.with(pageable);
return mongoTemplate.find(query, Document.class, PREFIX_MD_RELATION + catalogName);
}
public long countRelation(Pageable pageable, String catalogName) {
Query query = new Query(new Criteria().orOperator(Criteria.where("type").is("Input"),Criteria.where("type").is("Output")));
query.with(pageable);
return mongoTemplate.count(query, Document.class, PREFIX_MD_RELATION + catalogName);
}
} }
src/main/java/com/keymobile/metadata/metadataRelation/service/impl/TableServiceImpl.java
...@@ -50,23 +50,23 @@ public class TableServiceImpl implements ITableService { ...@@ -50,23 +50,23 @@ public class TableServiceImpl implements ITableService {
//作业 //作业
List<ReturnNode> etlJobs = new ArrayList<>(); List<ReturnNode> etlJobs = new ArrayList<>();
//获取关联作业， //获取关联作业，
List<BaseNode> sourceBaseNodes = neo4jTableRepository.getSourceTable(tableId); List<BaseNode> sourceBaseNodes = neo4jTableRepository.getSourceEtlJob(tableId);
for(BaseNode sourceBaseNode:sourceBaseNodes){ for(BaseNode sourceBaseNode:sourceBaseNodes){
if(sourceBaseNode.getMetadataId().startsWith("Procedure=")){ if(sourceBaseNode.getMetadataId().startsWith("ETLJob=")){
ReturnNode returnNode = new ReturnNode(); ReturnNode returnNode = new ReturnNode();
returnNode.setId(sourceBaseNode.getMetadataId()); returnNode.setId(sourceBaseNode.getMetadataId());
returnNode.setName(sourceBaseNode.getName()); returnNode.setName(sourceBaseNode.getName());
returnNode.setType("Procedure"); returnNode.setType("ETLJob");
etlJobs.add(returnNode); etlJobs.add(returnNode);
} }
} }
List<BaseNode> targetBaseNodes = neo4jTableRepository.getTargetTable(tableId); List<BaseNode> targetBaseNodes = neo4jTableRepository.getTargetEtlJob(tableId);
for(BaseNode targetBaseNode:targetBaseNodes){ for(BaseNode targetBaseNode:targetBaseNodes){
if(targetBaseNode.getMetadataId().startsWith("Procedure=")){ if(targetBaseNode.getMetadataId().startsWith("ETLJob=")){
ReturnNode returnNode = new ReturnNode(); ReturnNode returnNode = new ReturnNode();
returnNode.setId(targetBaseNode.getMetadataId()); returnNode.setId(targetBaseNode.getMetadataId());
returnNode.setName(targetBaseNode.getName()); returnNode.setName(targetBaseNode.getName());
returnNode.setType("Procedure"); returnNode.setType("ETLJob");
etlJobs.add(returnNode); etlJobs.add(returnNode);
} }
} }
...@@ -81,34 +81,46 @@ public class TableServiceImpl implements ITableService { ...@@ -81,34 +81,46 @@ public class TableServiceImpl implements ITableService {
relationObjects.put("模型",modelList); relationObjects.put("模型",modelList);
//资产 //资产
List<ReturnNode> assetList = new ArrayList<>(); List<ReturnNode> assetList = new ArrayList<>();
List<Map<String, String>> dataAssetGraphInfos = dataAssertRemoteService.getDataAssetGraphInfoByMetadataId(tableId); try{
for(Map<String, String> map : dataAssetGraphInfos){ List<Map<String, String>> dataAssetGraphInfos = dataAssertRemoteService.getDataAssetGraphInfoByMetadataId(tableId);
ReturnNode returnNode = new ReturnNode(); for(Map<String, String> map : dataAssetGraphInfos){
returnNode.setId(map.get("dataAssetId")); ReturnNode returnNode = new ReturnNode();
returnNode.setName(map.get("enName")); returnNode.setId(map.get("dataAssetId"));
returnNode.setCnName(map.get("cnName")); returnNode.setName(map.get("enName"));
assetList.add(returnNode); returnNode.setCnName(map.get("cnName"));
assetList.add(returnNode);
}
}catch (Exception e){
e.printStackTrace();
} }
relationObjects.put("资产",assetList); relationObjects.put("资产",assetList);
//标准 //标准
List<ReturnNode> standardList = new ArrayList<>(); List<ReturnNode> standardList = new ArrayList<>();
Map<String,Object> maps = new HashMap<>(); Map<String,Object> maps = new HashMap<>();
maps.put("metadataId",tableId); maps.put("metadataId",tableId);
Map<String, Object> standardMaps = dataStandardRemoteService.findStandardByMetadatId(1, 10,
maps);
List<LinkedHashMap<String,Object>> contentList = (List<LinkedHashMap<String,Object>>) standardMaps.get("content");
for(LinkedHashMap<String,Object> contentString:contentList){
String standardId = (String)contentString.get("standardId");
String enName = (String)contentString.get("enName");
String cnName = (String)contentString.get("cnName");
ReturnNode returnNode = new ReturnNode(); try{
returnNode.setId(standardId); Map<String, Object> standardMaps = dataStandardRemoteService.findStandardByMetadatId(1, 10,
returnNode.setName(enName); maps);
returnNode.setCnName(cnName); List<LinkedHashMap<String,Object>> contentList = (List<LinkedHashMap<String,Object>>) standardMaps.get("content");
standardList.add(returnNode); for(LinkedHashMap<String,Object> contentString:contentList){
String standardId = (String)contentString.get("standardId");
String enName = (String)contentString.get("enName");
String cnName = (String)contentString.get("cnName");
ReturnNode returnNode = new ReturnNode();
returnNode.setId(standardId);
returnNode.setName(enName);
returnNode.setCnName(cnName);
standardList.add(returnNode);
}
relationObjects.put("标准",standardList);
}catch (Exception e){
e.printStackTrace();
} }
relationObjects.put("标准",standardList);
List<ReturnNode> neo4jTableList = getCurrentTableInfo(tableId); List<ReturnNode> neo4jTableList = getCurrentTableInfo(tableId);
relationObjects.put("当前表",neo4jTableList); relationObjects.put("当前表",neo4jTableList);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment