Commit f909c3cf by qiuchaofei

1 关系添加属性:jobId,2 添加接口,给解析sql调用,更新局部关系。

parent cb037974
...@@ -556,6 +556,21 @@ public class MetaDataController { ...@@ -556,6 +556,21 @@ public class MetaDataController {
logger.info("返回结果,后台继续运行。"); logger.info("返回结果,后台继续运行。");
return flag; return flag;
} }
@ApiOperation(tags = "", value = "根据环境id与jobId同步元数据关系的接口")
@RequestMapping(path = "/synchroRelationByJobId", method = RequestMethod.POST)
public String synchroRelationByJobId(String catalogName,String jobId) {
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
LogManager.logInfo(LogConstants.CTX_Relation, "用户 "+ getUserName() + " 在 "+ sdf.format(new Date())+"同步"+catalogName+" JobId:"+jobId+"的元数据。" );
logger.info("开始异步迁移元数据:从mongo到neo4j");
String flag = "success";
asyncDataFromMongoToNeo4j.asyncJobFromMongo2Neo4j(catalogName,jobId);
logger.info("返回结果,后台继续运行。");
return flag;
}
private String getUserName() { private String getUserName() {
Object obj = SecurityContextHolder.getContext().getAuthentication().getPrincipal(); Object obj = SecurityContextHolder.getContext().getAuthentication().getPrincipal();
if (obj instanceof String) { if (obj instanceof String) {
......
...@@ -22,6 +22,8 @@ public class AsyncDataFromMongoToNeo4j { ...@@ -22,6 +22,8 @@ public class AsyncDataFromMongoToNeo4j {
private static final Logger logger = LoggerFactory.getLogger(AsyncDataFromMongoToNeo4j.class); private static final Logger logger = LoggerFactory.getLogger(AsyncDataFromMongoToNeo4j.class);
@Autowired @Autowired
private IMetadataService metadataService; private IMetadataService metadataService;
@Autowired
private IMetadataRelationService metadataRelationService;
@Autowired @Autowired
private JdbcTemplate jdbcTemplate; private JdbcTemplate jdbcTemplate;
...@@ -83,6 +85,12 @@ public class AsyncDataFromMongoToNeo4j { ...@@ -83,6 +85,12 @@ public class AsyncDataFromMongoToNeo4j {
logger.info("同步数据完成,用时:"+(end-start)); logger.info("同步数据完成,用时:"+(end-start));
} }
@Async
public void asyncJobFromMongo2Neo4j(String catalogName,String jobId) {
//先按照环境id,作业id,删除关系,然后同步关系,
// 一般来说,关系的字段已经存在neo4j,但是sql不一定存在,
// 如果sql不存在,需要先同步sql ,srcipt,job,再建立关系。
metadataRelationService.deleteRelationByJobId(catalogName,jobId);
metadataRelationService.syschroRelationFromMongo(catalogName,jobId);
}
} }
...@@ -69,6 +69,8 @@ public interface IBaseRelationshipService { ...@@ -69,6 +69,8 @@ public interface IBaseRelationshipService {
void saveRelation(String startMetadataId, String endMetadataid, String composition); void saveRelation(String startMetadataId, String endMetadataid, String composition);
void saveRelation(String startId, String endId, String relationType,String description); void saveRelation(String startId, String endId, String relationType,String jobId);
void saveRelation(String startId, String endId, String relationType,String jobId,String description);
} }
...@@ -8,4 +8,8 @@ public interface IMetadataRelationService { ...@@ -8,4 +8,8 @@ public interface IMetadataRelationService {
ReturnReslult filterJosnByModelName(String reulstJson,String filterModel,String retainModel); ReturnReslult filterJosnByModelName(String reulstJson,String filterModel,String retainModel);
//删除关系
void deleteRelationByJobId(String catalogName, String jobId);
//同步关系,如果元数据不存在,要先同步元数据
void syschroRelationFromMongo(String catalogName, String jobId);
} }
...@@ -183,7 +183,7 @@ public class BaseRelationshipServiceImpl implements IBaseRelationshipService { ...@@ -183,7 +183,7 @@ public class BaseRelationshipServiceImpl implements IBaseRelationshipService {
} }
@Override @Override
public void saveRelation(String startId, String endId, String relationType,String description) { public void saveRelation(String startId, String endId, String relationType,String jobId) {
Neo4jConfig neo4jConfig = new Neo4jConfig(); Neo4jConfig neo4jConfig = new Neo4jConfig();
Driver neo4jConnection = neo4jConfig.getNeo4jConnection(); Driver neo4jConnection = neo4jConfig.getNeo4jConnection();
Session session = neo4jConnection.session(); Session session = neo4jConnection.session();
...@@ -191,9 +191,26 @@ public class BaseRelationshipServiceImpl implements IBaseRelationshipService { ...@@ -191,9 +191,26 @@ public class BaseRelationshipServiceImpl implements IBaseRelationshipService {
// (m{metadataId:"Table=1=d644b631fa8c434e928bcd1f1665b060"}) // (m{metadataId:"Table=1=d644b631fa8c434e928bcd1f1665b060"})
// create (n)<-[r:Composition]-(m) // create (n)<-[r:Composition]-(m)
// return n,m,r // return n,m,r
String cypher = " match (n{metadataId:\""+startId+"\"}),(m{metadataId:\""+endId+"\"}) merge (n)-[r:"+relationType+"]->(m) set r.description ="+description; String cypher = " match (n{metadataId:\""+startId+"\"}),(m{metadataId:\""+endId+"\"}) merge (n)-[r:"+relationType+"]->(m) set r.jobId ="+jobId;
// logger.info("运行保存关系完成:" + cypher); // logger.info("运行保存关系完成:" + cypher);
session.run(cypher); session.run(cypher);
} }
@Override
public void saveRelation(String startId, String endId, String relationType,String jobId,String description) {
Neo4jConfig neo4jConfig = new Neo4jConfig();
Driver neo4jConnection = neo4jConfig.getNeo4jConnection();
Session session = neo4jConnection.session();
// match (n{metadataId:"Column=1=d9c2d67e56a3428e8e4f22918782437f"}),
// (m{metadataId:"Table=1=d644b631fa8c434e928bcd1f1665b060"})
// create (n)<-[r:Composition]-(m)
// return n,m,r
String cypher = " match (n{metadataId:\""+startId+"\"}),(m{metadataId:\""+endId+"\"}) " +
"merge (n)-[r:"+relationType+"]->(m) set r.jobId ="+description + " and r.description = "+ description;
// logger.info("运行保存关系完成:" + cypher);
session.run(cypher);
}
} }
...@@ -1852,13 +1852,11 @@ public class MetadataServiceImpl implements IMetadataService { ...@@ -1852,13 +1852,11 @@ public class MetadataServiceImpl implements IMetadataService {
relationshipService.saveRelation(sourceTableId,targetTableId,"流向"); relationshipService.saveRelation(sourceTableId,targetTableId,"流向");
} }
} }
@Override @Override
public void syschroTable2EtlJobRelations(String catalogName) { public void syschroTable2EtlJobRelations(String catalogName) {
Map<String, String> relationMap = new HashMap<>(); Map<String, String> relationMap = new HashMap<>();
Map<String,String> edgeIdMap = new HashMap<>(); Map<String,Map<String, String>> edgeIdMap = new HashMap<>();
int page =0,pageSize = 300; int page =0,pageSize = 300;
long time0 = System.currentTimeMillis(); long time0 = System.currentTimeMillis();
long totalElement = mongoDbServiceImpl.countRelation(PageRequest.of(page,pageSize),catalogName); long totalElement = mongoDbServiceImpl.countRelation(PageRequest.of(page,pageSize),catalogName);
...@@ -1869,13 +1867,17 @@ public class MetadataServiceImpl implements IMetadataService { ...@@ -1869,13 +1867,17 @@ public class MetadataServiceImpl implements IMetadataService {
logger.info("准备分页从mongo获取关系,每页:"+ pageSize); logger.info("准备分页从mongo获取关系,每页:"+ pageSize);
for(; page< totalPageCount;page++){ for(; page< totalPageCount;page++){
logger.info("开始从 mongo 第"+page+1 +" 次获取关系。"); logger.info("开始从 mongo 第"+page+1 +" 次获取关系。");
String description = "";
long time3 = System.currentTimeMillis(); long time3 = System.currentTimeMillis();
List<Document> relationList = mongoDbServiceImpl.findRelationByPage(PageRequest.of(page,pageSize,Sort.by("_id")),catalogName); List<Document> relationList = mongoDbServiceImpl.findRelationByPage(PageRequest.of(page,pageSize,Sort.by("_id")),catalogName);
long time4 = System.currentTimeMillis(); long time4 = System.currentTimeMillis();
logger.info("从 mongo 第"+page+1 +" 次获取关系结束,耗时:" +(time4-time3) ); logger.info("从 mongo 第"+page+1 +" 次获取关系结束,耗时:" +(time4-time3) );
for(Document relation :relationList){ for(Document relation :relationList){
description = relation.getString("description");
Map<String,String> attributeMap = new HashMap<>();
String jobId = relation.getString("jobId");
attributeMap.put("jobId",jobId);
String description = relation.getString("description");
attributeMap.put("description",description);
String type = relation.getString("type"); String type = relation.getString("type");
if(!(type.equals("Input") || type.equals("Output"))){ if(!(type.equals("Input") || type.equals("Output"))){
continue; continue;
...@@ -1884,7 +1886,7 @@ public class MetadataServiceImpl implements IMetadataService { ...@@ -1884,7 +1886,7 @@ public class MetadataServiceImpl implements IMetadataService {
String targetId =relation.getString("target"); // relationMongo.getTarget(); String targetId =relation.getString("target"); // relationMongo.getTarget();
//本身的字段级关系也要同步 //本身的字段级关系也要同步
edgeIdMap.put(sourceId+"_"+targetId,description); edgeIdMap.put(sourceId+"_"+targetId,attributeMap);
Map<String, Object> sourceData = metadataRepoRemoteService.getMetadata(sourceId); Map<String, Object> sourceData = metadataRepoRemoteService.getMetadata(sourceId);
Map<String, Object> targetData = metadataRepoRemoteService.getMetadata(targetId); Map<String, Object> targetData = metadataRepoRemoteService.getMetadata(targetId);
...@@ -1953,7 +1955,7 @@ public class MetadataServiceImpl implements IMetadataService { ...@@ -1953,7 +1955,7 @@ public class MetadataServiceImpl implements IMetadataService {
endId = (String )targetParent.get("_id"); endId = (String )targetParent.get("_id");
} }
String relationId = startId+"_"+endId; String relationId = startId+"_"+endId;
edgeIdMap.put(relationId,description); edgeIdMap.put(relationId,attributeMap);
} }
} }
...@@ -1967,11 +1969,14 @@ public class MetadataServiceImpl implements IMetadataService { ...@@ -1967,11 +1969,14 @@ public class MetadataServiceImpl implements IMetadataService {
String[] dataIds = edgeId.split("_"); String[] dataIds = edgeId.split("_");
String startId = dataIds[0]; String startId = dataIds[0];
String endId = dataIds[1]; String endId = dataIds[1];
String description = edgeIdMap.get(obj); Map<String,String> attributeMap0 = edgeIdMap.get(obj);
String description = attributeMap0.get("description");
String jobId = attributeMap0.get("jobId");
// String description = edgeIdMap.get(obj);
if(description ==null || description.equalsIgnoreCase("")){ if(description ==null || description.equalsIgnoreCase("")){
relationshipService.saveRelation(startId,endId,"流向"); relationshipService.saveRelation(startId,endId,"流向",jobId);
}else{ }else{
relationshipService.saveRelation(startId,endId,"流向",description); relationshipService.saveRelation(startId,endId,"流向",jobId,description);
} }
} }
......
...@@ -165,6 +165,27 @@ public class MongoDbServiceImpl { ...@@ -165,6 +165,27 @@ public class MongoDbServiceImpl {
return mongoTemplate.count(query, Document.class, PREFIX_MD_RELATION + catalogName); return mongoTemplate.count(query, Document.class, PREFIX_MD_RELATION + catalogName);
} }
public long countRelationByJobId(Pageable pageable, String catalogName,String jobId) {
Query query = new Query(new Criteria() .andOperator(Criteria.where("jobId").is(jobId),
new Criteria().orOperator(Criteria.where("type").is("Input"),Criteria.where("type").is("Output"))
)
);
// Query query = new Query(new Criteria().orOperator(Criteria.where("type").is("Input"),Criteria.where("type").is("Output")));
query.with(pageable);
return mongoTemplate.count(query, Document.class, PREFIX_MD_RELATION + catalogName);
}
public List<Document> findRelationByJobIdByPage(Pageable pageable, String catalogName,String jobId) {
Query query = new Query(new Criteria() .andOperator(Criteria.where("jobId").is(jobId),
new Criteria().orOperator(Criteria.where("type").is("Input"),Criteria.where("type").is("Output"))
)
);
// Query query = new Query(new Criteria().orOperator(Criteria.where("type").is("Input"),Criteria.where("type").is("Output")));
query.with(pageable);
return mongoTemplate.find(query, Document.class, PREFIX_MD_RELATION + catalogName);
}
public List<Document> find1104Relations( String catalogName) { public List<Document> find1104Relations( String catalogName) {
return mongoTemplate.findAll(Document.class, "Relation" + catalogName); return mongoTemplate.findAll(Document.class, "Relation" + catalogName);
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment