Commit f909c3cf by qiuchaofei

1 关系添加属性:jobId,2 添加接口,给解析sql调用,更新局部关系。

parent cb037974
......@@ -556,6 +556,21 @@ public class MetaDataController {
logger.info("返回结果,后台继续运行。");
return flag;
}
@ApiOperation(tags = "", value = "根据环境id与jobId同步元数据关系的接口")
@RequestMapping(path = "/synchroRelationByJobId", method = RequestMethod.POST)
public String synchroRelationByJobId(String catalogName,String jobId) {
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
LogManager.logInfo(LogConstants.CTX_Relation, "用户 "+ getUserName() + " 在 "+ sdf.format(new Date())+"同步"+catalogName+" JobId:"+jobId+"的元数据。" );
logger.info("开始异步迁移元数据:从mongo到neo4j");
String flag = "success";
asyncDataFromMongoToNeo4j.asyncJobFromMongo2Neo4j(catalogName,jobId);
logger.info("返回结果,后台继续运行。");
return flag;
}
private String getUserName() {
Object obj = SecurityContextHolder.getContext().getAuthentication().getPrincipal();
if (obj instanceof String) {
......
......@@ -22,6 +22,8 @@ public class AsyncDataFromMongoToNeo4j {
private static final Logger logger = LoggerFactory.getLogger(AsyncDataFromMongoToNeo4j.class);
@Autowired
private IMetadataService metadataService;
@Autowired
private IMetadataRelationService metadataRelationService;
@Autowired
private JdbcTemplate jdbcTemplate;
......@@ -83,6 +85,12 @@ public class AsyncDataFromMongoToNeo4j {
logger.info("同步数据完成,用时:"+(end-start));
}
@Async
public void asyncJobFromMongo2Neo4j(String catalogName,String jobId) {
//先按照环境id,作业id,删除关系,然后同步关系,
// 一般来说,关系的字段已经存在neo4j,但是sql不一定存在,
// 如果sql不存在,需要先同步sql ,srcipt,job,再建立关系。
metadataRelationService.deleteRelationByJobId(catalogName,jobId);
metadataRelationService.syschroRelationFromMongo(catalogName,jobId);
}
}
......@@ -69,6 +69,8 @@ public interface IBaseRelationshipService {
void saveRelation(String startMetadataId, String endMetadataid, String composition);
void saveRelation(String startId, String endId, String relationType,String description);
void saveRelation(String startId, String endId, String relationType,String jobId);
void saveRelation(String startId, String endId, String relationType,String jobId,String description);
}
......@@ -8,4 +8,8 @@ public interface IMetadataRelationService {
ReturnReslult filterJosnByModelName(String reulstJson,String filterModel,String retainModel);
//删除关系
void deleteRelationByJobId(String catalogName, String jobId);
//同步关系,如果元数据不存在,要先同步元数据
void syschroRelationFromMongo(String catalogName, String jobId);
}
......@@ -183,7 +183,7 @@ public class BaseRelationshipServiceImpl implements IBaseRelationshipService {
}
@Override
public void saveRelation(String startId, String endId, String relationType,String description) {
public void saveRelation(String startId, String endId, String relationType,String jobId) {
Neo4jConfig neo4jConfig = new Neo4jConfig();
Driver neo4jConnection = neo4jConfig.getNeo4jConnection();
Session session = neo4jConnection.session();
......@@ -191,9 +191,26 @@ public class BaseRelationshipServiceImpl implements IBaseRelationshipService {
// (m{metadataId:"Table=1=d644b631fa8c434e928bcd1f1665b060"})
// create (n)<-[r:Composition]-(m)
// return n,m,r
String cypher = " match (n{metadataId:\""+startId+"\"}),(m{metadataId:\""+endId+"\"}) merge (n)-[r:"+relationType+"]->(m) set r.description ="+description;
String cypher = " match (n{metadataId:\""+startId+"\"}),(m{metadataId:\""+endId+"\"}) merge (n)-[r:"+relationType+"]->(m) set r.jobId ="+jobId;
// logger.info("运行保存关系完成:" + cypher);
session.run(cypher);
}
@Override
public void saveRelation(String startId, String endId, String relationType,String jobId,String description) {
Neo4jConfig neo4jConfig = new Neo4jConfig();
Driver neo4jConnection = neo4jConfig.getNeo4jConnection();
Session session = neo4jConnection.session();
// match (n{metadataId:"Column=1=d9c2d67e56a3428e8e4f22918782437f"}),
// (m{metadataId:"Table=1=d644b631fa8c434e928bcd1f1665b060"})
// create (n)<-[r:Composition]-(m)
// return n,m,r
String cypher = " match (n{metadataId:\""+startId+"\"}),(m{metadataId:\""+endId+"\"}) " +
"merge (n)-[r:"+relationType+"]->(m) set r.jobId ="+description + " and r.description = "+ description;
// logger.info("运行保存关系完成:" + cypher);
session.run(cypher);
}
}
......@@ -1852,13 +1852,11 @@ public class MetadataServiceImpl implements IMetadataService {
relationshipService.saveRelation(sourceTableId,targetTableId,"流向");
}
}
@Override
public void syschroTable2EtlJobRelations(String catalogName) {
Map<String, String> relationMap = new HashMap<>();
Map<String,String> edgeIdMap = new HashMap<>();
Map<String,Map<String, String>> edgeIdMap = new HashMap<>();
int page =0,pageSize = 300;
long time0 = System.currentTimeMillis();
long totalElement = mongoDbServiceImpl.countRelation(PageRequest.of(page,pageSize),catalogName);
......@@ -1869,13 +1867,17 @@ public class MetadataServiceImpl implements IMetadataService {
logger.info("准备分页从mongo获取关系,每页:"+ pageSize);
for(; page< totalPageCount;page++){
logger.info("开始从 mongo 第"+page+1 +" 次获取关系。");
String description = "";
long time3 = System.currentTimeMillis();
List<Document> relationList = mongoDbServiceImpl.findRelationByPage(PageRequest.of(page,pageSize,Sort.by("_id")),catalogName);
long time4 = System.currentTimeMillis();
logger.info("从 mongo 第"+page+1 +" 次获取关系结束,耗时:" +(time4-time3) );
for(Document relation :relationList){
description = relation.getString("description");
Map<String,String> attributeMap = new HashMap<>();
String jobId = relation.getString("jobId");
attributeMap.put("jobId",jobId);
String description = relation.getString("description");
attributeMap.put("description",description);
String type = relation.getString("type");
if(!(type.equals("Input") || type.equals("Output"))){
continue;
......@@ -1884,7 +1886,7 @@ public class MetadataServiceImpl implements IMetadataService {
String targetId =relation.getString("target"); // relationMongo.getTarget();
//本身的字段级关系也要同步
edgeIdMap.put(sourceId+"_"+targetId,description);
edgeIdMap.put(sourceId+"_"+targetId,attributeMap);
Map<String, Object> sourceData = metadataRepoRemoteService.getMetadata(sourceId);
Map<String, Object> targetData = metadataRepoRemoteService.getMetadata(targetId);
......@@ -1953,7 +1955,7 @@ public class MetadataServiceImpl implements IMetadataService {
endId = (String )targetParent.get("_id");
}
String relationId = startId+"_"+endId;
edgeIdMap.put(relationId,description);
edgeIdMap.put(relationId,attributeMap);
}
}
......@@ -1967,11 +1969,14 @@ public class MetadataServiceImpl implements IMetadataService {
String[] dataIds = edgeId.split("_");
String startId = dataIds[0];
String endId = dataIds[1];
String description = edgeIdMap.get(obj);
Map<String,String> attributeMap0 = edgeIdMap.get(obj);
String description = attributeMap0.get("description");
String jobId = attributeMap0.get("jobId");
// String description = edgeIdMap.get(obj);
if(description ==null || description.equalsIgnoreCase("")){
relationshipService.saveRelation(startId,endId,"流向");
relationshipService.saveRelation(startId,endId,"流向",jobId);
}else{
relationshipService.saveRelation(startId,endId,"流向",description);
relationshipService.saveRelation(startId,endId,"流向",jobId,description);
}
}
......
......@@ -165,6 +165,27 @@ public class MongoDbServiceImpl {
return mongoTemplate.count(query, Document.class, PREFIX_MD_RELATION + catalogName);
}
public long countRelationByJobId(Pageable pageable, String catalogName,String jobId) {
Query query = new Query(new Criteria() .andOperator(Criteria.where("jobId").is(jobId),
new Criteria().orOperator(Criteria.where("type").is("Input"),Criteria.where("type").is("Output"))
)
);
// Query query = new Query(new Criteria().orOperator(Criteria.where("type").is("Input"),Criteria.where("type").is("Output")));
query.with(pageable);
return mongoTemplate.count(query, Document.class, PREFIX_MD_RELATION + catalogName);
}
public List<Document> findRelationByJobIdByPage(Pageable pageable, String catalogName,String jobId) {
Query query = new Query(new Criteria() .andOperator(Criteria.where("jobId").is(jobId),
new Criteria().orOperator(Criteria.where("type").is("Input"),Criteria.where("type").is("Output"))
)
);
// Query query = new Query(new Criteria().orOperator(Criteria.where("type").is("Input"),Criteria.where("type").is("Output")));
query.with(pageable);
return mongoTemplate.find(query, Document.class, PREFIX_MD_RELATION + catalogName);
}
public List<Document> find1104Relations( String catalogName) {
return mongoTemplate.findAll(Document.class, "Relation" + catalogName);
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment