Skip to content

N
neo4jRelation
Commit f523b450 by qiuchaofei
Options

1.同步临时表，2 表与表的关系，需要通过临时表，3临时节点的repository添加事务，4 通过id查询的接口，从list改成单个。

parent 610cdd9f
Showing with 76 additions and 50 deletions
src/main/java/com/keymobile/metadata/metadataRelation/respository/TempNodeRepository.java
......@@ -3,15 +3,19 @@ package com.keymobile.metadata.metadataRelation.respository;
import java.util.List;
import com.keymobile.metadata.metadataRelation.pojo.TempNode;
import org.springframework.data.jpa.repository.Modifying;
import org.springframework.data.neo4j.annotation.Query;
import org.springframework.data.neo4j.repository.Neo4jRepository;
import org.springframework.data.repository.query.Param;
import javax.transaction.Transactional;
/**
* @author mahx
* @version 1.0
* @date 2021/11/10 18:15
*/
@Transactional
public interface TempNodeRepository extends Neo4jRepository<TempNode, Long> {
Long deleteTempNodesByEtlScriptId(String etlScriptId);
......
src/main/java/com/keymobile/metadata/metadataRelation/respository/metadata/Neo4jETLJobRepository.java
......@@ -10,6 +10,6 @@ import java.util.List;
@Repository
public interface Neo4jETLJobRepository extends Neo4jRepository<Neo4jETLJob, Long> {
List<Neo4jETLJob> findNeo4jETLJobByMetadataId(String metadataId);
Neo4jETLJob findNeo4jETLJobByMetadataId(String metadataId);
}
src/main/java/com/keymobile/metadata/metadataRelation/respository/metadata/Neo4jETLScriptRepository.java
......@@ -4,10 +4,7 @@ import com.keymobile.metadata.metadataRelation.pojo.metadata.Neo4jETLScript;
import org.springframework.data.neo4j.repository.Neo4jRepository;
import org.springframework.stereotype.Repository;
import java.util.List;
@Repository
public interface Neo4jETLScriptRepository extends Neo4jRepository<Neo4jETLScript, Long> {
List<Neo4jETLScript> findNeo4jETLScriptByMetadataId(String metadataId);
Neo4jETLScript findNeo4jETLScriptByMetadataId(String metadataId);
}
src/main/java/com/keymobile/metadata/metadataRelation/respository/metadata/Neo4jFunctionRepository.java
......@@ -9,5 +9,5 @@ import java.util.List;
@Repository
public interface Neo4jFunctionRepository extends Neo4jRepository<Neo4jFunction, Long> {
List<Neo4jFunction> findNeo4jFunctionByMetadataId(String metadataId);
Neo4jFunction findNeo4jFunctionByMetadataId(String metadataId);
}
src/main/java/com/keymobile/metadata/metadataRelation/respository/metadata/Neo4jProcedureRepository.java
......@@ -10,5 +10,5 @@ import java.util.List;
@Repository
public interface Neo4jProcedureRepository extends Neo4jRepository<Neo4jProcedure, Long> {
List<Neo4jProcedure> findNeo4jProcedureByMetadataId(String metadataId);
Neo4jProcedure findNeo4jProcedureByMetadataId(String metadataId);
}
src/main/java/com/keymobile/metadata/metadataRelation/respository/metadata/Neo4jSchemaRepository.java
......@@ -11,7 +11,7 @@ import java.util.List;
@Repository
public interface Neo4jSchemaRepository extends Neo4jRepository<Neo4jSchema, Long> {
List<Neo4jSchema> findNeo4jSchemaByMetadataId(String metadataId);
Neo4jSchema findNeo4jSchemaByMetadataId(String metadataId);
@Query("match (n:Neo4jSchema{metadataId:{schemaId}} )-[r:Composition]->(m) return m ")
List<BaseNode> getTableBySchemaId(@Param("schemaId") String schemaId);
......
src/main/java/com/keymobile/metadata/metadataRelation/respository/metadata/Neo4jSystemRepository.java
......@@ -11,7 +11,7 @@ import java.util.List;
@Repository
public interface Neo4jSystemRepository extends Neo4jRepository<Neo4jSystem, Long> {
List<Neo4jSystem> findNeo4jSystemByMetadataId(String metadataId);
Neo4jSystem findNeo4jSystemByMetadataId(String metadataId);
List<Neo4jSystem> findNeo4jSystemByName(String metadataName);
// @Query("MATCH p=(n:MetaData {metadataId:{parentId}})-[r:Composition]->(m) RETURN m limit {limit} ")
......
src/main/java/com/keymobile/metadata/metadataRelation/respository/metadata/Neo4jTableRepository.java
......@@ -11,12 +11,14 @@ import java.util.List;
@Repository
public interface Neo4jTableRepository extends Neo4jRepository<Neo4jTable,Long> {
List<Neo4jTable> findNeo4jTableByMetadataId(String metadataId);
Neo4jTable findNeo4jTableByMetadataId(String metadataId);
@Query("match (n{metadataId:{tableId}} )<-[r:流向]-(m1)<-[r1:流向]-(m) return m ")
//match (n:Neo4jTable{metadataId:"Table=1=de175ee0b21c473f915cefaad6d798f1"})-[r1:流向]->(n1)-[r0:TempInput*0..]->(m)-[r:流向]->(m1) return m1
@Query("match (n:Neo4jTable{metadataId:{tableId}})<-[r1:流向]-(n1)<-[r0:TempRelation*0..]-(m)<-[r:流向]-(m1) return m1 ")
List<BaseNode> getSourceTable(@Param("tableId") String tableId);
@Query("match (n{metadataId:{tableId}} )-[r1:流向]->(m1)-[r:流向]->(m) return m ")
// @Query("match (n{metadataId:{tableId}} )-[r1:流向]->(m1)-[r:流向]->(m) return m ")
@Query("match (n:Neo4jTable{metadataId:{tableId}})-[r1:流向]->(n1)-[r0:TempRelation*0..]->(m)-[r:流向]->(m1) return m1 ")
List<BaseNode> getTargetTable(@Param("tableId") String tableId);
@Query("match (n{metadataId:{startTable}} )<-[r1:流向]-(m1)<-[r:流向]-(m{metadataId:{entTable}}) return m1 ")
......
src/main/java/com/keymobile/metadata/metadataRelation/service/AsyncDataFromMongoToNeo4j.java
......@@ -63,13 +63,13 @@ public class AsyncDataFromMongoToNeo4j {
return user;
}
});
//先删除同个环境的数据
metadataService.deleteNodeByCatalogName(catalogName);
metadataService.syschroMetadataFromMongo(catalogName,mongoDataList);
//先同步元数据，纵向关系,root开始
//再同步元数据的横向关系
metadataService.syschroTable2EtlJobRelations(catalogName);
// //先删除同个环境的数据
// metadataService.deleteNodeByCatalogName(catalogName);
// //先同步元数据，纵向关系,root开始，系统--schema--table--column，系统--etl作业--etl脚本--etlsql。
// metadataService.syschroMetadataFromMongo(catalogName,mongoDataList);
//
// //再同步元数据的横向关系
// metadataService.syschroTable2EtlJobRelations(catalogName);
// metadataService.syschroRelationFromMongo(catalogName);
// // 同步其他数据（模型，标准，质量等）
......
src/main/java/com/keymobile/metadata/metadataRelation/service/impl/MetadataServiceImpl.java
......@@ -1293,8 +1293,8 @@ public class MetadataServiceImpl implements IMetadataService {
String systemPath = catalogName+";"+systemData.getName();
neo4jSystem.setDataPath(systemPath);
//用名称查找，如果存在，就不用创建
List<Neo4jSystem> neo4jSchemaList = neo4jSystemRepository.findNeo4jSystemByMetadataId(neo4jSystem.getMetadataId());
if(neo4jSchemaList==null || neo4jSchemaList.size()==0){
Neo4jSystem neo4jSchemaList = neo4jSystemRepository.findNeo4jSystemByMetadataId(neo4jSystem.getMetadataId());
if(neo4jSchemaList==null ){
neo4jSystemRepository.save(neo4jSystem);
}
systemDataMap.put(systemData.get_id(),neo4jSystem);
......@@ -1325,8 +1325,8 @@ public class MetadataServiceImpl implements IMetadataService {
neo4jETLJob.setName(etlJobData.getName());
neo4jETLJob.setCnName(etlJobData.getCnName());
//如果存在，就不用创建
List<Neo4jETLJob> neo4jETLJobList1 = neo4jETLJobRepository.findNeo4jETLJobByMetadataId(neo4jETLJob.getMetadataId());
if(neo4jETLJobList1==null || neo4jETLJobList1.size()==0){
Neo4jETLJob neo4jETLJobList1 = neo4jETLJobRepository.findNeo4jETLJobByMetadataId(neo4jETLJob.getMetadataId());
if(neo4jETLJobList1==null){
neo4jETLJobList.add(neo4jETLJob);
etlJobCount++;
}
......@@ -1345,8 +1345,8 @@ public class MetadataServiceImpl implements IMetadataService {
neo4jETLScript.setName(etlScriptMongo.getName());
neo4jETLScript.setCnName(etlScriptMongo.getCnName());
//如果存在，就不用创建
List<Neo4jETLScript> neo4jScriptList = neo4jETLScriptRepository.findNeo4jETLScriptByMetadataId(neo4jETLScript.getMetadataId());
if(neo4jScriptList==null || neo4jScriptList.size()==0){
Neo4jETLScript neo4jScriptList = neo4jETLScriptRepository.findNeo4jETLScriptByMetadataId(neo4jETLScript.getMetadataId());
if(neo4jScriptList==null ){
neo4jETLScriptList.add(neo4jETLScript);
etlScriptCount++;
CompositionRelation compositionRelation = new CompositionRelation();
......@@ -1360,7 +1360,7 @@ public class MetadataServiceImpl implements IMetadataService {
// for (MongoData eltSqlMongo:eltSqlMongoList){
//
// String etlSqlId = eltSqlMongo.get_id();
// Neo4jETLSql neo4jETLSql =null;// neo4jETLSqlRepository.findNeo4jETLSqlByMetadataId(etlSqlId);
// Neo4jETLSql neo4jETLSql = neo4jETLSqlRepository.findNeo4jETLSqlByMetadataId(etlSqlId);
// if(neo4jETLSql == null){
// neo4jETLSql = new Neo4jETLSql();
//
......@@ -1479,8 +1479,8 @@ public class MetadataServiceImpl implements IMetadataService {
neo4jTableList.clear();
}
List<Neo4jTable> neo4jTableListExist = null;//neo4jTableRepository.findNeo4jTableByMetadataId(metadataId);
if(neo4jTableListExist == null || neo4jTableListExist.size() == 0){
Neo4jTable neo4jTableListExist = neo4jTableRepository.findNeo4jTableByMetadataId(metadataId);
if(neo4jTableListExist == null ){
Neo4jTable neo4jTable = new Neo4jTable();
neo4jTable.setMetadataId(metadataId);
neo4jTable.setName(metadataName);
......@@ -1506,7 +1506,7 @@ public class MetadataServiceImpl implements IMetadataService {
// List<MongoData> columnMongoDataList = mongoDbServiceImpl.findDataByparentId(metadataId, catalogName);
// for(MongoData columnMongoData:columnMongoDataList){
// String columnId = columnMongoData.get_id();
// Neo4jColumn neo4jColumnList1 = null;//neo4jColumnRepository.findNeo4jColumnByMetadataId(columnId);
// Neo4jColumn neo4jColumnList1 = neo4jColumnRepository.findNeo4jColumnByMetadataId(columnId);
//
// if(neo4jColumnList1==null){
// Neo4jColumn neo4jColumn = new Neo4jColumn();
......@@ -1605,7 +1605,6 @@ public class MetadataServiceImpl implements IMetadataService {
}
}
}
int count=1;//获取前100个表测试
if(neo4jTableList!=null && neo4jTableList.size()!=0){
neo4jTableRepository.saveAll(neo4jTableList);
}
......@@ -2265,31 +2264,59 @@ public class MetadataServiceImpl implements IMetadataService {
String etlScriptId = relation.getString("etlScriptId");
String etlJobId = relation.getString("etlJobId");
deleteTempNodeByCatalogName(sqlId);
MetaData metadata = metadataRepository.findMetaDataByMetadataId(sqlId);
if (metadata == null) {
Neo4jETLSql neo4jETLSql = neo4jETLSqlRepository.findNeo4jETLSqlByMetadataId(sqlId);
// MetaData metadata = metadataRepository.findMetaDataByMetadataId(sqlId);
Neo4jETLScript neo4jETLScript = neo4jETLScriptRepository.findNeo4jETLScriptByMetadataId(etlScriptId);
Long srciptId = neo4jETLScript.getId();
String source = relation.getString("source");
String target = relation.getString("target");
String type = relation.getString("type");
if ("Input".equalsIgnoreCase(type)) {
//上面方法得到的临时字段的id，怎么得到临时表的id？
String[] tableNames = StringUtils.split(source, '.');
String tableName = tableNames[1];
Long tableId = createNode(nameIdMap, catalogName, etlJobId, etlScriptId, tableName, true, null);
if (tableId != null) {
tempRelationRepository.mergeTempRelation(tableId, srciptId);
}
}
if ("Output".equalsIgnoreCase(type)) {
//上面方法得到的临时字段的id，怎么得到临时表的id？
String[] tableNames = StringUtils.split(target, '.');
String tableName = tableNames[1];
Long tableId = createNode(nameIdMap, catalogName, etlJobId, etlScriptId, tableName, true, null);
if (tableId != null) {
tempRelationRepository.mergeTempRelation(srciptId,tableId);
}
}
if (neo4jETLSql == null) {
logger.error("id : {} is not found", sqlId);
continue;
}
Long id = metadata.getId();
Long id = neo4jETLSql.getId();
String source = relation.getString("source");
String target = relation.getString("target");
if (StringUtils.isBlank(source) || StringUtils.isBlank(target)) {
continue;
}
String type = relation.getString("type");
if ("Input".equalsIgnoreCase(type)) {
Long nodeId = createNode(nameIdMap, catalogName, etlJobId, etlScriptId, source, false, null);
if (nodeId != null) {
tempRelationRepository.mergeTempRelation(nodeId, id);
}
}
if ("Output".equalsIgnoreCase(type)) {
Long nodeId = createNode(nameIdMap, catalogName, etlJobId, etlScriptId, target, false, null);
if (nodeId != null) {
tempRelationRepository.mergeTempRelation(id, nodeId);
}
}
}
}
......@@ -2314,7 +2341,7 @@ public class MetadataServiceImpl implements IMetadataService {
public void deleteTempNodeByCatalogName(String catalogName) {
try{
tempNodeRepository.deleteAllRelation(catalogName);
tempNodeRepository.deleteTempNodesByCatalogName(catalogName);
tempNodeRepository.deleteTempNodesByCatalogName(catalogName+"");
}catch(Exception e){
e.printStackTrace();
......
src/main/java/com/keymobile/metadata/metadataRelation/service/impl/TableServiceImpl.java
......@@ -126,20 +126,17 @@ public class TableServiceImpl implements ITableService {
}
private List<ReturnNode> getCurrentTableInfo(String tableId) {
List<Neo4jTable> neo4jTableList = neo4jTableRepository.findNeo4jTableByMetadataId(tableId);
Neo4jTable neo4jTable = neo4jTableRepository.findNeo4jTableByMetadataId(tableId);
List<ReturnNode> returnTableList = new ArrayList<>();
for(Neo4jTable neo4jTable:neo4jTableList){
ReturnNode returnNode = new ReturnNode();
returnNode.setId(neo4jTable.getMetadataId());
returnNode.setName(neo4jTable.getName());
returnNode.setType("Table");
returnTableList.add(returnNode);
ReturnNode returnNode = new ReturnNode();
returnNode.setId(neo4jTable.getMetadataId());
returnNode.setName(neo4jTable.getName());
returnNode.setType("Table");
returnTableList.add(returnNode);
// 获取属性
Map<String,String> attributeMap = getAttributeMap(tableId);
Map<String,String> attributeMap = getAttributeMap(tableId);
returnNode.setAttributeMaps(attributeMap);
}
returnNode.setAttributeMaps(attributeMap);
return returnTableList;
}
......@@ -263,8 +260,7 @@ public class TableServiceImpl implements ITableService {
Map<String,String> returnMap = new LinkedHashMap<>();
List<Neo4jTable> neo4jTableList = neo4jTableRepository.findNeo4jTableByMetadataId(tableId);
Neo4jTable neo4jTable = neo4jTableList.get(0);
Neo4jTable neo4jTable = neo4jTableRepository.findNeo4jTableByMetadataId(tableId);
// 英文名称，中文名称，说明，表规模（行），表大小（字节），更新时间，
// 所属系统，所属分层，加载脚本，质量情况，相关模型
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment