Commit f523b450 by qiuchaofei

1.同步临时表,2 表与表的关系,需要通过临时表,3临时节点的repository添加事务,4 通过id查询的接口,从list改成单个。

parent 610cdd9f
...@@ -3,15 +3,19 @@ package com.keymobile.metadata.metadataRelation.respository; ...@@ -3,15 +3,19 @@ package com.keymobile.metadata.metadataRelation.respository;
import java.util.List; import java.util.List;
import com.keymobile.metadata.metadataRelation.pojo.TempNode; import com.keymobile.metadata.metadataRelation.pojo.TempNode;
import org.springframework.data.jpa.repository.Modifying;
import org.springframework.data.neo4j.annotation.Query; import org.springframework.data.neo4j.annotation.Query;
import org.springframework.data.neo4j.repository.Neo4jRepository; import org.springframework.data.neo4j.repository.Neo4jRepository;
import org.springframework.data.repository.query.Param; import org.springframework.data.repository.query.Param;
import javax.transaction.Transactional;
/** /**
* @author mahx * @author mahx
* @version 1.0 * @version 1.0
* @date 2021/11/10 18:15 * @date 2021/11/10 18:15
*/ */
@Transactional
public interface TempNodeRepository extends Neo4jRepository<TempNode, Long> { public interface TempNodeRepository extends Neo4jRepository<TempNode, Long> {
Long deleteTempNodesByEtlScriptId(String etlScriptId); Long deleteTempNodesByEtlScriptId(String etlScriptId);
......
...@@ -10,6 +10,6 @@ import java.util.List; ...@@ -10,6 +10,6 @@ import java.util.List;
@Repository @Repository
public interface Neo4jETLJobRepository extends Neo4jRepository<Neo4jETLJob, Long> { public interface Neo4jETLJobRepository extends Neo4jRepository<Neo4jETLJob, Long> {
List<Neo4jETLJob> findNeo4jETLJobByMetadataId(String metadataId); Neo4jETLJob findNeo4jETLJobByMetadataId(String metadataId);
} }
...@@ -4,10 +4,7 @@ import com.keymobile.metadata.metadataRelation.pojo.metadata.Neo4jETLScript; ...@@ -4,10 +4,7 @@ import com.keymobile.metadata.metadataRelation.pojo.metadata.Neo4jETLScript;
import org.springframework.data.neo4j.repository.Neo4jRepository; import org.springframework.data.neo4j.repository.Neo4jRepository;
import org.springframework.stereotype.Repository; import org.springframework.stereotype.Repository;
import java.util.List;
@Repository @Repository
public interface Neo4jETLScriptRepository extends Neo4jRepository<Neo4jETLScript, Long> { public interface Neo4jETLScriptRepository extends Neo4jRepository<Neo4jETLScript, Long> {
Neo4jETLScript findNeo4jETLScriptByMetadataId(String metadataId);
List<Neo4jETLScript> findNeo4jETLScriptByMetadataId(String metadataId);
} }
...@@ -9,5 +9,5 @@ import java.util.List; ...@@ -9,5 +9,5 @@ import java.util.List;
@Repository @Repository
public interface Neo4jFunctionRepository extends Neo4jRepository<Neo4jFunction, Long> { public interface Neo4jFunctionRepository extends Neo4jRepository<Neo4jFunction, Long> {
List<Neo4jFunction> findNeo4jFunctionByMetadataId(String metadataId); Neo4jFunction findNeo4jFunctionByMetadataId(String metadataId);
} }
...@@ -10,5 +10,5 @@ import java.util.List; ...@@ -10,5 +10,5 @@ import java.util.List;
@Repository @Repository
public interface Neo4jProcedureRepository extends Neo4jRepository<Neo4jProcedure, Long> { public interface Neo4jProcedureRepository extends Neo4jRepository<Neo4jProcedure, Long> {
List<Neo4jProcedure> findNeo4jProcedureByMetadataId(String metadataId); Neo4jProcedure findNeo4jProcedureByMetadataId(String metadataId);
} }
...@@ -11,7 +11,7 @@ import java.util.List; ...@@ -11,7 +11,7 @@ import java.util.List;
@Repository @Repository
public interface Neo4jSchemaRepository extends Neo4jRepository<Neo4jSchema, Long> { public interface Neo4jSchemaRepository extends Neo4jRepository<Neo4jSchema, Long> {
List<Neo4jSchema> findNeo4jSchemaByMetadataId(String metadataId); Neo4jSchema findNeo4jSchemaByMetadataId(String metadataId);
@Query("match (n:Neo4jSchema{metadataId:{schemaId}} )-[r:Composition]->(m) return m ") @Query("match (n:Neo4jSchema{metadataId:{schemaId}} )-[r:Composition]->(m) return m ")
List<BaseNode> getTableBySchemaId(@Param("schemaId") String schemaId); List<BaseNode> getTableBySchemaId(@Param("schemaId") String schemaId);
......
...@@ -11,7 +11,7 @@ import java.util.List; ...@@ -11,7 +11,7 @@ import java.util.List;
@Repository @Repository
public interface Neo4jSystemRepository extends Neo4jRepository<Neo4jSystem, Long> { public interface Neo4jSystemRepository extends Neo4jRepository<Neo4jSystem, Long> {
List<Neo4jSystem> findNeo4jSystemByMetadataId(String metadataId); Neo4jSystem findNeo4jSystemByMetadataId(String metadataId);
List<Neo4jSystem> findNeo4jSystemByName(String metadataName); List<Neo4jSystem> findNeo4jSystemByName(String metadataName);
// @Query("MATCH p=(n:MetaData {metadataId:{parentId}})-[r:Composition]->(m) RETURN m limit {limit} ") // @Query("MATCH p=(n:MetaData {metadataId:{parentId}})-[r:Composition]->(m) RETURN m limit {limit} ")
......
...@@ -11,12 +11,14 @@ import java.util.List; ...@@ -11,12 +11,14 @@ import java.util.List;
@Repository @Repository
public interface Neo4jTableRepository extends Neo4jRepository<Neo4jTable,Long> { public interface Neo4jTableRepository extends Neo4jRepository<Neo4jTable,Long> {
List<Neo4jTable> findNeo4jTableByMetadataId(String metadataId); Neo4jTable findNeo4jTableByMetadataId(String metadataId);
@Query("match (n{metadataId:{tableId}} )<-[r:流向]-(m1)<-[r1:流向]-(m) return m ") //match (n:Neo4jTable{metadataId:"Table=1=de175ee0b21c473f915cefaad6d798f1"})-[r1:流向]->(n1)-[r0:TempInput*0..]->(m)-[r:流向]->(m1) return m1
@Query("match (n:Neo4jTable{metadataId:{tableId}})<-[r1:流向]-(n1)<-[r0:TempRelation*0..]-(m)<-[r:流向]-(m1) return m1 ")
List<BaseNode> getSourceTable(@Param("tableId") String tableId); List<BaseNode> getSourceTable(@Param("tableId") String tableId);
@Query("match (n{metadataId:{tableId}} )-[r1:流向]->(m1)-[r:流向]->(m) return m ") // @Query("match (n{metadataId:{tableId}} )-[r1:流向]->(m1)-[r:流向]->(m) return m ")
@Query("match (n:Neo4jTable{metadataId:{tableId}})-[r1:流向]->(n1)-[r0:TempRelation*0..]->(m)-[r:流向]->(m1) return m1 ")
List<BaseNode> getTargetTable(@Param("tableId") String tableId); List<BaseNode> getTargetTable(@Param("tableId") String tableId);
@Query("match (n{metadataId:{startTable}} )<-[r1:流向]-(m1)<-[r:流向]-(m{metadataId:{entTable}}) return m1 ") @Query("match (n{metadataId:{startTable}} )<-[r1:流向]-(m1)<-[r:流向]-(m{metadataId:{entTable}}) return m1 ")
......
...@@ -63,13 +63,13 @@ public class AsyncDataFromMongoToNeo4j { ...@@ -63,13 +63,13 @@ public class AsyncDataFromMongoToNeo4j {
return user; return user;
} }
}); });
//先删除同个环境的数据 // //先删除同个环境的数据
metadataService.deleteNodeByCatalogName(catalogName); // metadataService.deleteNodeByCatalogName(catalogName);
metadataService.syschroMetadataFromMongo(catalogName,mongoDataList); // //先同步元数据,纵向关系,root开始,系统--schema--table--column,系统--etl作业--etl脚本--etlsql。
// metadataService.syschroMetadataFromMongo(catalogName,mongoDataList);
//先同步元数据,纵向关系,root开始 //
//再同步元数据的横向关系 // //再同步元数据的横向关系
metadataService.syschroTable2EtlJobRelations(catalogName); // metadataService.syschroTable2EtlJobRelations(catalogName);
// metadataService.syschroRelationFromMongo(catalogName); // metadataService.syschroRelationFromMongo(catalogName);
// // 同步其他数据(模型,标准,质量等) // // 同步其他数据(模型,标准,质量等)
......
...@@ -1293,8 +1293,8 @@ public class MetadataServiceImpl implements IMetadataService { ...@@ -1293,8 +1293,8 @@ public class MetadataServiceImpl implements IMetadataService {
String systemPath = catalogName+";"+systemData.getName(); String systemPath = catalogName+";"+systemData.getName();
neo4jSystem.setDataPath(systemPath); neo4jSystem.setDataPath(systemPath);
//用名称查找,如果存在,就不用创建 //用名称查找,如果存在,就不用创建
List<Neo4jSystem> neo4jSchemaList = neo4jSystemRepository.findNeo4jSystemByMetadataId(neo4jSystem.getMetadataId()); Neo4jSystem neo4jSchemaList = neo4jSystemRepository.findNeo4jSystemByMetadataId(neo4jSystem.getMetadataId());
if(neo4jSchemaList==null || neo4jSchemaList.size()==0){ if(neo4jSchemaList==null ){
neo4jSystemRepository.save(neo4jSystem); neo4jSystemRepository.save(neo4jSystem);
} }
systemDataMap.put(systemData.get_id(),neo4jSystem); systemDataMap.put(systemData.get_id(),neo4jSystem);
...@@ -1325,8 +1325,8 @@ public class MetadataServiceImpl implements IMetadataService { ...@@ -1325,8 +1325,8 @@ public class MetadataServiceImpl implements IMetadataService {
neo4jETLJob.setName(etlJobData.getName()); neo4jETLJob.setName(etlJobData.getName());
neo4jETLJob.setCnName(etlJobData.getCnName()); neo4jETLJob.setCnName(etlJobData.getCnName());
//如果存在,就不用创建 //如果存在,就不用创建
List<Neo4jETLJob> neo4jETLJobList1 = neo4jETLJobRepository.findNeo4jETLJobByMetadataId(neo4jETLJob.getMetadataId()); Neo4jETLJob neo4jETLJobList1 = neo4jETLJobRepository.findNeo4jETLJobByMetadataId(neo4jETLJob.getMetadataId());
if(neo4jETLJobList1==null || neo4jETLJobList1.size()==0){ if(neo4jETLJobList1==null){
neo4jETLJobList.add(neo4jETLJob); neo4jETLJobList.add(neo4jETLJob);
etlJobCount++; etlJobCount++;
} }
...@@ -1345,8 +1345,8 @@ public class MetadataServiceImpl implements IMetadataService { ...@@ -1345,8 +1345,8 @@ public class MetadataServiceImpl implements IMetadataService {
neo4jETLScript.setName(etlScriptMongo.getName()); neo4jETLScript.setName(etlScriptMongo.getName());
neo4jETLScript.setCnName(etlScriptMongo.getCnName()); neo4jETLScript.setCnName(etlScriptMongo.getCnName());
//如果存在,就不用创建 //如果存在,就不用创建
List<Neo4jETLScript> neo4jScriptList = neo4jETLScriptRepository.findNeo4jETLScriptByMetadataId(neo4jETLScript.getMetadataId()); Neo4jETLScript neo4jScriptList = neo4jETLScriptRepository.findNeo4jETLScriptByMetadataId(neo4jETLScript.getMetadataId());
if(neo4jScriptList==null || neo4jScriptList.size()==0){ if(neo4jScriptList==null ){
neo4jETLScriptList.add(neo4jETLScript); neo4jETLScriptList.add(neo4jETLScript);
etlScriptCount++; etlScriptCount++;
CompositionRelation compositionRelation = new CompositionRelation(); CompositionRelation compositionRelation = new CompositionRelation();
...@@ -1360,7 +1360,7 @@ public class MetadataServiceImpl implements IMetadataService { ...@@ -1360,7 +1360,7 @@ public class MetadataServiceImpl implements IMetadataService {
// for (MongoData eltSqlMongo:eltSqlMongoList){ // for (MongoData eltSqlMongo:eltSqlMongoList){
// //
// String etlSqlId = eltSqlMongo.get_id(); // String etlSqlId = eltSqlMongo.get_id();
// Neo4jETLSql neo4jETLSql =null;// neo4jETLSqlRepository.findNeo4jETLSqlByMetadataId(etlSqlId); // Neo4jETLSql neo4jETLSql = neo4jETLSqlRepository.findNeo4jETLSqlByMetadataId(etlSqlId);
// if(neo4jETLSql == null){ // if(neo4jETLSql == null){
// neo4jETLSql = new Neo4jETLSql(); // neo4jETLSql = new Neo4jETLSql();
// //
...@@ -1479,8 +1479,8 @@ public class MetadataServiceImpl implements IMetadataService { ...@@ -1479,8 +1479,8 @@ public class MetadataServiceImpl implements IMetadataService {
neo4jTableList.clear(); neo4jTableList.clear();
} }
List<Neo4jTable> neo4jTableListExist = null;//neo4jTableRepository.findNeo4jTableByMetadataId(metadataId); Neo4jTable neo4jTableListExist = neo4jTableRepository.findNeo4jTableByMetadataId(metadataId);
if(neo4jTableListExist == null || neo4jTableListExist.size() == 0){ if(neo4jTableListExist == null ){
Neo4jTable neo4jTable = new Neo4jTable(); Neo4jTable neo4jTable = new Neo4jTable();
neo4jTable.setMetadataId(metadataId); neo4jTable.setMetadataId(metadataId);
neo4jTable.setName(metadataName); neo4jTable.setName(metadataName);
...@@ -1506,7 +1506,7 @@ public class MetadataServiceImpl implements IMetadataService { ...@@ -1506,7 +1506,7 @@ public class MetadataServiceImpl implements IMetadataService {
// List<MongoData> columnMongoDataList = mongoDbServiceImpl.findDataByparentId(metadataId, catalogName); // List<MongoData> columnMongoDataList = mongoDbServiceImpl.findDataByparentId(metadataId, catalogName);
// for(MongoData columnMongoData:columnMongoDataList){ // for(MongoData columnMongoData:columnMongoDataList){
// String columnId = columnMongoData.get_id(); // String columnId = columnMongoData.get_id();
// Neo4jColumn neo4jColumnList1 = null;//neo4jColumnRepository.findNeo4jColumnByMetadataId(columnId); // Neo4jColumn neo4jColumnList1 = neo4jColumnRepository.findNeo4jColumnByMetadataId(columnId);
// //
// if(neo4jColumnList1==null){ // if(neo4jColumnList1==null){
// Neo4jColumn neo4jColumn = new Neo4jColumn(); // Neo4jColumn neo4jColumn = new Neo4jColumn();
...@@ -1605,7 +1605,6 @@ public class MetadataServiceImpl implements IMetadataService { ...@@ -1605,7 +1605,6 @@ public class MetadataServiceImpl implements IMetadataService {
} }
} }
} }
int count=1;//获取前100个表测试
if(neo4jTableList!=null && neo4jTableList.size()!=0){ if(neo4jTableList!=null && neo4jTableList.size()!=0){
neo4jTableRepository.saveAll(neo4jTableList); neo4jTableRepository.saveAll(neo4jTableList);
} }
...@@ -2265,31 +2264,59 @@ public class MetadataServiceImpl implements IMetadataService { ...@@ -2265,31 +2264,59 @@ public class MetadataServiceImpl implements IMetadataService {
String etlScriptId = relation.getString("etlScriptId"); String etlScriptId = relation.getString("etlScriptId");
String etlJobId = relation.getString("etlJobId"); String etlJobId = relation.getString("etlJobId");
deleteTempNodeByCatalogName(sqlId); deleteTempNodeByCatalogName(sqlId);
MetaData metadata = metadataRepository.findMetaDataByMetadataId(sqlId); Neo4jETLSql neo4jETLSql = neo4jETLSqlRepository.findNeo4jETLSqlByMetadataId(sqlId);
if (metadata == null) { // MetaData metadata = metadataRepository.findMetaDataByMetadataId(sqlId);
Neo4jETLScript neo4jETLScript = neo4jETLScriptRepository.findNeo4jETLScriptByMetadataId(etlScriptId);
Long srciptId = neo4jETLScript.getId();
String source = relation.getString("source");
String target = relation.getString("target");
String type = relation.getString("type");
if ("Input".equalsIgnoreCase(type)) {
//上面方法得到的临时字段的id,怎么得到临时表的id?
String[] tableNames = StringUtils.split(source, '.');
String tableName = tableNames[1];
Long tableId = createNode(nameIdMap, catalogName, etlJobId, etlScriptId, tableName, true, null);
if (tableId != null) {
tempRelationRepository.mergeTempRelation(tableId, srciptId);
}
}
if ("Output".equalsIgnoreCase(type)) {
//上面方法得到的临时字段的id,怎么得到临时表的id?
String[] tableNames = StringUtils.split(target, '.');
String tableName = tableNames[1];
Long tableId = createNode(nameIdMap, catalogName, etlJobId, etlScriptId, tableName, true, null);
if (tableId != null) {
tempRelationRepository.mergeTempRelation(srciptId,tableId);
}
}
if (neo4jETLSql == null) {
logger.error("id : {} is not found", sqlId); logger.error("id : {} is not found", sqlId);
continue; continue;
} }
Long id = metadata.getId(); Long id = neo4jETLSql.getId();
String source = relation.getString("source");
String target = relation.getString("target");
if (StringUtils.isBlank(source) || StringUtils.isBlank(target)) { if (StringUtils.isBlank(source) || StringUtils.isBlank(target)) {
continue; continue;
} }
String type = relation.getString("type");
if ("Input".equalsIgnoreCase(type)) { if ("Input".equalsIgnoreCase(type)) {
Long nodeId = createNode(nameIdMap, catalogName, etlJobId, etlScriptId, source, false, null); Long nodeId = createNode(nameIdMap, catalogName, etlJobId, etlScriptId, source, false, null);
if (nodeId != null) { if (nodeId != null) {
tempRelationRepository.mergeTempRelation(nodeId, id); tempRelationRepository.mergeTempRelation(nodeId, id);
} }
} }
if ("Output".equalsIgnoreCase(type)) { if ("Output".equalsIgnoreCase(type)) {
Long nodeId = createNode(nameIdMap, catalogName, etlJobId, etlScriptId, target, false, null); Long nodeId = createNode(nameIdMap, catalogName, etlJobId, etlScriptId, target, false, null);
if (nodeId != null) { if (nodeId != null) {
tempRelationRepository.mergeTempRelation(id, nodeId); tempRelationRepository.mergeTempRelation(id, nodeId);
} }
} }
} }
} }
...@@ -2314,7 +2341,7 @@ public class MetadataServiceImpl implements IMetadataService { ...@@ -2314,7 +2341,7 @@ public class MetadataServiceImpl implements IMetadataService {
public void deleteTempNodeByCatalogName(String catalogName) { public void deleteTempNodeByCatalogName(String catalogName) {
try{ try{
tempNodeRepository.deleteAllRelation(catalogName); tempNodeRepository.deleteAllRelation(catalogName);
tempNodeRepository.deleteTempNodesByCatalogName(catalogName); tempNodeRepository.deleteTempNodesByCatalogName(catalogName+"");
}catch(Exception e){ }catch(Exception e){
e.printStackTrace(); e.printStackTrace();
......
...@@ -126,20 +126,17 @@ public class TableServiceImpl implements ITableService { ...@@ -126,20 +126,17 @@ public class TableServiceImpl implements ITableService {
} }
private List<ReturnNode> getCurrentTableInfo(String tableId) { private List<ReturnNode> getCurrentTableInfo(String tableId) {
List<Neo4jTable> neo4jTableList = neo4jTableRepository.findNeo4jTableByMetadataId(tableId); Neo4jTable neo4jTable = neo4jTableRepository.findNeo4jTableByMetadataId(tableId);
List<ReturnNode> returnTableList = new ArrayList<>(); List<ReturnNode> returnTableList = new ArrayList<>();
ReturnNode returnNode = new ReturnNode();
for(Neo4jTable neo4jTable:neo4jTableList){ returnNode.setId(neo4jTable.getMetadataId());
ReturnNode returnNode = new ReturnNode(); returnNode.setName(neo4jTable.getName());
returnNode.setId(neo4jTable.getMetadataId()); returnNode.setType("Table");
returnNode.setName(neo4jTable.getName()); returnTableList.add(returnNode);
returnNode.setType("Table");
returnTableList.add(returnNode);
// 获取属性 // 获取属性
Map<String,String> attributeMap = getAttributeMap(tableId); Map<String,String> attributeMap = getAttributeMap(tableId);
returnNode.setAttributeMaps(attributeMap); returnNode.setAttributeMaps(attributeMap);
}
return returnTableList; return returnTableList;
} }
...@@ -263,8 +260,7 @@ public class TableServiceImpl implements ITableService { ...@@ -263,8 +260,7 @@ public class TableServiceImpl implements ITableService {
Map<String,String> returnMap = new LinkedHashMap<>(); Map<String,String> returnMap = new LinkedHashMap<>();
List<Neo4jTable> neo4jTableList = neo4jTableRepository.findNeo4jTableByMetadataId(tableId); Neo4jTable neo4jTable = neo4jTableRepository.findNeo4jTableByMetadataId(tableId);
Neo4jTable neo4jTable = neo4jTableList.get(0);
// 英文名称,中文名称,说明,表规模(行),表大小(字节),更新时间, // 英文名称,中文名称,说明,表规模(行),表大小(字节),更新时间,
// 所属系统,所属分层,加载脚本,质量情况,相关模型 // 所属系统,所属分层,加载脚本,质量情况,相关模型
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment