Skip to content

N
neo4jRelation
Commit 744d3d59 by qiuchaofei
Options

1.修改schema的标签名称，2.添加1104报表的关系。3.同步schema的标签时添加日志。

parent bfc204a6
Showing with 188 additions and 62 deletions
src/main/java/com/keymobile/metadata/metadataRelation/controller/RelationalGraphController.java
......@@ -306,6 +306,7 @@ public class RelationalGraphController {
@ApiOperation(tags = "", value = "更新所有schema的标签")
@RequestMapping(path = "/updateAllTagFromSchema", method = RequestMethod.GET)
public boolean updateTagFromAllSchema(){
logger.info("开始同步所有schema的标签。");
return schemaService.updateTagFromAllSchema();
}
......
src/main/java/com/keymobile/metadata/metadataRelation/respository/metadata/Neo4jTableRepository.java
......@@ -15,6 +15,8 @@ import java.util.List;
public interface Neo4jTableRepository extends Neo4jRepository<Neo4jTable,Long> {
Neo4jTable findNeo4jTableByMetadataId(String metadataId);
List<Neo4jTable> findNeo4jTableByNameAndIsSystemAndIsEnvironment(String name,String systemName,String isEnvironment);
//match (n:Neo4jTable{metadataId:"Table=1=de175ee0b21c473f915cefaad6d798f1"})-[r1:流向]->(n1)-[r0:TempInput*0..]->(m)-[r:流向]->(m1) return m1
@Query("match (n:Neo4jTable{metadataId:{tableId}})<-[r1:流向]-(n1)<-[r0:TempRelation*0..]-(m)<-[r:流向]-(m1) return m1 ")
List<BaseNode> getSourceTable(@Param("tableId") String tableId);
......
src/main/java/com/keymobile/metadata/metadataRelation/service/AsyncDataFromMongoToNeo4j.java
......@@ -71,6 +71,8 @@ public class AsyncDataFromMongoToNeo4j {
//再同步元数据的横向关系
metadataService.syschroTable2EtlJobRelations(catalogName);
//同步1104报表
metadataService.syschro1104Relations(catalogName);
// metadataService.syschroRelationFromMongo(catalogName);
// // 同步其他数据（模型，标准，质量等）
//
......
src/main/java/com/keymobile/metadata/metadataRelation/service/IMetadataService.java
......@@ -107,4 +107,5 @@ public interface IMetadataService {
void deleteNodeByCatalogName(String catalogName);
void syschro1104Relations(String catalogName);
}
src/main/java/com/keymobile/metadata/metadataRelation/service/impl/MetadataServiceImpl.java
......@@ -23,6 +23,7 @@ import com.keymobile.metadata.metadataRelation.service.IMetadataService;
import com.keymobile.metadata.metadataRelation.service.ISchemaService;
import com.keymobile.metadata.metadataRelation.service.ITableService;
import com.keymobile.metadata.metadataRelation.util.Neo4jTool;
import com.keymobile.metadata.metadataRelation.util.SchemaLayerObject;
import org.apache.commons.lang.StringUtils;
import org.bson.Document;
import org.neo4j.driver.v1.*;
......@@ -1324,13 +1325,13 @@ public class MetadataServiceImpl implements IMetadataService {
int procedureCount = 1;
List<String> layer = new ArrayList<>();
layer.add("贴源数据层");
layer.add("基础数据层");
layer.add("整合数据层");
layer.add("统计数据层");
layer.add("分析数据层");
layer.add("集市数据层");
layer.add("应用数据层");
layer.add(SchemaLayerObject.tieyuanceng);
layer.add(SchemaLayerObject.jichuceng);
layer.add(SchemaLayerObject.zhengheceng);
layer.add(SchemaLayerObject.tongjiceng);
layer.add(SchemaLayerObject.fenxiceng);
layer.add(SchemaLayerObject.jishiceng);
layer.add(SchemaLayerObject.yingyongceng);
Random r = new Random(1);
......@@ -1736,6 +1737,116 @@ public class MetadataServiceImpl implements IMetadataService {
long time2 = System.currentTimeMillis();
logger.info("用时："+(time2-time1));
}
@Override
public void syschro1104Relations(String catalogName) {
List<Document> relationList = mongoDbServiceImpl.find1104Relations("1104") ;
for(Document relation :relationList){
String system = relation.getString("system");
String sourceSysTable = relation.getString("sourceSysTable");
String sourceTable = relation.getString("sourceTable");
String sourceTableCnName = relation.getString("sourceTableCnName");
String sourceColumn = relation.getString("sourceColumn");
String sourceColumnDes = relation.getString("sourceColumnDes");
String targetTable = relation.getString("targetTable");
String targetTableCnName = relation.getString("targetTableCnName");
String targetColumn = relation.getString("targetColumn");
String targetColumnDes = relation.getString("targetColumnDes");
// system 不为空的，就是物理表，用 system，sourceSysTable 查找对应的neo4jTable，
// system为空的，先用 id为1104=sourceSysTable查找，如果找到就创建关系，
// 如果找不到，需要创建新节点，再建立关系
// targetTable， 先用 id为1104=targetTable查找，如果找到就创建关系，
// 如果找不到，需要创建新节点，再建立关系，
String sourceTableId = "";
if(system!= null ){
List<Neo4jTable> neo4jTableList = neo4jTableRepository.findNeo4jTableByNameAndIsSystemAndIsEnvironment(sourceSysTable, system,catalogName);
if(neo4jTableList!=null &&neo4jTableList.size()!=0){
sourceTableId = neo4jTableList.get(0).getMetadataId();
}
}else {
sourceTableId = "Table=1104="+sourceTable;
}
Neo4jTable neo4jTable = neo4jTableRepository.findNeo4jTableByMetadataId(sourceTableId);
if(neo4jTable == null){
neo4jTable = new Neo4jTable();
neo4jTable.setMetadataId(sourceTableId);
neo4jTable.setIsSystem("1104");
neo4jTable.setName(sourceTable);
neo4jTable.setCnName(sourceTableCnName);
neo4jTable = neo4jTableRepository.save(neo4jTable);
}
String sourceColumnId = "";
List<Neo4jColumn> neo4jColumnList = neo4jTableRepository.getColumnByTableId(sourceTableId);
for(Neo4jColumn neo4jColumn:neo4jColumnList){
if(neo4jColumn.getName().equalsIgnoreCase(sourceColumn)){
sourceColumnId = neo4jColumn.getMetadataId();
break;
}
}
if(sourceColumnId == null || sourceColumnId.equalsIgnoreCase("")){
sourceColumnId = "Column=1104="+sourceTable+"="+sourceColumn;
Neo4jColumn souceNeo4jColumn = neo4jColumnRepository.findNeo4jColumnByMetadataId(sourceColumnId);
if(souceNeo4jColumn == null){
souceNeo4jColumn = new Neo4jColumn();
souceNeo4jColumn.setMetadataId(sourceColumnId);
souceNeo4jColumn.setName(sourceColumn);
souceNeo4jColumn.setCnName(sourceColumnDes);
souceNeo4jColumn = neo4jColumnRepository.save(souceNeo4jColumn);
}
CompositionRelation table2Column = new CompositionRelation();
table2Column.setStart(neo4jTable);
table2Column.setEnd(souceNeo4jColumn);
table2Column.setName("table--column");
compositionRelationRespository.save(table2Column);
}
String targetTableId = "Table=1104="+targetTable;
Neo4jTable targetNeo4jTable = neo4jTableRepository.findNeo4jTableByMetadataId(targetTableId);
if(targetNeo4jTable == null ){
targetNeo4jTable = new Neo4jTable();
targetNeo4jTable.setMetadataId(targetTableId);
targetNeo4jTable.setIsSystem("1104");
targetNeo4jTable.setName(targetTable);
targetNeo4jTable.setCnName(targetTableCnName);
targetNeo4jTable = neo4jTableRepository.save(targetNeo4jTable);
}
String targetColumnId = "";
List<Neo4jColumn> targetColumnList = neo4jTableRepository.getColumnByTableId(targetTableId);
for(Neo4jColumn neo4jColumn:targetColumnList){
if(neo4jColumn.getName().equalsIgnoreCase(targetColumn)){
targetColumnId = neo4jColumn.getMetadataId();
break;
}
}
if(targetColumnId == null || targetColumnId.equalsIgnoreCase("")){
targetColumnId = "Column=1104="+targetTable+"="+targetColumn;
Neo4jColumn targetNeo4jColumn = neo4jColumnRepository.findNeo4jColumnByMetadataId(targetColumnId);
if(targetNeo4jColumn == null){
targetNeo4jColumn = new Neo4jColumn();
targetNeo4jColumn.setMetadataId(targetColumnId);
targetNeo4jColumn.setName(targetColumn);
targetNeo4jColumn.setCnName(targetColumnDes);
targetNeo4jColumn = neo4jColumnRepository.save(targetNeo4jColumn);
}
CompositionRelation targetTable2Column = new CompositionRelation();
targetTable2Column.setStart(targetNeo4jTable);
targetTable2Column.setEnd(targetNeo4jColumn);
targetTable2Column.setName("table--column");
compositionRelationRespository.save(targetTable2Column);
}
relationshipService.saveRelation(sourceColumnId,targetColumnId,"流向");
relationshipService.saveRelation(sourceTableId,targetTableId,"流向");
}
}
@Override
public void syschroTable2EtlJobRelations(String catalogName) {
......
src/main/java/com/keymobile/metadata/metadataRelation/service/impl/MongoDbServiceImpl.java
......@@ -164,4 +164,9 @@ public class MongoDbServiceImpl {
query.with(pageable);
return mongoTemplate.count(query, Document.class, PREFIX_MD_RELATION + catalogName);
}
public List<Document> find1104Relations( String catalogName) {
return mongoTemplate.findAll(Document.class, "Relation" + catalogName);
}
}
src/main/java/com/keymobile/metadata/metadataRelation/service/impl/SchemaServiceImpl.java
......@@ -5,8 +5,10 @@ import com.keymobile.metadata.metadataRelation.pojo.metadata.Neo4jSchema;
import com.keymobile.metadata.metadataRelation.pojo.returnBean.ReturnNode;
import com.keymobile.metadata.metadataRelation.remote.TagRemoteService;
import com.keymobile.metadata.metadataRelation.respository.metadata.Neo4jSchemaRepository;
import com.keymobile.metadata.metadataRelation.respository.metadata.Neo4jSystemRepository;
import com.keymobile.metadata.metadataRelation.service.ISchemaService;
import com.keymobile.metadata.metadataRelation.util.SchemaLayerObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
......@@ -100,18 +102,19 @@ public class SchemaServiceImpl implements ISchemaService {
return schemaMap;
}
private static final Logger logger = LoggerFactory.getLogger(SchemaServiceImpl.class);
@Override
public String getLabelBySchemaId(String schemaId) {
List<String> layer = new ArrayList<>();
layer.add("贴源数据层");
layer.add("基础数据层");
layer.add("整合数据层");
layer.add("统计数据层");
layer.add("分析数据层");
layer.add("集市数据层");
layer.add("应用数据层");
layer.add(SchemaLayerObject.tieyuanceng);
layer.add(SchemaLayerObject.jichuceng);
layer.add(SchemaLayerObject.zhengheceng);
layer.add(SchemaLayerObject.tongjiceng);
layer.add(SchemaLayerObject.fenxiceng);
layer.add(SchemaLayerObject.jishiceng);
layer.add(SchemaLayerObject.yingyongceng);
String returnLabel = "";
List<String> metadataIds = new ArrayList<>();
metadataIds.add(schemaId);
......@@ -143,13 +146,13 @@ public class SchemaServiceImpl implements ISchemaService {
* 3.更新schema
*/
List<String> layer = new ArrayList<>();
layer.add("贴源数据层");
layer.add("基础数据层");
layer.add("整合数据层");
layer.add("统计数据层");
layer.add("分析数据层");
layer.add("集市数据层");
layer.add("应用数据层");
layer.add(SchemaLayerObject.tieyuanceng);
layer.add(SchemaLayerObject.jichuceng);
layer.add(SchemaLayerObject.zhengheceng);
layer.add(SchemaLayerObject.tongjiceng);
layer.add(SchemaLayerObject.fenxiceng);
layer.add(SchemaLayerObject.jishiceng);
layer.add(SchemaLayerObject.yingyongceng);
List<String> schemaIdList = new ArrayList<>();
Map<String,Neo4jSchema> neo4jSchemaMap = new HashMap<>();
......@@ -158,10 +161,11 @@ public class SchemaServiceImpl implements ISchemaService {
schemaIdList.add(neo4jSchema.getMetadataId());
neo4jSchemaMap.put(neo4jSchema.getMetadataId(),neo4jSchema);
}
logger.info("开始调用标签接口...");
Map<String,List<Map<String,Object>>> returnResults = tagRemoteService.getTagByMetadataId(schemaIdList, "metadata", "", true);
logger.info("调用标签接口完成。。。。");
for(Object obj : returnResults.keySet()){
String schemaId = (String)obj;
System.out.println(obj+" : "+returnResults.get(obj));
List<Map<String,Object>> mapList = returnResults.get(obj);
for(Map<String,Object> map : mapList){
String label =(String) map.get("name");
......@@ -169,12 +173,13 @@ public class SchemaServiceImpl implements ISchemaService {
Neo4jSchema neo4jSchema = neo4jSchemaMap.get(schemaId);
neo4jSchema.setLabel(label);
neo4jSchemaRepository.save(neo4jSchema);
logger.info("schema："+neo4jSchema.getName() +" 的标签是："+label);
break;
}
}
}
return false;
logger.info("同步所有schema的标签完成。。。。");
return true;
}
@Override
......@@ -182,13 +187,13 @@ public class SchemaServiceImpl implements ISchemaService {
Map<String,String> schemaIdAndLabelMap = new HashMap<>();
List<String> layer = new ArrayList<>();
layer.add("贴源数据层");
layer.add("基础数据层");
layer.add("整合数据层");
layer.add("统计数据层");
layer.add("分析数据层");
layer.add("集市数据层");
layer.add("应用数据层");
layer.add(SchemaLayerObject.tieyuanceng);
layer.add(SchemaLayerObject.jichuceng);
layer.add(SchemaLayerObject.zhengheceng);
layer.add(SchemaLayerObject.tongjiceng);
layer.add(SchemaLayerObject.fenxiceng);
layer.add(SchemaLayerObject.jishiceng);
layer.add(SchemaLayerObject.yingyongceng);
String returnLabel = "";
try {
......
src/main/java/com/keymobile/metadata/metadataRelation/service/impl/SystemServiceImpl.java
......@@ -7,6 +7,7 @@ import com.keymobile.metadata.metadataRelation.pojo.returnBean.ReturnNode;
import com.keymobile.metadata.metadataRelation.pojo.returnBean.ReturnReslult;
import com.keymobile.metadata.metadataRelation.respository.metadata.Neo4jSystemRepository;
import com.keymobile.metadata.metadataRelation.service.ISystemService;
import com.keymobile.metadata.metadataRelation.util.SchemaLayerObject;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
......@@ -19,13 +20,6 @@ public class SystemServiceImpl implements ISystemService {
private Neo4jSystemRepository neo4jSystemRepository;
/**
* layer.add("贴源数据层");
* layer.add("基础数据层");
* layer.add("整合数据层");
* layer.add("统计数据层");
* layer.add("分析数据层");
* layer.add("集市数据层");
* layer.add("应用数据层");
* @param systemId
* @return
*/
......@@ -57,19 +51,19 @@ public class SystemServiceImpl implements ISystemService {
returnNode.setType("Schema");
String label = neo4jSchema.getLabel();
if("贴源数据层".equals(label)){
if(SchemaLayerObject.tieyuanceng.equals(label)){
tyshcList.add(returnNode);
}else if("基础数据层".equals(label)){
}else if(SchemaLayerObject.jichuceng.equals(label)){
jcshcList.add(returnNode);
}else if("整合数据层".equals(label)){
}else if(SchemaLayerObject.zhengheceng.equals(label)){
zhshcList.add(returnNode);
}else if("统计数据层".equals(label)){
}else if(SchemaLayerObject.tongjiceng.equals(label)){
tjshcList.add(returnNode);
}else if("分析数据层".equals(label)){
}else if(SchemaLayerObject.fenxiceng.equals(label)){
fxshcList.add(returnNode);
}else if("集市数据层".equals(label)){
}else if(SchemaLayerObject.jishiceng.equals(label)){
jsshcList.add(returnNode);
}else if("应用数据层".equals(label)){
}else if(SchemaLayerObject.yingyongceng.equals(label)){
yysjcList.add(returnNode);
}else {
// schemaMap.get("未分层").add(returnNode);
......@@ -86,35 +80,27 @@ public class SystemServiceImpl implements ISystemService {
// schemaMap.put(label,returnNodeList);
// }
}
/**
* schemaMap.put("贴源数据层", new ArrayList<>() );
* schemaMap.put("基础数据层", new ArrayList<>() );
* schemaMap.put("整合数据层", new ArrayList<>() );
* schemaMap.put("统计数据层", new ArrayList<>() );
* schemaMap.put("分析数据层", new ArrayList<>() );
* schemaMap.put("集市数据层", new ArrayList<>() );
* schemaMap.put("应用数据层", new ArrayList<>() );
*/
if(tyshcList !=null && tyshcList.size()!=0){
schemaMap.put("贴源数据层",tyshcList);
schemaMap.put(SchemaLayerObject.tieyuanceng,tyshcList);
}
if(jcshcList !=null && jcshcList.size()!=0){
schemaMap.put("基础数据层",jcshcList);
schemaMap.put(SchemaLayerObject.jichuceng,jcshcList);
}
if(zhshcList !=null && zhshcList.size()!=0){
schemaMap.put("整合数据层",zhshcList);
schemaMap.put(SchemaLayerObject.zhengheceng,zhshcList);
}
if(tjshcList !=null && tjshcList.size()!=0){
schemaMap.put("统计数据层",tjshcList);
schemaMap.put(SchemaLayerObject.tongjiceng,tjshcList);
}
if(fxshcList !=null && fxshcList.size()!=0){
schemaMap.put("分析数据层",fxshcList);
schemaMap.put(SchemaLayerObject.fenxiceng,fxshcList);
}
if(jsshcList !=null && jsshcList.size()!=0){
schemaMap.put("集市数据层",jsshcList);
schemaMap.put(SchemaLayerObject.jishiceng,jsshcList);
}
if(yysjcList !=null && yysjcList.size()!=0){
schemaMap.put("应用数据层",yysjcList);
schemaMap.put(SchemaLayerObject.yingyongceng,yysjcList);
}
return schemaMap;
}
......
src/main/java/com/keymobile/metadata/metadataRelation/util/SchemaLayerObject.java 0 → 100644
package com.keymobile.metadata.metadataRelation.util;
public class SchemaLayerObject {
public static String tieyuanceng = "贴源层";
public static String jichuceng = "基础层";
public static String zhengheceng = "整合层";
public static String tongjiceng = "统计层";
public static String fenxiceng = "分析层";
public static String jishiceng = "集市层";
public static String yingyongceng = "应用层";
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment