Skip to content

N
neo4jRelation
Commit d972de1d by qiuchaofei
Options

1调用标签的接口，用来获取schema的分层。2添加删除元数据的接口，

parent 8ba11d2b
Showing with 234 additions and 54 deletions
src/main/java/com/keymobile/metadata/metadataRelation/pojo/metadata/Neo4jSchema.java
......@@ -13,6 +13,6 @@ public class Neo4jSchema extends BaseNode {
public void setLabel(String label) {
this.label = label;
}
//分层
private String label;
}
src/main/java/com/keymobile/metadata/metadataRelation/remote/TagRemoteService.java 0 → 100644
package com.keymobile.metadata.metadataRelation.remote;
import org.springframework.cloud.openfeign.FeignClient;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RequestParam;
import java.util.List;
import java.util.Map;
@FeignClient(name = "tagger")
public interface TagRemoteService {
@RequestMapping(value = "/tagResource/getResourceTagIn", method = RequestMethod.POST)
Map<String,List<Map<String,Object>>> getTagByMetadataId(@RequestParam("resourceIds") List<String> resourceIds,
@RequestParam("type") String type,
@RequestParam("creator") String creator,
@RequestParam("includeAll") boolean includeAll);
}
src/main/java/com/keymobile/metadata/metadataRelation/respository/MetadataRepository.java
......@@ -13,6 +13,9 @@ public interface MetadataRepository extends Neo4jRepository<MetaData, Long> {
List<MetaData> findMetadataByName(String name);
//删除指定节点
@Query("match (n) where n.metadataId =~'.*+catalogName+.*' return n")
void deleteMetaDataByCatalogName(String catalogName);
@Query("match (n:MetaData ) where n.metadataId =~{modelName} return n")
List<MetaData> findMetaDataByModelName(@Param("modelName") String dataName);
......
src/main/java/com/keymobile/metadata/metadataRelation/respository/metadata/Neo4jTableRepository.java
......@@ -28,4 +28,6 @@ public interface Neo4jTableRepository extends Neo4jRepository<Neo4jTable,Long> {
List<BaseNode> getTargetEtlJob(@Param("tableId") String tableId);
@Query("match (n{metadataId:{tableId}} ) <-[r:Composition]-(m) return m ")
List<BaseNode> getSchema(@Param("tableId") String tableId);
}
src/main/java/com/keymobile/metadata/metadataRelation/service/AsyncDataFromMongoToNeo4j.java
......@@ -47,7 +47,8 @@ public class AsyncDataFromMongoToNeo4j {
return user;
}
});
//先删除同个环境的数据
metadataService.deleteNodeByCatalogName(catalogName);
metadataService.syschroMetadataFromMongo(catalogName,mongoDataList);
//先同步元数据，纵向关系,root开始
......
src/main/java/com/keymobile/metadata/metadataRelation/service/IMetadataService.java
......@@ -102,4 +102,6 @@ public interface IMetadataService {
void syschroMetadataFromMongo(String catalogName,List<MongoData> mongoDataList);
//同步mongo中表与表，表与作业的关系
void syschroTable2EtlJobRelations(String catalogName);
void deleteNodeByCatalogName(String catalogName);
}
src/main/java/com/keymobile/metadata/metadataRelation/service/ISchemaService.java
......@@ -9,4 +9,9 @@ public interface ISchemaService {
Map<String,String> getSchemaBySystem(String systemId);
Map<String, List<ReturnNode>> getTablesBySchemaId(String schemaId);
String getLabelBySchemaId(String schemaId);
Map<String,String> getSchemaAndLabelMap(List<String> schemaIds);
}
src/main/java/com/keymobile/metadata/metadataRelation/service/impl/MetadataServiceImpl.java
......@@ -20,6 +20,7 @@ import com.keymobile.metadata.metadataRelation.respository.*;
import com.keymobile.metadata.metadataRelation.respository.metadata.*;
import com.keymobile.metadata.metadataRelation.service.IBaseRelationshipService;
import com.keymobile.metadata.metadataRelation.service.IMetadataService;
import com.keymobile.metadata.metadataRelation.service.ISchemaService;
import com.keymobile.metadata.metadataRelation.util.Neo4jTool;
import org.apache.commons.lang.StringUtils;
import org.bson.Document;
......@@ -92,6 +93,10 @@ public class MetadataServiceImpl implements IMetadataService {
@Autowired
private Neo4jETLScriptRepository neo4jETLScriptRepository;
@Autowired
private ISchemaService schemaService;
@Override
public List<MetaData> findNodeByName(String dataName) {
return metadataRepository.findMetaData(dataName, 3);
......@@ -1273,7 +1278,7 @@ public class MetadataServiceImpl implements IMetadataService {
Map<String ,Neo4jSystem> systemDataMap = new HashMap<>();
for(MongoData systemData:systemDataList){
Neo4jSystem neo4jSystem = new Neo4jSystem();
String systemId = "System="+systemData.get_id();
String systemId = "System="+catalogName+"="+systemData.get_id();
neo4jSystem.setMetadataId(systemId);
neo4jSystem.setName(systemData.getName());
neo4jSystem.setCnName(systemData.getCnName());
......@@ -1297,11 +1302,8 @@ public class MetadataServiceImpl implements IMetadataService {
String mongoId = dataBaseData.get_id();
if(mongoId.startsWith("Server=")){
//如果是server，就把etljob找出来
List<MongoData> etlJobDataList= mongoDbServiceImpl.findDataByparentId(dataBaseData.get_id(),collectionName);
List<Neo4jETLJob> neo4jETLJobList = new ArrayList<>();
for(MongoData etlJobData:etlJobDataList){
if(etlJobCount%200 == 0){
logger.info("同步了"+etlJobCount);
......@@ -1342,28 +1344,30 @@ public class MetadataServiceImpl implements IMetadataService {
compositionRelation.setEnd(neo4jETLScript);
compositionRelation.setName("etlJob--Script");
compositionRelationList.add(compositionRelation);
}
}
if(neo4jETLScriptList !=null && neo4jETLScriptList.size()!=0 ){
neo4jETLScriptRepository.saveAll(neo4jETLScriptList);
}
}
if(neo4jETLJobList !=null && neo4jETLJobList.size()!=0 ){
neo4jETLJobRepository.saveAll(neo4jETLJobList);
}
}else if(mongoId.startsWith("Database=")){
List<MongoData> schemaDataList= mongoDbServiceImpl.findDataByparentId(dataBaseData.get_id(),collectionName);
List<String> schemaIds = new ArrayList<>();
for(MongoData schemaData:schemaDataList){
schemaIds.add(schemaData.get_id());
}
Map<String,String> shemaIdAndLabel = schemaService.getSchemaAndLabelMap(schemaIds);
for(MongoData schemaData:schemaDataList){
//系统与schema的关系
String schemaData_id = schemaData.get_id();
Neo4jSchema neo4jSchema = new Neo4jSchema();
neo4jSchema.setMetadataId(schemaData.get_id());
neo4jSchema.setMetadataId(schemaData_id);
neo4jSchema.setName(schemaData.getName());
neo4jSchema.setCnName(schemaData.getCnName());
......@@ -1375,6 +1379,9 @@ public class MetadataServiceImpl implements IMetadataService {
}
}
String schemaPath = neo4jSystem.getDataPath()+";"+schemaData.getName();
if(shemaIdAndLabel.get(schemaData_id)!=null){
neo4jSchema.setLabel(shemaIdAndLabel.get(schemaData_id));
}
neo4jSchema.setDataPath(schemaPath);
int ran1 = r.nextInt(6);
if(ran1>6){
......@@ -1761,6 +1768,7 @@ public class MetadataServiceImpl implements IMetadataService {
logger.info("创建了多少关系："+relationMap.size());
}
@Override
public void syschroRelationFromMongo(String catalogName) {
//先从mongo读取关系，写入neo4j
......@@ -2214,11 +2222,24 @@ public class MetadataServiceImpl implements IMetadataService {
}
@Override
public void deleteNodeByCatalogName(String catalogName) {
Neo4jConfig neo4jConfig = new Neo4jConfig();
Driver neo4jConnection = neo4jConfig.getNeo4jConnection();
Session session = neo4jConnection.session();
String cypher = "match (n) where n.metadataId =~'.*="+catalogName+"=.*' detach delete n";
session.run(cypher);
}
@Override
public void deleteTempNodeByCatalogName(String catalogName) {
tempNodeRepository.deleteAllRelation(catalogName);
tempNodeRepository.deleteTempNodesByCatalogName(catalogName);
}
@Override
public List<TempNode> findAllTempNode() {
return (List<TempNode>) tempNodeRepository.findAll();
......
src/main/java/com/keymobile/metadata/metadataRelation/service/impl/SchemaServiceImpl.java
......@@ -3,6 +3,7 @@ package com.keymobile.metadata.metadataRelation.service.impl;
import com.keymobile.metadata.metadataRelation.pojo.BaseNode;
import com.keymobile.metadata.metadataRelation.pojo.metadata.Neo4jSchema;
import com.keymobile.metadata.metadataRelation.pojo.returnBean.ReturnNode;
import com.keymobile.metadata.metadataRelation.remote.TagRemoteService;
import com.keymobile.metadata.metadataRelation.respository.metadata.Neo4jSchemaRepository;
import com.keymobile.metadata.metadataRelation.respository.metadata.Neo4jSystemRepository;
import com.keymobile.metadata.metadataRelation.service.ISchemaService;
......@@ -19,6 +20,9 @@ public class SchemaServiceImpl implements ISchemaService {
@Autowired
private Neo4jSchemaRepository neo4jSchemaRepository;
@Autowired
private TagRemoteService tagRemoteService;
@Override
public Map<String, String> getSchemaBySystem(String systemId) {
return null;
......@@ -97,4 +101,72 @@ public class SchemaServiceImpl implements ISchemaService {
return schemaMap;
}
@Override
public String getLabelBySchemaId(String schemaId) {
List<String> layer = new ArrayList<>();
layer.add("贴源数据层");
layer.add("基础数据层");
layer.add("整合数据层");
layer.add("统计数据层");
layer.add("分析数据层");
layer.add("集市数据层");
layer.add("应用数据层");
String returnLabel = "";
List<String> metadataIds = new ArrayList<>();
metadataIds.add(schemaId);
try{
Map<String,List<Map<String,Object>>> returnResults = tagRemoteService.getTagByMetadataId(metadataIds, "metadata", "", true);
for(Object obj : returnResults.keySet()){
System.out.println(obj+" : "+returnResults.get(obj));
List<Map<String,Object>> mapList = returnResults.get(obj);
for(Map<String,Object> map : mapList){
String label =(String) map.get("name");
if(layer.contains(label)){
returnLabel = label;
}
}
}
}catch (Exception e){
e.printStackTrace();
}
return returnLabel;
}
@Override
public Map<String, String> getSchemaAndLabelMap(List<String> schemaIds) {
Map<String,String> schemaIdAndLabelMap = new HashMap<>();
List<String> layer = new ArrayList<>();
layer.add("贴源数据层");
layer.add("基础数据层");
layer.add("整合数据层");
layer.add("统计数据层");
layer.add("分析数据层");
layer.add("集市数据层");
layer.add("应用数据层");
String returnLabel = "";
try {
Map<String,List<Map<String,Object>>> returnResults = tagRemoteService.getTagByMetadataId(schemaIds, "metadata", "", true);
for(Object obj : returnResults.keySet()){
String schemaId = (String)obj;
System.out.println(obj+" : "+returnResults.get(obj));
List<Map<String,Object>> mapList = returnResults.get(obj);
for(Map<String,Object> map : mapList){
String label =(String) map.get("name");
if(layer.contains(label)){
schemaIdAndLabelMap.put(schemaId,label);
}
}
}
}catch (Exception e){
e.printStackTrace();
}
return schemaIdAndLabelMap;
}
}
src/main/java/com/keymobile/metadata/metadataRelation/service/impl/SystemServiceImpl.java
......@@ -9,10 +9,7 @@ import com.keymobile.metadata.metadataRelation.service.ISystemService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.*;
@Service
public class SystemServiceImpl implements ISystemService {
......@@ -20,11 +17,37 @@ public class SystemServiceImpl implements ISystemService {
@Autowired
private Neo4jSystemRepository neo4jSystemRepository;
/**
* layer.add("贴源数据层");
* layer.add("基础数据层");
* layer.add("整合数据层");
* layer.add("统计数据层");
* layer.add("分析数据层");
* layer.add("集市数据层");
* layer.add("应用数据层");
* @param systemId
* @return
*/
@Override
public Map<String ,List<ReturnNode>> getSchemaBySystemId(String systemId) {
Map<String ,List<ReturnNode>> schemaMap = new HashMap<>();
Map<String ,List<ReturnNode>> schemaMap = new LinkedHashMap<>();
schemaMap.put("贴源数据层", new ArrayList<>() );
List<Neo4jSchema> neo4jSchemaList = neo4jSystemRepository.getSchemaBySystemId(systemId);
schemaMap.put("基础数据层", new ArrayList<>() );
schemaMap.put("整合数据层", new ArrayList<>() );
schemaMap.put("统计数据层", new ArrayList<>() );
schemaMap.put("分析数据层", new ArrayList<>() );
schemaMap.put("集市数据层", new ArrayList<>() );
schemaMap.put("应用数据层", new ArrayList<>() );
List<Neo4jSchema> neo4jSchemaList = new ArrayList<>();
schemaMap.put("未分层", new ArrayList<>() );
try{
neo4jSchemaList = neo4jSystemRepository.getSchemaBySystemId(systemId);
}
catch (Exception e){
e.printStackTrace();
}
for(Neo4jSchema neo4jSchema:neo4jSchemaList){
ReturnNode returnNode = new ReturnNode();
......@@ -33,19 +56,34 @@ public class SystemServiceImpl implements ISystemService {
returnNode.setType("Schema");
String label = neo4jSchema.getLabel();
if(label == null || label.equals("")){
label = "未分层";
}
if(schemaMap .containsKey(label)){
List<ReturnNode> returnNodeList = schemaMap.get(label);
if(!returnNodeList.contains(returnNode)){
returnNodeList.add(returnNode);
}
}else{
List<ReturnNode> returnNodeList = new ArrayList<>();
returnNodeList.add(returnNode);
schemaMap.put(label,returnNodeList);
if("贴源数据层".equals(label)){
schemaMap.get(label).add(returnNode);
}else if("基础数据层".equals(label)){
schemaMap.get(label).add(returnNode);
}else if("整合数据层".equals(label)){
schemaMap.get(label).add(returnNode);
}else if("统计数据层".equals(label)){
schemaMap.get(label).add(returnNode);
}else if("分析数据层".equals(label)){
schemaMap.get(label).add(returnNode);
}else if("集市数据层".equals(label)){
schemaMap.get(label).add(returnNode);
}else if("应用数据层".equals(label)){
schemaMap.get(label).add(returnNode);
}else {
schemaMap.get("未分层").add(returnNode);
}
// if(schemaMap .containsKey(label)){
// List<ReturnNode> returnNodeList = schemaMap.get(label);
// if(!returnNodeList.contains(returnNode)){
// returnNodeList.add(returnNode);
// }
// }else{
// List<ReturnNode> returnNodeList = new ArrayList<>();
// returnNodeList.add(returnNode);
// schemaMap.put(label,returnNodeList);
// }
}
return schemaMap;
}
......
src/main/java/com/keymobile/metadata/metadataRelation/service/impl/TableServiceImpl.java
......@@ -3,12 +3,14 @@ package com.keymobile.metadata.metadataRelation.service.impl;
import com.keymobile.metadata.metadataRelation.config.Neo4jConfig;
import com.keymobile.metadata.metadataRelation.pojo.BaseNode;
import com.keymobile.metadata.metadataRelation.pojo.MetaData;
import com.keymobile.metadata.metadataRelation.pojo.metadata.Neo4jSchema;
import com.keymobile.metadata.metadataRelation.pojo.metadata.Neo4jSystem;
import com.keymobile.metadata.metadataRelation.pojo.metadata.Neo4jTable;
import com.keymobile.metadata.metadataRelation.pojo.returnBean.ReturnNode;
import com.keymobile.metadata.metadataRelation.remote.DataAssertRemoteService;
import com.keymobile.metadata.metadataRelation.remote.DataStandardRemoteService;
import com.keymobile.metadata.metadataRelation.remote.MetadataRepoRemoteService;
import com.keymobile.metadata.metadataRelation.respository.metadata.Neo4jSchemaRepository;
import com.keymobile.metadata.metadataRelation.respository.metadata.Neo4jSystemRepository;
import com.keymobile.metadata.metadataRelation.respository.metadata.Neo4jTableRepository;
import com.keymobile.metadata.metadataRelation.service.ITableService;
......@@ -33,6 +35,10 @@ public class TableServiceImpl implements ITableService {
@Autowired
private Neo4jSystemRepository neo4jSystemRepository;
@Autowired
private Neo4jSchemaRepository neo4jSchemaRepository;
@Autowired
private MetadataRepoRemoteService metadataRepoRemoteService;
......@@ -45,7 +51,6 @@ public class TableServiceImpl implements ITableService {
@Override
public Map<String, List<ReturnNode>> getRelationObjectByTableId(String tableId) {
//传入表id，查找与该表相关的“模型，资产，标准，质量，作业”，
Map<String, List<ReturnNode>> relationObjects = new HashMap<>();
//作业
List<ReturnNode> etlJobs = new ArrayList<>();
......@@ -121,8 +126,10 @@ public class TableServiceImpl implements ITableService {
}
long start = System.currentTimeMillis();
List<ReturnNode> neo4jTableList = getCurrentTableInfo(tableId);
long ent = System.currentTimeMillis();
logger.info("查找当前表用时："+(ent-start));
relationObjects.put("当前表",neo4jTableList);
return relationObjects;
......@@ -283,7 +290,12 @@ public class TableServiceImpl implements ITableService {
Map<String,String> returnMap = new HashMap<>();
Map<String, Object> attriuteMap = null;
try{
attriuteMap = metadataRepoRemoteService.getMetadata(tableId);
long start = System.currentTimeMillis();
attriuteMap = metadataRepoRemoteService.getMetadata(tableId);
long end = System.currentTimeMillis();
logger.info(tableId+"调用repo服务用时:"+(end-start));
}catch (Exception e){
logger.info("元数据服务异常："+e.getMessage());
}
......@@ -303,9 +315,6 @@ public class TableServiceImpl implements ITableService {
// 英文名称，中文名称，说明，表规模（行），表大小（字节），更新时间，
// 所属系统，所属分层，加载脚本，质量情况，相关模型
String name = (String) attriuteMap.get("name");
// if (attriuteMap.get("cnName") != null && !attriuteMap.get("cnName").equals("")) {
// name = (String) attriuteMap.get("cnName");
// }
returnMap.put("英文名称",name);
String cnName = (String) attriuteMap.get("cnName");
returnMap.put("中文名称",cnName);
......@@ -330,16 +339,23 @@ public class TableServiceImpl implements ITableService {
//所属分层,从schema获取
String parentId = (String) attriuteMap.get("parentId");
Map<String, Object> schemaAttributeMap = null;
String label = null;
String schemaName = "";
try{
schemaAttributeMap = metadataRepoRemoteService.getMetadata(parentId);
long start = System.currentTimeMillis();
neo4jTableRepository.getSchema(tableId);
List<Neo4jSchema> neo4jSchemaList = neo4jSchemaRepository.findNeo4jSchemaByMetadataId(parentId);
if(neo4jSchemaList !=null && neo4jSchemaList.size()!=0){
Neo4jSchema neo4jSchema = neo4jSchemaList.get(0);
schemaName = neo4jSchema.getName();
label = neo4jSchema.getLabel();
}
long end = System.currentTimeMillis();
logger.info(tableId+"调用repo服务用时:"+(end-start));
}catch (Exception e){
logger.info("元数据服务异常："+e.getMessage());
}
String label = (String)schemaAttributeMap.get("label");
if(label== null ){
List<String> layer = new ArrayList<>();
......@@ -359,16 +375,16 @@ public class TableServiceImpl implements ITableService {
label = layer.get(ran1);
}
returnMap.put("所属分层",label);
List<String> paths =(List<String>) attriuteMap.get("namePathList");
if(paths !=null ){
StringBuffer sb = new StringBuffer();
int size = paths.size();
for(int i =0 ;i<size;i++){
sb.append(paths.get(i)+";");
}
returnMap.put("路径",sb.toString());
}
returnMap.put("所属Schema",schemaName);
// List<String> paths =(List<String>) attriuteMap.get("namePathList");
// if(paths !=null ){
// StringBuffer sb = new StringBuffer();
// int size = paths.size();
// for(int i =0 ;i<size;i++){
// sb.append(paths.get(i)+";");
// }
// returnMap.put("路径",sb.toString());
// }
return returnMap;
}
......@@ -383,7 +399,7 @@ public class TableServiceImpl implements ITableService {
returnNode.setName(baseNode.getName());
returnNode.setType("Table");
// 获取属性
Map<String,String> attributeMap = getAttributeMap(tableId);
Map<String,String> attributeMap = getAttributeMap(metadataId);
String systemString = attributeMap.get("所属系统");
String belongSystem = "";
......@@ -417,7 +433,7 @@ public class TableServiceImpl implements ITableService {
returnNode.setType("Table");
// 获取属性
Map<String,String> attributeMap = getAttributeMap(tableId);
Map<String,String> attributeMap = getAttributeMap(metadataId);
String systemString = attributeMap.get("所属系统");
String belongSystem = "";
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment