Commit 7d20204d by qiuchaofei

1,查询孤立点的语句,2,修改操作日志内容,3同步流向关系时 添加日志。

parent e3732c49
package com.keymobile.metadata.metadataRelation.controller;
import com.keymobile.metadata.metadataRelation.pojo.returnBean.ReturnNode;
import com.keymobile.metadata.metadataRelation.service.DataRelationAnalyService;
import io.swagger.annotations.Api;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RestController;
import java.util.Map;
@RestController
@Api(value = "元数据的分析", tags = "metadata analysis")
public class DataRelationAnalyController {
private static final Logger logger = LoggerFactory.getLogger(DataRelationAnalyController.class);
@Autowired
private DataRelationAnalyService dataRelationAnalyService;
@RequestMapping(path = "/getIsolatedNode", method = RequestMethod.GET)
public Map<String, ReturnNode> getIsolatedNode(String parentId) {
logger.info("查找指定schema下面的孤立点。");
return dataRelationAnalyService.findTableWithoutRelations(parentId);
}
}
......@@ -4,10 +4,8 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.stream.Collectors;
import com.keymobile.metadata.metadataRelation.logging.LogConstants;
......@@ -29,6 +27,8 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.system.ApplicationHome;
import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.security.core.userdetails.UserDetails;
import org.springframework.util.CollectionUtils;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartFile;
......@@ -547,8 +547,8 @@ public class MetaDataController {
@ApiOperation(tags = "", value = "同步元数据关系的接口")
@RequestMapping(path = "/syschroRelationFromMongo", method = RequestMethod.POST)
public String synchroRelationFromMongo(String catalogName) {
LogManager.logInfo(LogConstants.CTX_Relation, "同步"+catalogName+"的元数据。" );
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
LogManager.logInfo(LogConstants.CTX_Relation, "用户 "+ getUserName() + " 在 "+ sdf.format(new Date())+"同步"+catalogName+"的元数据。" );
logger.info("开始异步迁移元数据:从mongo到neo4j");
String flag = "success";
asyncDataFromMongoToNeo4j.asyncDataFromMongoToNeo4j(catalogName);
......@@ -556,7 +556,19 @@ public class MetaDataController {
logger.info("返回结果,后台继续运行。");
return flag;
}
private String getUserName() {
Object obj = SecurityContextHolder.getContext().getAuthentication().getPrincipal();
if (obj instanceof String) {
return obj.toString();
}
UserDetails userDetails = (UserDetails) SecurityContextHolder.getContext().getAuthentication().getPrincipal();
String[] concat = userDetails.getUsername().split(":");
String username = null;
if (concat.length >= 1) {
username = concat[0];
}
return username;
}
@PostMapping("/addTempRelation")
public void addTempRelation(@RequestParam String catalogName, @RequestBody List<Map<String, Object>> list) {
if (!CollectionUtils.isEmpty(list)) {
......
package com.keymobile.metadata.metadataRelation.controller;
import com.keymobile.metadata.metadataRelation.config.FeignClientConfig;
import com.keymobile.metadata.metadataRelation.logging.LogConstants;
import com.keymobile.metadata.metadataRelation.logging.LogManager;
import com.keymobile.metadata.metadataRelation.pojo.metadata.Neo4jSchema;
......@@ -13,15 +14,20 @@ import com.keymobile.metadata.metadataRelation.service.impl.SystemServiceImpl;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import net.sf.json.JSONObject;
import org.apache.catalina.security.SecurityConfig;
import org.apache.commons.lang3.time.DateUtils;
import org.hibernate.loader.custom.Return;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.security.core.userdetails.UserDetails;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RestController;
import java.sql.SQLOutput;
import java.text.SimpleDateFormat;
import java.util.*;
@RestController
......@@ -241,7 +247,8 @@ public class RelationalGraphController {
@ApiOperation(tags = "", value = "传入表的id,返回表的上下游一层关系。")
@RequestMapping(path = "/getRelationTablesTableId", method = RequestMethod.GET)
public Map<String,List<ReturnNode>> getRelationTablesTableId(String tableId){
LogManager.logInfo(LogConstants.CTX_Relation, "查看"+tableId+"的全链分析" );
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
LogManager.logInfo(LogConstants.CTX_Relation, "用户 "+ getUserName() + " 在 "+ sdf.format(new Date()) + " 查看了 "+tableId+"的全链分析" );
Map<String, List<ReturnNode>> relationTables = new HashMap<>();
long start = System.currentTimeMillis();
......@@ -260,12 +267,28 @@ public class RelationalGraphController {
return relationTables;
}
private String getUserName() {
Object obj = SecurityContextHolder.getContext().getAuthentication().getPrincipal();
if (obj instanceof String) {
return obj.toString();
}
UserDetails userDetails = (UserDetails) SecurityContextHolder.getContext().getAuthentication().getPrincipal();
String[] concat = userDetails.getUsername().split(":");
String username = null;
if (concat.length >= 1) {
username = concat[0];
}
return username;
}
//表的分析,关系分析--表与模型,表与质量,表与标准,表与作业的关系
@ApiOperation(tags = "", value = "传入表的id,返回与表相关的模型,标准,质量,资产,作业等对象")
@RequestMapping(path = "/getRelationObjectByTableId", method = RequestMethod.GET)
public Map<String,List<ReturnNode>> getRelationObjectByTableId(String tableId){
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
LogManager.logInfo(LogConstants.CTX_Relation, "用户 "+ getUserName() + " 在 "+ sdf.format(new Date()) + " 查看了 "+tableId+"的关系分析" );
LogManager.logInfo(LogConstants.CTX_Relation, "查看"+tableId+"的关系分析" );
long start = System.currentTimeMillis();
Map<String, List<ReturnNode>> relationObjects = tableService.getRelationObjectByTableId (tableId);
long end = System.currentTimeMillis();
......
package com.keymobile.metadata.metadataRelation.service;
import com.keymobile.metadata.metadataRelation.pojo.returnBean.ReturnNode;
import java.util.Map;
public interface DataRelationAnalyService {
Map<String, ReturnNode> findTableWithoutRelations(String schemaId);
}
package com.keymobile.metadata.metadataRelation.service.impl;
import com.keymobile.metadata.metadataRelation.config.Neo4jConfig;
import com.keymobile.metadata.metadataRelation.pojo.MetaData;
import com.keymobile.metadata.metadataRelation.pojo.returnBean.ReturnNode;
import com.keymobile.metadata.metadataRelation.service.DataRelationAnalyService;
import com.keymobile.metadata.metadataRelation.util.Neo4jTool;
import org.neo4j.driver.v1.*;
import org.neo4j.driver.v1.types.Node;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@Service
public class DataRelationAnalyServiceImpl implements DataRelationAnalyService {
private static final Logger logger = LoggerFactory.getLogger(DataRelationAnalyServiceImpl.class);
@Override
public Map<String, ReturnNode> findTableWithoutRelations(String parentId) {
Map<String, ReturnNode> returnNodeMap = new HashMap<>();
//区分传入的id是系统还是schema,还是为空,为空则找出所有的孤立点
//match (n:Neo4jTable) where not ((n)-[:`流向`]-(:Neo4jTable)) return n
Neo4jConfig neo4jConfig = new Neo4jConfig();
Driver neo4jConnection = neo4jConfig.getNeo4jConnection();
Session session = neo4jConnection.session();
String condition = "";
if(parentId !=null && parentId .startsWith("System=")){
condition= "<-[r1:Composition]-(:Neo4jSchema)<-[r2:Composition]-(m:Neo4jSystem{metadataId:\""+parentId+"\"})";
}else if(parentId !=null && parentId .startsWith("Schema=")){
condition= "<-[r1:Composition]-(m:Neo4jSchema{metadataId:\""+parentId+"\"})";
}else {
}
String cypher = "match (n:Neo4jTable) " + condition + "where not ((n)-[:`流向`]-()) return n";
logger .info("查找孤立表的语句:"+cypher);
StatementResult result = session.run(cypher);
while(result.hasNext()){
Record record = result.next();
List<Value> values = record.values();
for (Value value : values) {
if(value.type().name().equals("NODE")){
Node node = value.asNode();
Map<String, Object> stringObjectMap = node.asMap();
MetaData metaData = new MetaData();
Neo4jTool.transMap2Bean(stringObjectMap, metaData);
if(metaData.getMetadataId() == null ){
metaData.setMetadataId("System="+metaData.getName());
}
ReturnNode returnNode = new ReturnNode();
returnNode.setName(metaData.getName());
returnNode.setId(metaData.getMetadataId());
returnNode.setType(metaData.getMetaModel());
returnNode.setCnName(metaData.getCnName());
returnNodeMap.put(metaData.getMetadataId(),returnNode);
}
}
}
return returnNodeMap;
}
}
......@@ -1857,19 +1857,23 @@ public class MetadataServiceImpl implements IMetadataService {
}
@Override
public void syschroTable2EtlJobRelations(String catalogName) {
Map<String, String> relationMap = new HashMap<>();
Map<String,String> edgeIdMap = new HashMap<>();
int page =0,pageSize = 300;
long time0 = System.currentTimeMillis();
long totalElement = mongoDbServiceImpl.countRelation(PageRequest.of(page,pageSize),catalogName);
long time1 = System.currentTimeMillis();
logger.info("从 mmongo获取的关系的总数量是:"+ totalElement +" ,耗时:" +(time1-time0) );
int totalPageCount = ((int)totalElement/pageSize)+1;
logger.info("从 mmongo分页获取关系,每页:"+ pageSize);
for(; page< totalPageCount;page++){
logger.info("开始从 mmongo 第"+page +" 次获取关系。");
String description = "";
List<Document> relationList = mongoDbServiceImpl.findRelationByPage(PageRequest.of(page,pageSize,Sort.by("_id")),catalogName);
long time3 = System.currentTimeMillis();
List<Document> relationList = mongoDbServiceImpl.findRelationByPage(PageRequest.of(page,pageSize,Sort.by("_id")),catalogName);
long time4 = System.currentTimeMillis();
logger.info("从 mmongo 第"+page +" 次获取关系。 ,耗时:" +(time4-time3) );
for(Document relation :relationList){
description = relation.getString("description");
String type = relation.getString("type");
......@@ -1973,7 +1977,7 @@ public class MetadataServiceImpl implements IMetadataService {
}
long end = System.currentTimeMillis();
logger.info("创建了多少关系:"+edgeIdMap.size() + " ,用时:"+(end-start));
logger.info("一共在neo4j,创建了多少关系:"+edgeIdMap.size() + " ,总用时:"+(end-start));
// List<RelationMongo> relationMongoList = mongoDbServiceImpl.findAllRelationByCatalog(catalogName);
// int size = 0;
......
......@@ -18,4 +18,9 @@ public class SchemaLayerObject {
public static String jishicengData = "集市数据层";
public static String yingyongcengData = "应用数据层";
//支持的关系类型
public static String table2TableRelations = "表-表关系";
public static String job2JobRelations = "作业-作业关系";
public static String script2ScriptRelations = "脚本-脚本关系";
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment