Skip to content
Projects
Groups
Snippets
Help
This project
Loading...
Sign in / Register
Toggle navigation
N
neo4jRelation
Overview
Overview
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
qiuchaofei
neo4jRelation
Commits
7d20204d
Commit
7d20204d
authored
Apr 21, 2022
by
qiuchaofei
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
1,查询孤立点的语句,2,修改操作日志内容,3同步流向关系时 添加日志。
parent
e3732c49
Hide whitespace changes
Inline
Side-by-side
Showing
7 changed files
with
170 additions
and
15 deletions
+170
-15
DataRelationAnalyController.java
...adataRelation/controller/DataRelationAnalyController.java
+31
-0
MetaDataController.java
...adata/metadataRelation/controller/MetaDataController.java
+19
-7
RelationalGraphController.java
...etadataRelation/controller/RelationalGraphController.java
+25
-2
DataRelationAnalyService.java
...ta/metadataRelation/service/DataRelationAnalyService.java
+11
-0
DataRelationAnalyServiceImpl.java
...taRelation/service/impl/DataRelationAnalyServiceImpl.java
+69
-0
MetadataServiceImpl.java
...ta/metadataRelation/service/impl/MetadataServiceImpl.java
+10
-6
SchemaLayerObject.java
...ile/metadata/metadataRelation/util/SchemaLayerObject.java
+5
-0
No files found.
src/main/java/com/keymobile/metadata/metadataRelation/controller/DataRelationAnalyController.java
0 → 100644
View file @
7d20204d
package
com
.
keymobile
.
metadata
.
metadataRelation
.
controller
;
import
com.keymobile.metadata.metadataRelation.pojo.returnBean.ReturnNode
;
import
com.keymobile.metadata.metadataRelation.service.DataRelationAnalyService
;
import
io.swagger.annotations.Api
;
import
org.slf4j.Logger
;
import
org.slf4j.LoggerFactory
;
import
org.springframework.beans.factory.annotation.Autowired
;
import
org.springframework.web.bind.annotation.RequestMapping
;
import
org.springframework.web.bind.annotation.RequestMethod
;
import
org.springframework.web.bind.annotation.RestController
;
import
java.util.Map
;
@RestController
@Api
(
value
=
"元数据的分析"
,
tags
=
"metadata analysis"
)
public
class
DataRelationAnalyController
{
private
static
final
Logger
logger
=
LoggerFactory
.
getLogger
(
DataRelationAnalyController
.
class
);
@Autowired
private
DataRelationAnalyService
dataRelationAnalyService
;
@RequestMapping
(
path
=
"/getIsolatedNode"
,
method
=
RequestMethod
.
GET
)
public
Map
<
String
,
ReturnNode
>
getIsolatedNode
(
String
parentId
)
{
logger
.
info
(
"查找指定schema下面的孤立点。"
);
return
dataRelationAnalyService
.
findTableWithoutRelations
(
parentId
);
}
}
src/main/java/com/keymobile/metadata/metadataRelation/controller/MetaDataController.java
View file @
7d20204d
...
...
@@ -4,10 +4,8 @@ import java.io.File;
import
java.io.FileInputStream
;
import
java.io.IOException
;
import
java.io.InputStream
;
import
java.util.ArrayList
;
import
java.util.HashMap
;
import
java.util.List
;
import
java.util.Map
;
import
java.text.SimpleDateFormat
;
import
java.util.*
;
import
java.util.stream.Collectors
;
import
com.keymobile.metadata.metadataRelation.logging.LogConstants
;
...
...
@@ -29,6 +27,8 @@ import org.slf4j.Logger;
import
org.slf4j.LoggerFactory
;
import
org.springframework.beans.factory.annotation.Autowired
;
import
org.springframework.boot.system.ApplicationHome
;
import
org.springframework.security.core.context.SecurityContextHolder
;
import
org.springframework.security.core.userdetails.UserDetails
;
import
org.springframework.util.CollectionUtils
;
import
org.springframework.web.bind.annotation.*
;
import
org.springframework.web.multipart.MultipartFile
;
...
...
@@ -547,8 +547,8 @@ public class MetaDataController {
@ApiOperation
(
tags
=
""
,
value
=
"同步元数据关系的接口"
)
@RequestMapping
(
path
=
"/syschroRelationFromMongo"
,
method
=
RequestMethod
.
POST
)
public
String
synchroRelationFromMongo
(
String
catalogName
)
{
LogManager
.
logInfo
(
LogConstants
.
CTX_Relation
,
"同步"
+
catalogName
+
"的元数据。"
);
SimpleDateFormat
sdf
=
new
SimpleDateFormat
(
"yyyy-MM-dd HH:mm:ss"
);
LogManager
.
logInfo
(
LogConstants
.
CTX_Relation
,
"
用户 "
+
getUserName
()
+
" 在 "
+
sdf
.
format
(
new
Date
())+
"
同步"
+
catalogName
+
"的元数据。"
);
logger
.
info
(
"开始异步迁移元数据:从mongo到neo4j"
);
String
flag
=
"success"
;
asyncDataFromMongoToNeo4j
.
asyncDataFromMongoToNeo4j
(
catalogName
);
...
...
@@ -556,7 +556,19 @@ public class MetaDataController {
logger
.
info
(
"返回结果,后台继续运行。"
);
return
flag
;
}
private
String
getUserName
()
{
Object
obj
=
SecurityContextHolder
.
getContext
().
getAuthentication
().
getPrincipal
();
if
(
obj
instanceof
String
)
{
return
obj
.
toString
();
}
UserDetails
userDetails
=
(
UserDetails
)
SecurityContextHolder
.
getContext
().
getAuthentication
().
getPrincipal
();
String
[]
concat
=
userDetails
.
getUsername
().
split
(
":"
);
String
username
=
null
;
if
(
concat
.
length
>=
1
)
{
username
=
concat
[
0
];
}
return
username
;
}
@PostMapping
(
"/addTempRelation"
)
public
void
addTempRelation
(
@RequestParam
String
catalogName
,
@RequestBody
List
<
Map
<
String
,
Object
>>
list
)
{
if
(!
CollectionUtils
.
isEmpty
(
list
))
{
...
...
src/main/java/com/keymobile/metadata/metadataRelation/controller/RelationalGraphController.java
View file @
7d20204d
package
com
.
keymobile
.
metadata
.
metadataRelation
.
controller
;
import
com.keymobile.metadata.metadataRelation.config.FeignClientConfig
;
import
com.keymobile.metadata.metadataRelation.logging.LogConstants
;
import
com.keymobile.metadata.metadataRelation.logging.LogManager
;
import
com.keymobile.metadata.metadataRelation.pojo.metadata.Neo4jSchema
;
...
...
@@ -13,15 +14,20 @@ import com.keymobile.metadata.metadataRelation.service.impl.SystemServiceImpl;
import
io.swagger.annotations.Api
;
import
io.swagger.annotations.ApiOperation
;
import
net.sf.json.JSONObject
;
import
org.apache.catalina.security.SecurityConfig
;
import
org.apache.commons.lang3.time.DateUtils
;
import
org.hibernate.loader.custom.Return
;
import
org.slf4j.Logger
;
import
org.slf4j.LoggerFactory
;
import
org.springframework.beans.factory.annotation.Autowired
;
import
org.springframework.security.core.context.SecurityContextHolder
;
import
org.springframework.security.core.userdetails.UserDetails
;
import
org.springframework.web.bind.annotation.RequestMapping
;
import
org.springframework.web.bind.annotation.RequestMethod
;
import
org.springframework.web.bind.annotation.RestController
;
import
java.sql.SQLOutput
;
import
java.text.SimpleDateFormat
;
import
java.util.*
;
@RestController
...
...
@@ -241,7 +247,8 @@ public class RelationalGraphController {
@ApiOperation
(
tags
=
""
,
value
=
"传入表的id,返回表的上下游一层关系。"
)
@RequestMapping
(
path
=
"/getRelationTablesTableId"
,
method
=
RequestMethod
.
GET
)
public
Map
<
String
,
List
<
ReturnNode
>>
getRelationTablesTableId
(
String
tableId
){
LogManager
.
logInfo
(
LogConstants
.
CTX_Relation
,
"查看"
+
tableId
+
"的全链分析"
);
SimpleDateFormat
sdf
=
new
SimpleDateFormat
(
"yyyy-MM-dd HH:mm:ss"
);
LogManager
.
logInfo
(
LogConstants
.
CTX_Relation
,
"用户 "
+
getUserName
()
+
" 在 "
+
sdf
.
format
(
new
Date
())
+
" 查看了 "
+
tableId
+
"的全链分析"
);
Map
<
String
,
List
<
ReturnNode
>>
relationTables
=
new
HashMap
<>();
long
start
=
System
.
currentTimeMillis
();
...
...
@@ -260,12 +267,28 @@ public class RelationalGraphController {
return
relationTables
;
}
private
String
getUserName
()
{
Object
obj
=
SecurityContextHolder
.
getContext
().
getAuthentication
().
getPrincipal
();
if
(
obj
instanceof
String
)
{
return
obj
.
toString
();
}
UserDetails
userDetails
=
(
UserDetails
)
SecurityContextHolder
.
getContext
().
getAuthentication
().
getPrincipal
();
String
[]
concat
=
userDetails
.
getUsername
().
split
(
":"
);
String
username
=
null
;
if
(
concat
.
length
>=
1
)
{
username
=
concat
[
0
];
}
return
username
;
}
//表的分析,关系分析--表与模型,表与质量,表与标准,表与作业的关系
@ApiOperation
(
tags
=
""
,
value
=
"传入表的id,返回与表相关的模型,标准,质量,资产,作业等对象"
)
@RequestMapping
(
path
=
"/getRelationObjectByTableId"
,
method
=
RequestMethod
.
GET
)
public
Map
<
String
,
List
<
ReturnNode
>>
getRelationObjectByTableId
(
String
tableId
){
SimpleDateFormat
sdf
=
new
SimpleDateFormat
(
"yyyy-MM-dd HH:mm:ss"
);
LogManager
.
logInfo
(
LogConstants
.
CTX_Relation
,
"用户 "
+
getUserName
()
+
" 在 "
+
sdf
.
format
(
new
Date
())
+
" 查看了 "
+
tableId
+
"的关系分析"
);
LogManager
.
logInfo
(
LogConstants
.
CTX_Relation
,
"查看"
+
tableId
+
"的关系分析"
);
long
start
=
System
.
currentTimeMillis
();
Map
<
String
,
List
<
ReturnNode
>>
relationObjects
=
tableService
.
getRelationObjectByTableId
(
tableId
);
long
end
=
System
.
currentTimeMillis
();
...
...
src/main/java/com/keymobile/metadata/metadataRelation/service/DataRelationAnalyService.java
0 → 100644
View file @
7d20204d
package
com
.
keymobile
.
metadata
.
metadataRelation
.
service
;
import
com.keymobile.metadata.metadataRelation.pojo.returnBean.ReturnNode
;
import
java.util.Map
;
public
interface
DataRelationAnalyService
{
Map
<
String
,
ReturnNode
>
findTableWithoutRelations
(
String
schemaId
);
}
src/main/java/com/keymobile/metadata/metadataRelation/service/impl/DataRelationAnalyServiceImpl.java
0 → 100644
View file @
7d20204d
package
com
.
keymobile
.
metadata
.
metadataRelation
.
service
.
impl
;
import
com.keymobile.metadata.metadataRelation.config.Neo4jConfig
;
import
com.keymobile.metadata.metadataRelation.pojo.MetaData
;
import
com.keymobile.metadata.metadataRelation.pojo.returnBean.ReturnNode
;
import
com.keymobile.metadata.metadataRelation.service.DataRelationAnalyService
;
import
com.keymobile.metadata.metadataRelation.util.Neo4jTool
;
import
org.neo4j.driver.v1.*
;
import
org.neo4j.driver.v1.types.Node
;
import
org.slf4j.Logger
;
import
org.slf4j.LoggerFactory
;
import
org.springframework.stereotype.Service
;
import
java.util.HashMap
;
import
java.util.List
;
import
java.util.Map
;
@Service
public
class
DataRelationAnalyServiceImpl
implements
DataRelationAnalyService
{
private
static
final
Logger
logger
=
LoggerFactory
.
getLogger
(
DataRelationAnalyServiceImpl
.
class
);
@Override
public
Map
<
String
,
ReturnNode
>
findTableWithoutRelations
(
String
parentId
)
{
Map
<
String
,
ReturnNode
>
returnNodeMap
=
new
HashMap
<>();
//区分传入的id是系统还是schema,还是为空,为空则找出所有的孤立点
//match (n:Neo4jTable) where not ((n)-[:`流向`]-(:Neo4jTable)) return n
Neo4jConfig
neo4jConfig
=
new
Neo4jConfig
();
Driver
neo4jConnection
=
neo4jConfig
.
getNeo4jConnection
();
Session
session
=
neo4jConnection
.
session
();
String
condition
=
""
;
if
(
parentId
!=
null
&&
parentId
.
startsWith
(
"System="
)){
condition
=
"<-[r1:Composition]-(:Neo4jSchema)<-[r2:Composition]-(m:Neo4jSystem{metadataId:\""
+
parentId
+
"\"})"
;
}
else
if
(
parentId
!=
null
&&
parentId
.
startsWith
(
"Schema="
)){
condition
=
"<-[r1:Composition]-(m:Neo4jSchema{metadataId:\""
+
parentId
+
"\"})"
;
}
else
{
}
String
cypher
=
"match (n:Neo4jTable) "
+
condition
+
"where not ((n)-[:`流向`]-()) return n"
;
logger
.
info
(
"查找孤立表的语句:"
+
cypher
);
StatementResult
result
=
session
.
run
(
cypher
);
while
(
result
.
hasNext
()){
Record
record
=
result
.
next
();
List
<
Value
>
values
=
record
.
values
();
for
(
Value
value
:
values
)
{
if
(
value
.
type
().
name
().
equals
(
"NODE"
)){
Node
node
=
value
.
asNode
();
Map
<
String
,
Object
>
stringObjectMap
=
node
.
asMap
();
MetaData
metaData
=
new
MetaData
();
Neo4jTool
.
transMap2Bean
(
stringObjectMap
,
metaData
);
if
(
metaData
.
getMetadataId
()
==
null
){
metaData
.
setMetadataId
(
"System="
+
metaData
.
getName
());
}
ReturnNode
returnNode
=
new
ReturnNode
();
returnNode
.
setName
(
metaData
.
getName
());
returnNode
.
setId
(
metaData
.
getMetadataId
());
returnNode
.
setType
(
metaData
.
getMetaModel
());
returnNode
.
setCnName
(
metaData
.
getCnName
());
returnNodeMap
.
put
(
metaData
.
getMetadataId
(),
returnNode
);
}
}
}
return
returnNodeMap
;
}
}
src/main/java/com/keymobile/metadata/metadataRelation/service/impl/MetadataServiceImpl.java
View file @
7d20204d
...
...
@@ -1857,19 +1857,23 @@ public class MetadataServiceImpl implements IMetadataService {
}
@Override
public
void
syschroTable2EtlJobRelations
(
String
catalogName
)
{
Map
<
String
,
String
>
relationMap
=
new
HashMap
<>();
Map
<
String
,
String
>
edgeIdMap
=
new
HashMap
<>();
int
page
=
0
,
pageSize
=
300
;
long
time0
=
System
.
currentTimeMillis
();
long
totalElement
=
mongoDbServiceImpl
.
countRelation
(
PageRequest
.
of
(
page
,
pageSize
),
catalogName
);
long
time1
=
System
.
currentTimeMillis
();
logger
.
info
(
"从 mmongo获取的关系的总数量是:"
+
totalElement
+
" ,耗时:"
+(
time1
-
time0
)
);
int
totalPageCount
=
((
int
)
totalElement
/
pageSize
)+
1
;
logger
.
info
(
"从 mmongo分页获取关系,每页:"
+
pageSize
);
for
(;
page
<
totalPageCount
;
page
++){
logger
.
info
(
"开始从 mmongo 第"
+
page
+
" 次获取关系。"
);
String
description
=
""
;
List
<
Document
>
relationList
=
mongoDbServiceImpl
.
findRelationByPage
(
PageRequest
.
of
(
page
,
pageSize
,
Sort
.
by
(
"_id"
)),
catalogName
);
long
time3
=
System
.
currentTimeMillis
();
List
<
Document
>
relationList
=
mongoDbServiceImpl
.
findRelationByPage
(
PageRequest
.
of
(
page
,
pageSize
,
Sort
.
by
(
"_id"
)),
catalogName
);
long
time4
=
System
.
currentTimeMillis
();
logger
.
info
(
"从 mmongo 第"
+
page
+
" 次获取关系。 ,耗时:"
+(
time4
-
time3
)
);
for
(
Document
relation
:
relationList
){
description
=
relation
.
getString
(
"description"
);
String
type
=
relation
.
getString
(
"type"
);
...
...
@@ -1973,7 +1977,7 @@ public class MetadataServiceImpl implements IMetadataService {
}
long
end
=
System
.
currentTimeMillis
();
logger
.
info
(
"
创建了多少关系:"
+
edgeIdMap
.
size
()
+
" ,
用时:"
+(
end
-
start
));
logger
.
info
(
"
一共在neo4j,创建了多少关系:"
+
edgeIdMap
.
size
()
+
" ,总
用时:"
+(
end
-
start
));
// List<RelationMongo> relationMongoList = mongoDbServiceImpl.findAllRelationByCatalog(catalogName);
// int size = 0;
...
...
src/main/java/com/keymobile/metadata/metadataRelation/util/SchemaLayerObject.java
View file @
7d20204d
...
...
@@ -18,4 +18,9 @@ public class SchemaLayerObject {
public
static
String
jishicengData
=
"集市数据层"
;
public
static
String
yingyongcengData
=
"应用数据层"
;
//支持的关系类型
public
static
String
table2TableRelations
=
"表-表关系"
;
public
static
String
job2JobRelations
=
"作业-作业关系"
;
public
static
String
script2ScriptRelations
=
"脚本-脚本关系"
;
}
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment