Skip to content
Projects
Groups
Snippets
Help
This project
Loading...
Sign in / Register
Toggle navigation
N
neo4jRelation
Overview
Overview
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
qiuchaofei
neo4jRelation
Commits
5f373286
Commit
5f373286
authored
Jun 15, 2022
by
qiuchaofei
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
1.修改查找子节点的方式,2不删除数据。
parent
f3008c19
Show whitespace changes
Inline
Side-by-side
Showing
7 changed files
with
205 additions
and
59 deletions
+205
-59
Neo4jSchemaRepository.java
...aRelation/respository/metadata/Neo4jSchemaRepository.java
+2
-0
AsyncDataFromMongoToNeo4j.java
...a/metadataRelation/service/AsyncDataFromMongoToNeo4j.java
+2
-2
ISchemaService.java
...ile/metadata/metadataRelation/service/ISchemaService.java
+5
-0
DataRelationAnalyServiceImpl.java
...taRelation/service/impl/DataRelationAnalyServiceImpl.java
+4
-1
MetadataServiceImpl.java
...ta/metadataRelation/service/impl/MetadataServiceImpl.java
+113
-36
SchemaServiceImpl.java
...data/metadataRelation/service/impl/SchemaServiceImpl.java
+69
-10
SystemServiceImpl.java
...data/metadataRelation/service/impl/SystemServiceImpl.java
+10
-10
No files found.
src/main/java/com/keymobile/metadata/metadataRelation/respository/metadata/Neo4jSchemaRepository.java
View file @
5f373286
...
@@ -21,6 +21,8 @@ public interface Neo4jSchemaRepository extends Neo4jRepository<Neo4jSchema, Long
...
@@ -21,6 +21,8 @@ public interface Neo4jSchemaRepository extends Neo4jRepository<Neo4jSchema, Long
@Query
(
"match (n:Neo4jSchema{metadataId:{schemaId}} )-[r:Composition]->(m:neo4jTable) return m order by m.name "
)
@Query
(
"match (n:Neo4jSchema{metadataId:{schemaId}} )-[r:Composition]->(m:neo4jTable) return m order by m.name "
)
List
<
BaseNode
>
getTableBySchemaId
(
@Param
(
"schemaId"
)
String
schemaId
);
List
<
BaseNode
>
getTableBySchemaId
(
@Param
(
"schemaId"
)
String
schemaId
);
@Query
(
"match (n:Neo4jSchema{metadataId:{schemaId}} )-[r:Composition]->(m:neo4jTable) return m order by m.name "
)
List
<
BaseNode
>
getColumnBySchemaId
(
@Param
(
"schemaId"
)
String
schemaId
);
@Query
(
"match (n:Neo4jSchema{metadataId:{schemaId}} )-[r:Composition]->(m:neo4jView) return m order by m.name "
)
@Query
(
"match (n:Neo4jSchema{metadataId:{schemaId}} )-[r:Composition]->(m:neo4jView) return m order by m.name "
)
List
<
BaseNode
>
getViewBySchemaId
(
@Param
(
"schemaId"
)
String
schemaId
);
List
<
BaseNode
>
getViewBySchemaId
(
@Param
(
"schemaId"
)
String
schemaId
);
...
...
src/main/java/com/keymobile/metadata/metadataRelation/service/AsyncDataFromMongoToNeo4j.java
View file @
5f373286
...
@@ -71,10 +71,10 @@ public class AsyncDataFromMongoToNeo4j {
...
@@ -71,10 +71,10 @@ public class AsyncDataFromMongoToNeo4j {
});
});
//先删除同个环境的数据
//先删除同个环境的数据
long
time0
=
System
.
currentTimeMillis
();
long
time0
=
System
.
currentTimeMillis
();
metadataService
.
deleteNodeByCatalogName
(
catalogName
);
//
metadataService.deleteNodeByCatalogName(catalogName);
long
time1
=
System
.
currentTimeMillis
();
long
time1
=
System
.
currentTimeMillis
();
logger
.
info
(
"删除"
+
catalogName
+
"的数据完成,耗时:"
+(
time1
-
time0
));
//
logger.info("删除"+catalogName+"的数据完成,耗时:"+(time1-time0));
//先同步元数据,纵向关系,root开始,系统--schema--table--column,系统--etl作业--etl脚本--etlsql。
//先同步元数据,纵向关系,root开始,系统--schema--table--column,系统--etl作业--etl脚本--etlsql。
metadataService
.
syschroMetadataFromMongo
(
catalogName
,
mongoDataList
);
metadataService
.
syschroMetadataFromMongo
(
catalogName
,
mongoDataList
);
...
...
src/main/java/com/keymobile/metadata/metadataRelation/service/ISchemaService.java
View file @
5f373286
...
@@ -18,4 +18,9 @@ public interface ISchemaService {
...
@@ -18,4 +18,9 @@ public interface ISchemaService {
boolean
updateTagFromSchema
(
String
schemaId
);
boolean
updateTagFromSchema
(
String
schemaId
);
boolean
updateTagFromAllSchema
();
boolean
updateTagFromAllSchema
();
Map
<
String
,
String
>
getTableIdFromSchemaId
(
String
schemaId
);
Map
<
String
,
String
>
getColumnIdFromSchemaId
(
String
schemaId
);
}
}
src/main/java/com/keymobile/metadata/metadataRelation/service/impl/DataRelationAnalyServiceImpl.java
View file @
5f373286
...
@@ -4,7 +4,10 @@ import com.keymobile.metadata.metadataRelation.pojo.MetaData;
...
@@ -4,7 +4,10 @@ import com.keymobile.metadata.metadataRelation.pojo.MetaData;
import
com.keymobile.metadata.metadataRelation.pojo.returnBean.ReturnNode
;
import
com.keymobile.metadata.metadataRelation.pojo.returnBean.ReturnNode
;
import
com.keymobile.metadata.metadataRelation.service.DataRelationAnalyService
;
import
com.keymobile.metadata.metadataRelation.service.DataRelationAnalyService
;
import
com.keymobile.metadata.metadataRelation.util.Neo4jTool
;
import
com.keymobile.metadata.metadataRelation.util.Neo4jTool
;
import
org.neo4j.driver.v1.*
;
import
org.neo4j.driver.v1.Record
;
import
org.neo4j.driver.v1.Session
;
import
org.neo4j.driver.v1.StatementResult
;
import
org.neo4j.driver.v1.Value
;
import
org.neo4j.driver.v1.types.Node
;
import
org.neo4j.driver.v1.types.Node
;
import
org.slf4j.Logger
;
import
org.slf4j.Logger
;
import
org.slf4j.LoggerFactory
;
import
org.slf4j.LoggerFactory
;
...
...
src/main/java/com/keymobile/metadata/metadataRelation/service/impl/MetadataServiceImpl.java
View file @
5f373286
...
@@ -17,10 +17,7 @@ import com.keymobile.metadata.metadataRelation.pojo.returnBean.ReturnReslult;
...
@@ -17,10 +17,7 @@ import com.keymobile.metadata.metadataRelation.pojo.returnBean.ReturnReslult;
import
com.keymobile.metadata.metadataRelation.remote.MetadataRepoRemoteService
;
import
com.keymobile.metadata.metadataRelation.remote.MetadataRepoRemoteService
;
import
com.keymobile.metadata.metadataRelation.respository.*
;
import
com.keymobile.metadata.metadataRelation.respository.*
;
import
com.keymobile.metadata.metadataRelation.respository.metadata.*
;
import
com.keymobile.metadata.metadataRelation.respository.metadata.*
;
import
com.keymobile.metadata.metadataRelation.service.IBaseRelationshipService
;
import
com.keymobile.metadata.metadataRelation.service.*
;
import
com.keymobile.metadata.metadataRelation.service.IMetadataService
;
import
com.keymobile.metadata.metadataRelation.service.ISchemaService
;
import
com.keymobile.metadata.metadataRelation.service.ITableService
;
import
com.keymobile.metadata.metadataRelation.util.Neo4jTool
;
import
com.keymobile.metadata.metadataRelation.util.Neo4jTool
;
import
com.keymobile.metadata.metadataRelation.util.SchemaLayerObject
;
import
com.keymobile.metadata.metadataRelation.util.SchemaLayerObject
;
import
com.mongodb.BasicDBObject
;
import
com.mongodb.BasicDBObject
;
...
@@ -38,6 +35,9 @@ import org.springframework.boot.system.ApplicationHome;
...
@@ -38,6 +35,9 @@ import org.springframework.boot.system.ApplicationHome;
import
org.springframework.data.domain.PageRequest
;
import
org.springframework.data.domain.PageRequest
;
import
org.springframework.data.domain.Sort
;
import
org.springframework.data.domain.Sort
;
import
org.springframework.data.mongodb.core.MongoTemplate
;
import
org.springframework.data.mongodb.core.MongoTemplate
;
import
org.springframework.data.mongodb.core.query.Criteria
;
import
org.springframework.data.mongodb.core.query.Meta
;
import
org.springframework.data.mongodb.core.query.Query
;
import
org.springframework.stereotype.Service
;
import
org.springframework.stereotype.Service
;
import
org.springframework.util.CollectionUtils
;
import
org.springframework.util.CollectionUtils
;
...
@@ -111,6 +111,8 @@ public class MetadataServiceImpl implements IMetadataService {
...
@@ -111,6 +111,8 @@ public class MetadataServiceImpl implements IMetadataService {
@Autowired
@Autowired
private
ISchemaService
schemaService
;
private
ISchemaService
schemaService
;
@Autowired
private
ISystemService
systemService
;
private
String
PREFIX_METADATA_NODE
=
"Metadata_"
;
private
String
PREFIX_METADATA_NODE
=
"Metadata_"
;
@Override
@Override
...
@@ -1226,12 +1228,7 @@ public class MetadataServiceImpl implements IMetadataService {
...
@@ -1226,12 +1228,7 @@ public class MetadataServiceImpl implements IMetadataService {
}
}
}
}
long
columnTime0
=
System
.
currentTimeMillis
();
Map
<
String
,
Neo4jTable
>
neo4jTableIdMap
=
new
HashMap
<>();
saveAllTableByBatch
(
catalogName
,
compositionRelationList
,
neo4jSchemaMap
,
neo4jTableIdMap
);
long
columnTime1
=
System
.
currentTimeMillis
();
//专门做字段解析,批量获取字段,进行组装。
saveAllColumnByBatch
(
catalogName
,
compositionRelationList
,
neo4jTableIdMap
);
logger
.
info
(
"创建的system数量:"
+
systemDataMap
.
size
());
logger
.
info
(
"创建的system数量:"
+
systemDataMap
.
size
());
logger
.
info
(
"创建的schema:"
+(
objectCountMap
.
get
(
SchemaCountString
)-
1
));
logger
.
info
(
"创建的schema:"
+(
objectCountMap
.
get
(
SchemaCountString
)-
1
));
...
@@ -1259,7 +1256,12 @@ public class MetadataServiceImpl implements IMetadataService {
...
@@ -1259,7 +1256,12 @@ public class MetadataServiceImpl implements IMetadataService {
}
}
private
void
saveAllTableByBatch
(
String
catalogName
,
List
<
CompositionRelation
>
compositionRelationList
,
private
void
saveAllTableByBatch
(
String
catalogName
,
List
<
CompositionRelation
>
compositionRelationList
,
Map
<
String
,
Neo4jSchema
>
neo4jSchemaMap
,
Map
<
String
,
Neo4jTable
>
neo4jTableIdMap
)
{
Map
<
String
,
Neo4jSchema
>
neo4jSchemaMap
,
Map
<
String
,
Neo4jTable
>
neo4jTableIdMap
,
String
schemaId
)
{
long
time1
=
System
.
currentTimeMillis
();
Map
<
String
,
String
>
tableIdMap
=
schemaService
.
getTableIdFromSchemaId
(
schemaId
);
long
time2
=
System
.
currentTimeMillis
();
logger
.
info
(
"从neo4j 获取 schema:"
+
schemaId
+
" 的表个数:"
+
tableIdMap
.
size
()
+
" , 耗时:"
+(
time2
-
time1
));
List
<
Neo4jTable
>
neo4jTableList
=
new
ArrayList
<>();
List
<
Neo4jTable
>
neo4jTableList
=
new
ArrayList
<>();
int
tableCount
=
1
;
int
tableCount
=
1
;
...
@@ -1269,14 +1271,21 @@ public class MetadataServiceImpl implements IMetadataService {
...
@@ -1269,14 +1271,21 @@ public class MetadataServiceImpl implements IMetadataService {
String
tablePrefix
=
"Catalog,Database,Schema,Table"
;
String
tablePrefix
=
"Catalog,Database,Schema,Table"
;
BasicDBObject
query
=
new
BasicDBObject
();
BasicDBObject
query
=
new
BasicDBObject
();
query
.
put
(
"_class"
,
tablePrefix
);
query
.
put
(
"_class"
,
tablePrefix
);
query
.
put
(
"parentId"
,
schemaId
);
MongoCursor
<
Document
>
tableCursor
=
this
.
mongoTemplate
.
getCollection
(
PREFIX_METADATA_NODE
+
catalogName
).
MongoCursor
<
Document
>
tableCursor
=
this
.
mongoTemplate
.
getCollection
(
PREFIX_METADATA_NODE
+
catalogName
).
find
(
query
).
noCursorTimeout
(
true
).
batchSize
(
batchSize
).
iterator
();
find
(
query
).
noCursorTimeout
(
true
).
batchSize
(
batchSize
).
iterator
();
while
(
tableCursor
.
hasNext
())
{
while
(
tableCursor
.
hasNext
())
{
Document
tableDocument
=
tableCursor
.
next
();
Document
tableDocument
=
tableCursor
.
next
();
String
tableId
=(
String
)
tableDocument
.
get
(
"_id"
);
String
tableId
=(
String
)
tableDocument
.
get
(
"_id"
);
//存在了,就不写入
if
(
tableIdMap
.
containsKey
(
tableId
)){
continue
;
}
String
tableName
=(
String
)
tableDocument
.
get
(
"name"
);
String
tableName
=(
String
)
tableDocument
.
get
(
"name"
);
String
tableCnName
=(
String
)
tableDocument
.
get
(
"cnName"
);
String
tableCnName
=(
String
)
tableDocument
.
get
(
"cnName"
);
String
schemaId
=(
String
)
tableDocument
.
get
(
"parentId"
);
//
String schemaId =(String) tableDocument.get("parentId");
Neo4jSchema
neo4jSchema
=
neo4jSchemaMap
.
get
(
schemaId
);
Neo4jSchema
neo4jSchema
=
neo4jSchemaMap
.
get
(
schemaId
);
...
@@ -1298,6 +1307,7 @@ public class MetadataServiceImpl implements IMetadataService {
...
@@ -1298,6 +1307,7 @@ public class MetadataServiceImpl implements IMetadataService {
neo4jTable
.
setComment
((
String
)
tableDocument
.
get
(
"comment"
));
neo4jTable
.
setComment
((
String
)
tableDocument
.
get
(
"comment"
));
neo4jTable
.
setCnName
((
String
)
tableDocument
.
get
(
CNNameString
));
neo4jTable
.
setCnName
((
String
)
tableDocument
.
get
(
CNNameString
));
neo4jTable
.
setTableCount
(
""
+
tableDocument
.
get
(
"tableRowsCount"
));
neo4jTable
.
setTableCount
(
""
+
tableDocument
.
get
(
"tableRowsCount"
));
neo4jTableList
.
add
(
neo4jTable
);
neo4jTableList
.
add
(
neo4jTable
);
CompositionRelation
schem2Table
=
new
CompositionRelation
();
CompositionRelation
schem2Table
=
new
CompositionRelation
();
schem2Table
.
setStart
(
neo4jSchema
);
schem2Table
.
setStart
(
neo4jSchema
);
...
@@ -1323,7 +1333,7 @@ public class MetadataServiceImpl implements IMetadataService {
...
@@ -1323,7 +1333,7 @@ public class MetadataServiceImpl implements IMetadataService {
neo4jTableList
.
clear
();
neo4jTableList
.
clear
();
}
}
long
columnTime1
=
System
.
currentTimeMillis
();
long
columnTime1
=
System
.
currentTimeMillis
();
logger
.
info
(
"创建的table数量:"
+
tableCount
+
" 用时:"
+(
columnTime1
-
columnTime0
));
logger
.
info
(
"
新
创建的table数量:"
+
tableCount
+
" 用时:"
+(
columnTime1
-
columnTime0
));
}
}
private
void
saveDataBaseDataToNeo4j
(
MongoData
dataBaseData
,
String
catalogName
,
Map
<
String
,
Neo4jSystem
>
systemDataMap
,
private
void
saveDataBaseDataToNeo4j
(
MongoData
dataBaseData
,
String
catalogName
,
Map
<
String
,
Neo4jSystem
>
systemDataMap
,
...
@@ -1338,6 +1348,7 @@ public class MetadataServiceImpl implements IMetadataService {
...
@@ -1338,6 +1348,7 @@ public class MetadataServiceImpl implements IMetadataService {
List
<
String
>
schemaIds
=
getSchemaIds
(
schemaDataList
);
List
<
String
>
schemaIds
=
getSchemaIds
(
schemaDataList
);
Map
<
String
,
String
>
shemaIdAndLabel
=
schemaService
.
getSchemaAndLabelMap
(
schemaIds
);
Map
<
String
,
String
>
shemaIdAndLabel
=
schemaService
.
getSchemaAndLabelMap
(
schemaIds
);
Map
<
String
,
ReturnNode
>
schemaMap
=
schemaService
.
getSchemaBySystemId
(
neo4jSystem
.
getMetadataId
());
for
(
MongoData
schemaData:
schemaDataList
){
for
(
MongoData
schemaData:
schemaDataList
){
//系统与schema的关系
//系统与schema的关系
...
@@ -1354,21 +1365,32 @@ public class MetadataServiceImpl implements IMetadataService {
...
@@ -1354,21 +1365,32 @@ public class MetadataServiceImpl implements IMetadataService {
neo4jSchema
.
setDataPath
(
schemaPath
);
neo4jSchema
.
setDataPath
(
schemaPath
);
//如果存在,就不用创建
//如果存在,就不用创建
Neo4jSchema
neo4jSchemaList
=
neo4jSchemaRepository
.
findNeo4jSchemaByMetadataId
(
neo4jSchema
.
getMetadataId
());
if
(!
schemaMap
.
containsKey
(
neo4jSchema
.
getMetadataId
())){
if
(
neo4jSchemaList
==
null
){
neo4jSchemaRepository
.
save
(
neo4jSchema
);
// neo4jSchemaRepository.save(neo4jSchema);
schemaCount
++;
schemaCount
++;
}
CompositionRelation
compositionRelation
=
new
CompositionRelation
();
CompositionRelation
compositionRelation
=
new
CompositionRelation
();
compositionRelation
.
setStart
(
neo4jSystem
);
compositionRelation
.
setStart
(
neo4jSystem
);
compositionRelation
.
setEnd
(
neo4jSchema
);
compositionRelation
.
setEnd
(
neo4jSchema
);
compositionRelation
.
setName
(
"System--Schema"
);
compositionRelation
.
setName
(
"System--Schema"
);
compositionRelationList
.
add
(
compositionRelation
);
compositionRelationList
.
add
(
compositionRelation
);
}
// relationshipService.saveRelation(neo4jSystem.getMetadataId(),neo4jSchema.getMetadataId(),"Composition");
// relationshipService.saveRelation(neo4jSystem.getMetadataId(),neo4jSchema.getMetadataId(),"Composition");
neo4jSchema
.
setDataPath
(
schemaPath
);
neo4jSchema
.
setDataPath
(
schemaPath
);
neo4jSchemaMap
.
put
(
schemaData_id
,
neo4jSchema
);
neo4jSchemaMap
.
put
(
schemaData_id
,
neo4jSchema
);
long
columnTime0
=
System
.
currentTimeMillis
();
Map
<
String
,
Neo4jTable
>
neo4jTableIdMap
=
new
HashMap
<>();
saveAllTableByBatch
(
catalogName
,
compositionRelationList
,
neo4jSchemaMap
,
neo4jTableIdMap
,
neo4jSchema
.
getMetadataId
());
long
columnTime1
=
System
.
currentTimeMillis
();
//专门做字段解析,批量获取字段,进行组装。
saveAllColumnByBatch
(
catalogName
,
compositionRelationList
,
neo4jTableIdMap
,
neo4jSchema
.
getMetadataId
());
// Neo4jSchema neo4jSchemaList = neo4jSchemaRepository.findNeo4jSchemaByMetadataId(neo4jSchema.getMetadataId());
// if(neo4jSchemaList==null ){
//// neo4jSchemaRepository.save(neo4jSchema);
// schemaCount++;
// }
//根据schema获取表级数据,再按照类型分类:表,视图,作业,函数等
//根据schema获取表级数据,再按照类型分类:表,视图,作业,函数等
// int page =0,pageSize = 500;
// int page =0,pageSize = 500;
...
@@ -1383,26 +1405,56 @@ public class MetadataServiceImpl implements IMetadataService {
...
@@ -1383,26 +1405,56 @@ public class MetadataServiceImpl implements IMetadataService {
objectCountMap
.
put
(
SchemaCountString
,
schemaCount
);
objectCountMap
.
put
(
SchemaCountString
,
schemaCount
);
}
}
private
void
saveAllColumnByBatch
(
String
catalogName
,
List
<
CompositionRelation
>
compositionRelationList
,
private
void
saveAllColumnByBatch
(
String
catalogName
,
List
<
CompositionRelation
>
compositionRelationList
,
Map
<
String
,
Neo4jTable
>
neo4jTableIdMap
)
{
Map
<
String
,
Neo4jTable
>
neo4jTableIdMap
,
String
schemaId
)
{
long
columnTime1
=
System
.
currentTimeMillis
();
long
columnTime1
=
System
.
currentTimeMillis
();
int
columnCount
=
0
;
// objectCountMap.get(ColumnCountString);
int
columnCount
=
0
;
// objectCountMap.get(ColumnCountString);
String
ColumnPrefix
=
"Catalog,Database,Schema,Table,Column"
;
String
ColumnPrefix
=
"Catalog,Database,Schema,Table,Column"
;
int
batchSize
=
5000
;
int
batchSize
=
5000
;
BasicDBObject
query
=
new
BasicDBObject
();
query
.
put
(
"_class"
,
ColumnPrefix
);
Query
query
=
new
Query
();
MongoCursor
<
Document
>
columnCursor
=
this
.
mongoTemplate
.
getCollection
(
PREFIX_METADATA_NODE
+
catalogName
).
Criteria
criteria
=
Criteria
.
where
(
"_class"
).
in
(
ColumnPrefix
);
find
(
query
).
noCursorTimeout
(
true
).
batchSize
(
batchSize
).
iterator
();
criteria
.
and
(
"idPath"
).
regex
(
"^.*"
+
schemaId
+
".*$"
);
// BasicDBObject query = new BasicDBObject();;
query
.
addCriteria
(
criteria
);
MongoCursor
<
Document
>
columnCursor
=
mongoTemplate
.
getCollection
(
PREFIX_METADATA_NODE
+
catalogName
)
.
find
(
query
.
getQueryObject
())
.
noCursorTimeout
(
true
)
.
batchSize
(
batchSize
)
.
iterator
();
// BasicDBObject query = new BasicDBObject();
// query.put("_class", ColumnPrefix);
// MongoCursor<Document> columnCursor = this.mongoTemplate.getCollection(PREFIX_METADATA_NODE + catalogName).
// find(query).noCursorTimeout(true).batchSize(batchSize).iterator();
List
<
Neo4jColumn
>
neo4jColumnList
=
new
ArrayList
<>();
List
<
Neo4jColumn
>
neo4jColumnList
=
new
ArrayList
<>();
long
time1
=
System
.
currentTimeMillis
();
Map
<
String
,
String
>
columnFromSchemaId
=
schemaService
.
getColumnIdFromSchemaId
(
schemaId
);
long
time2
=
System
.
currentTimeMillis
();
logger
.
info
(
"从neo4j 获取 schema:"
+
schemaId
+
" 的字段个数:"
+
columnFromSchemaId
.
size
()
+
" , 耗时:"
+(
time2
-
time1
));
while
(
columnCursor
.
hasNext
())
{
while
(
columnCursor
.
hasNext
())
{
Document
columnDocument
=
columnCursor
.
next
();
Document
columnDocument
=
columnCursor
.
next
();
String
columnId
=(
String
)
columnDocument
.
get
(
"_id"
);
String
columnId
=(
String
)
columnDocument
.
get
(
"_id"
);
if
(
columnFromSchemaId
.
containsKey
(
columnId
)){
continue
;
}
String
metadataName
=(
String
)
columnDocument
.
get
(
"name"
);
String
metadataName
=(
String
)
columnDocument
.
get
(
"name"
);
String
metadataCnName
=(
String
)
columnDocument
.
get
(
"cnName"
);
String
metadataCnName
=(
String
)
columnDocument
.
get
(
"cnName"
);
String
tableId
=
(
String
)
columnDocument
.
get
(
"parentId"
);
String
tableId
=
(
String
)
columnDocument
.
get
(
"parentId"
);
//
//
Neo4jTable
neo4jTable
=
neo4jTableIdMap
.
get
(
tableId
);
if
(
neo4jTable
==
null
){
continue
;
}
Neo4jColumn
neo4jColumn
=
new
Neo4jColumn
();
Neo4jColumn
neo4jColumn
=
new
Neo4jColumn
();
neo4jColumn
.
setMetadataId
(
columnId
);
neo4jColumn
.
setMetadataId
(
columnId
);
neo4jColumn
.
setName
(
metadataName
);
neo4jColumn
.
setName
(
metadataName
);
...
@@ -1410,10 +1462,6 @@ public class MetadataServiceImpl implements IMetadataService {
...
@@ -1410,10 +1462,6 @@ public class MetadataServiceImpl implements IMetadataService {
neo4jColumnList
.
add
(
neo4jColumn
);
neo4jColumnList
.
add
(
neo4jColumn
);
CompositionRelation
table2Column
=
new
CompositionRelation
();
CompositionRelation
table2Column
=
new
CompositionRelation
();
Neo4jTable
neo4jTable
=
neo4jTableIdMap
.
get
(
tableId
);
if
(
neo4jTable
==
null
){
continue
;
}
table2Column
.
setStart
(
neo4jTable
);
table2Column
.
setStart
(
neo4jTable
);
table2Column
.
setEnd
(
neo4jColumn
);
table2Column
.
setEnd
(
neo4jColumn
);
table2Column
.
setName
(
"Table--Column"
);
table2Column
.
setName
(
"Table--Column"
);
...
@@ -1430,7 +1478,7 @@ public class MetadataServiceImpl implements IMetadataService {
...
@@ -1430,7 +1478,7 @@ public class MetadataServiceImpl implements IMetadataService {
neo4jColumnList
.
clear
();
neo4jColumnList
.
clear
();
}
}
long
columnTime2
=
System
.
currentTimeMillis
();
long
columnTime2
=
System
.
currentTimeMillis
();
logger
.
info
(
"
创建
的column数量:"
+
columnCount
+
",用时:"
+(
columnTime2
-
columnTime1
));
logger
.
info
(
"
更新
的column数量:"
+
columnCount
+
",用时:"
+(
columnTime2
-
columnTime1
));
}
}
private
void
saveTableDocument
(
List
<
Document
>
tableDocumentList
,
Neo4jSchema
neo4jSchema
,
Map
<
String
,
Integer
>
objectCountMap
,
private
void
saveTableDocument
(
List
<
Document
>
tableDocumentList
,
Neo4jSchema
neo4jSchema
,
Map
<
String
,
Integer
>
objectCountMap
,
...
@@ -1852,6 +1900,9 @@ public class MetadataServiceImpl implements IMetadataService {
...
@@ -1852,6 +1900,9 @@ public class MetadataServiceImpl implements IMetadataService {
private
Map
<
String
,
Neo4jSystem
>
getSystemDataMap
(
List
<
MongoData
>
systemDataList
,
String
catalogName
)
{
private
Map
<
String
,
Neo4jSystem
>
getSystemDataMap
(
List
<
MongoData
>
systemDataList
,
String
catalogName
)
{
Map
<
String
,
Neo4jSystem
>
systemDataMap
=
new
HashMap
<>();
Map
<
String
,
Neo4jSystem
>
systemDataMap
=
new
HashMap
<>();
Map
<
String
,
ReturnNode
>
systemMap
=
systemService
.
getSystemByEnvId
(
catalogName
);
for
(
MongoData
systemData:
systemDataList
){
for
(
MongoData
systemData:
systemDataList
){
Neo4jSystem
neo4jSystem
=
new
Neo4jSystem
();
Neo4jSystem
neo4jSystem
=
new
Neo4jSystem
();
String
systemId
=
SystemModelEqual
+
catalogName
+
"="
+
systemData
.
get_id
();
String
systemId
=
SystemModelEqual
+
catalogName
+
"="
+
systemData
.
get_id
();
...
@@ -1861,8 +1912,7 @@ public class MetadataServiceImpl implements IMetadataService {
...
@@ -1861,8 +1912,7 @@ public class MetadataServiceImpl implements IMetadataService {
String
systemPath
=
catalogName
+
";"
+
systemData
.
getName
();
String
systemPath
=
catalogName
+
";"
+
systemData
.
getName
();
neo4jSystem
.
setDataPath
(
systemPath
);
neo4jSystem
.
setDataPath
(
systemPath
);
//用名称查找,如果存在,就不用创建
//用名称查找,如果存在,就不用创建
Neo4jSystem
neo4jSystemList
=
neo4jSystemRepository
.
findNeo4jSystemByMetadataId
(
neo4jSystem
.
getMetadataId
());
if
(!
systemMap
.
containsKey
(
systemId
)){
if
(
neo4jSystemList
==
null
){
neo4jSystemRepository
.
save
(
neo4jSystem
);
neo4jSystemRepository
.
save
(
neo4jSystem
);
}
}
systemDataMap
.
put
(
systemData
.
get_id
(),
neo4jSystem
);
systemDataMap
.
put
(
systemData
.
get_id
(),
neo4jSystem
);
...
@@ -2012,7 +2062,7 @@ public class MetadataServiceImpl implements IMetadataService {
...
@@ -2012,7 +2062,7 @@ public class MetadataServiceImpl implements IMetadataService {
if
(
baseNodeMap
.
get
(
targetId
)!=
null
){
if
(
baseNodeMap
.
get
(
targetId
)!=
null
){
endNode
=
baseNodeMap
.
get
(
targetId
);
endNode
=
baseNodeMap
.
get
(
targetId
);
}
else
{
}
else
{
endNode
=
getNodeByMetadataId
(
targetId
);
//neo4jColumnRepository.findNeo4jColumnByMetadataId(sourceId);
endNode
=
getNodeByMetadataId
(
targetId
);
baseNodeMap
.
put
(
targetId
,
endNode
);
baseNodeMap
.
put
(
targetId
,
endNode
);
}
}
if
(
startNode
==
null
||
endNode
==
null
){
if
(
startNode
==
null
||
endNode
==
null
){
...
@@ -2079,6 +2129,7 @@ public class MetadataServiceImpl implements IMetadataService {
...
@@ -2079,6 +2129,7 @@ public class MetadataServiceImpl implements IMetadataService {
Map
<
String
,
DependencyRelation
>
dependencyRelationMap
=
new
HashMap
<>();
Map
<
String
,
DependencyRelation
>
dependencyRelationMap
=
new
HashMap
<>();
//保存已经查过的节点
Map
<
String
,
BaseNode
>
baseNodeMap
=
new
HashMap
<>();
Map
<
String
,
BaseNode
>
baseNodeMap
=
new
HashMap
<>();
int
page
=
0
,
pageSize
=
300
;
int
page
=
0
,
pageSize
=
300
;
...
@@ -2131,7 +2182,7 @@ public class MetadataServiceImpl implements IMetadataService {
...
@@ -2131,7 +2182,7 @@ public class MetadataServiceImpl implements IMetadataService {
if
(
baseNodeMap
.
get
(
targetId
)!=
null
){
if
(
baseNodeMap
.
get
(
targetId
)!=
null
){
endNode
=
baseNodeMap
.
get
(
targetId
);
endNode
=
baseNodeMap
.
get
(
targetId
);
}
else
{
}
else
{
endNode
=
getNodeByMetadataId
(
targetId
);
//neo4jColumnRepository.findNeo4jColumnByMetadataId(sourceId);
endNode
=
getNodeByMetadataId
(
targetId
);
baseNodeMap
.
put
(
targetId
,
endNode
);
baseNodeMap
.
put
(
targetId
,
endNode
);
}
}
if
(
startNode
==
null
||
endNode
==
null
){
if
(
startNode
==
null
||
endNode
==
null
){
...
@@ -2237,7 +2288,9 @@ public class MetadataServiceImpl implements IMetadataService {
...
@@ -2237,7 +2288,9 @@ public class MetadataServiceImpl implements IMetadataService {
}
}
private
BaseNode
getNodeByMetadataId
(
String
metadataId
)
{
private
BaseNode
getNodeByMetadataId
(
String
metadataId
)
{
BaseNode
baseNode
=
null
;
BaseNode
baseNode
=
new
BaseNode
();
try
{
long
time1
=
System
.
currentTimeMillis
();
if
(
metadataId
.
startsWith
(
ColumnModelQueal
)){
if
(
metadataId
.
startsWith
(
ColumnModelQueal
)){
baseNode
=
neo4jColumnRepository
.
findNeo4jColumnByMetadataId
(
metadataId
);
baseNode
=
neo4jColumnRepository
.
findNeo4jColumnByMetadataId
(
metadataId
);
}
else
if
(
metadataId
.
startsWith
(
TableModelEqual
)){
}
else
if
(
metadataId
.
startsWith
(
TableModelEqual
)){
...
@@ -2247,6 +2300,30 @@ public class MetadataServiceImpl implements IMetadataService {
...
@@ -2247,6 +2300,30 @@ public class MetadataServiceImpl implements IMetadataService {
}
else
if
(
metadataId
.
startsWith
(
EtlSqlModelQueal
)){
}
else
if
(
metadataId
.
startsWith
(
EtlSqlModelQueal
)){
baseNode
=
neo4jETLSqlRepository
.
findNeo4jETLSqlByMetadataId
(
metadataId
);
baseNode
=
neo4jETLSqlRepository
.
findNeo4jETLSqlByMetadataId
(
metadataId
);
}
}
long
time2
=
System
.
currentTimeMillis
();
logger
.
info
(
"查询:"
+
metadataId
+
" ,用时:"
+
(
time2
-
time1
));
}
catch
(
Exception
e
){
logger
.
info
(
"采用cypher查询。"
);
String
cypher
=
"match(n{metadataId:\""
+
metadataId
+
"\"}) return n limit 1 "
;
StatementResult
statementResult
=
session
.
run
(
cypher
);
while
(
statementResult
.
hasNext
()){
Record
record
=
statementResult
.
next
();
List
<
Value
>
values
=
record
.
values
();
for
(
Value
value
:
values
)
{
if
(
value
.
type
().
name
().
equals
(
"NODE"
)){
Node
node
=
value
.
asNode
();
Map
<
String
,
Object
>
stringObjectMap
=
node
.
asMap
();
// MetaData metaData = new MetaData();
Neo4jTool
.
transMap2Bean
(
stringObjectMap
,
baseNode
);
// if(metaData.getMetadataId() == null ){
// metaData.setMetadataId(SystemModelEqual+metaData.getName());
// }
// metaDataList.add(metaData);
}
}
}
}
return
baseNode
;
return
baseNode
;
...
@@ -2281,13 +2358,13 @@ public class MetadataServiceImpl implements IMetadataService {
...
@@ -2281,13 +2358,13 @@ public class MetadataServiceImpl implements IMetadataService {
if
(
baseNodeMap
.
get
(
startId
)!=
null
){
if
(
baseNodeMap
.
get
(
startId
)!=
null
){
startNode
=
baseNodeMap
.
get
(
startId
);
startNode
=
baseNodeMap
.
get
(
startId
);
}
else
{
}
else
{
startNode
=
getNodeByMetadataId
(
startId
)
;
//neo4jColumnRepository.findNeo4jColumnByMetadataId(startId);
startNode
=
getNodeByMetadataId
(
startId
)
;
baseNodeMap
.
put
(
startId
,
startNode
);
baseNodeMap
.
put
(
startId
,
startNode
);
}
}
if
(
baseNodeMap
.
get
(
endId
)!=
null
){
if
(
baseNodeMap
.
get
(
endId
)!=
null
){
endNode
=
baseNodeMap
.
get
(
endId
);
endNode
=
baseNodeMap
.
get
(
endId
);
}
else
{
}
else
{
endNode
=
getNodeByMetadataId
(
endId
);
//neo4jColumnRepository.findNeo4jColumnByMetadataId(endId);
endNode
=
getNodeByMetadataId
(
endId
);
baseNodeMap
.
put
(
endId
,
endNode
);
baseNodeMap
.
put
(
endId
,
endNode
);
}
}
if
(
startNode
!=
null
&&
endNode
!=
null
){
if
(
startNode
!=
null
&&
endNode
!=
null
){
...
...
src/main/java/com/keymobile/metadata/metadataRelation/service/impl/SchemaServiceImpl.java
View file @
5f373286
...
@@ -10,6 +10,9 @@ import com.keymobile.metadata.metadataRelation.respository.metadata.Neo4jSystemR
...
@@ -10,6 +10,9 @@ import com.keymobile.metadata.metadataRelation.respository.metadata.Neo4jSystemR
import
com.keymobile.metadata.metadataRelation.service.ISchemaService
;
import
com.keymobile.metadata.metadataRelation.service.ISchemaService
;
import
com.keymobile.metadata.metadataRelation.util.SchemaLayerObject
;
import
com.keymobile.metadata.metadataRelation.util.SchemaLayerObject
;
import
org.hibernate.loader.custom.Return
;
import
org.hibernate.loader.custom.Return
;
import
org.neo4j.driver.v1.Record
;
import
org.neo4j.driver.v1.Session
;
import
org.neo4j.driver.v1.StatementResult
;
import
org.slf4j.Logger
;
import
org.slf4j.Logger
;
import
org.slf4j.LoggerFactory
;
import
org.slf4j.LoggerFactory
;
import
org.springframework.beans.factory.annotation.Autowired
;
import
org.springframework.beans.factory.annotation.Autowired
;
...
@@ -27,6 +30,8 @@ public class SchemaServiceImpl implements ISchemaService {
...
@@ -27,6 +30,8 @@ public class SchemaServiceImpl implements ISchemaService {
@Autowired
@Autowired
private
TagRemoteService
tagRemoteService
;
private
TagRemoteService
tagRemoteService
;
@Autowired
private
Session
session
;
@Autowired
@Autowired
private
Neo4jSystemRepository
neo4jSystemRepository
;
private
Neo4jSystemRepository
neo4jSystemRepository
;
...
@@ -49,11 +54,11 @@ public class SchemaServiceImpl implements ISchemaService {
...
@@ -49,11 +54,11 @@ public class SchemaServiceImpl implements ISchemaService {
* 26 });
* 26 });
*/
*/
List
<
Neo4jSchema
>
neo4jSchemas
=
neo4jSystemRepository
.
getSchemaBySystemId
(
systemId
);
List
<
Neo4jSchema
>
neo4jSchemas
=
neo4jSystemRepository
.
getSchemaBySystemId
(
systemId
);
System
.
out
.
println
(
"排序前:"
);
//
System.out.println("排序前:");
for
(
Neo4jSchema
neo4jSystem:
neo4jSchemas
){
//
for(Neo4jSchema neo4jSystem:neo4jSchemas){
//
System
.
out
.
println
(
"neo4jSystem:"
+
neo4jSystem
.
getName
());
//
System.out.println("neo4jSystem:"+neo4jSystem.getName());
}
//
}
Collections
.
sort
(
neo4jSchemas
,
new
Comparator
<
Neo4jSchema
>()
{
Collections
.
sort
(
neo4jSchemas
,
new
Comparator
<
Neo4jSchema
>()
{
@Override
@Override
public
int
compare
(
Neo4jSchema
o1
,
Neo4jSchema
o2
){
public
int
compare
(
Neo4jSchema
o1
,
Neo4jSchema
o2
){
...
@@ -61,11 +66,11 @@ public class SchemaServiceImpl implements ISchemaService {
...
@@ -61,11 +66,11 @@ public class SchemaServiceImpl implements ISchemaService {
return
compare
.
compare
(
o1
.
getName
(),
o2
.
getName
());
return
compare
.
compare
(
o1
.
getName
(),
o2
.
getName
());
}
}
});
});
System
.
out
.
println
(
"排序后:"
);
//
System.out.println("排序后:");
for
(
Neo4jSchema
neo4jSystem:
neo4jSchemas
){
//
for(Neo4jSchema neo4jSystem:neo4jSchemas){
//
System
.
out
.
println
(
"neo4jSystem:"
+
neo4jSystem
.
getName
());
//
System.out.println("neo4jSystem:"+neo4jSystem.getName());
}
//
}
for
(
Neo4jSchema
neo4jSchema:
neo4jSchemas
){
for
(
Neo4jSchema
neo4jSchema:
neo4jSchemas
){
ReturnNode
returnNode
=
new
ReturnNode
();
ReturnNode
returnNode
=
new
ReturnNode
();
returnNode
.
setId
(
neo4jSchema
.
getMetadataId
());
returnNode
.
setId
(
neo4jSchema
.
getMetadataId
());
...
@@ -292,6 +297,60 @@ public class SchemaServiceImpl implements ISchemaService {
...
@@ -292,6 +297,60 @@ public class SchemaServiceImpl implements ISchemaService {
return
true
;
return
true
;
}
}
@Override
public
Map
<
String
,
String
>
getTableIdFromSchemaId
(
String
schemaId
)
{
Map
<
String
,
String
>
tableIdMap
=
new
HashMap
<>();
//批量获取指定schema下面的table
String
countCypher
=
"match (n:Neo4jSchema{metadataId:\""
+
schemaId
+
"\"})-->(m:Neo4jTable) return count(m) as count"
;
String
detailCypher
=
"match (n:Neo4jSchema{metadataId:\""
+
schemaId
+
"\"})-->(m:Neo4jTable) return m skip 10 limit 10"
;
StatementResult
countResult
=
session
.
run
(
countCypher
);
int
jdcount
=
0
;
while
(
countResult
.
hasNext
()){
Record
record
=
countResult
.
next
();
jdcount
=
record
.
get
(
"count"
).
asInt
();
}
//分页获取 表
int
pageSize
=
100
;
int
page
=
jdcount
/
pageSize
;
for
(
int
i
=
0
;
i
<
page
+
1
;
i
++){
detailCypher
=
"match (n:Neo4jSchema{metadataId:\""
+
schemaId
+
"\"})-->(m:Neo4jTable) return m.metadataId as id skip "
+(
page
*
pageSize
)
+
" limit "
+
pageSize
;
StatementResult
detailResult
=
session
.
run
(
detailCypher
);
while
(
detailResult
.
hasNext
()){
Record
record
=
detailResult
.
next
();
String
metadataId
=
record
.
get
(
"id"
).
asString
();
tableIdMap
.
put
(
metadataId
,
""
);
}
}
return
tableIdMap
;
}
@Override
public
Map
<
String
,
String
>
getColumnIdFromSchemaId
(
String
schemaId
)
{
Map
<
String
,
String
>
columnIdMap
=
new
HashMap
<>();
//批量获取指定schema下面的table
String
countCypher
=
"match (n:Neo4jSchema{metadataId:\""
+
schemaId
+
"\"})-[:Composition]->(m:Neo4jTable)-[:Composition]->(mn:Neo4jColumn) return count(mn) as count"
;
String
detailCypher
=
"match (n:Neo4jSchema{metadataId:\""
+
schemaId
+
"\"})-->(m:Neo4jTable) return m skip 10 limit 10"
;
StatementResult
countResult
=
session
.
run
(
countCypher
);
int
jdcount
=
0
;
while
(
countResult
.
hasNext
()){
Record
record
=
countResult
.
next
();
jdcount
=
record
.
get
(
"count"
).
asInt
();
}
//分页获取 表
int
pageSize
=
100
;
int
page
=
jdcount
/
pageSize
;
for
(
int
i
=
0
;
i
<
page
+
1
;
i
++){
detailCypher
=
"match (n:Neo4jSchema{metadataId:\""
+
schemaId
+
"\"})-[:Composition]->(m:Neo4jTable)-[:Composition]->(mn:Neo4jColumn) return mn.metadataId as id skip "
+(
i
*
pageSize
)
+
" limit "
+
pageSize
;
StatementResult
detailResult
=
session
.
run
(
detailCypher
);
while
(
detailResult
.
hasNext
()){
Record
record
=
detailResult
.
next
();
String
metadataId
=
record
.
get
(
"id"
).
asString
();
columnIdMap
.
put
(
metadataId
,
""
);
}
}
return
columnIdMap
;
}
private
void
saveLabelFromTag
(
List
<
String
>
layer
,
Map
<
String
,
Neo4jSchema
>
neo4jSchemaMap
,
Map
<
String
,
List
<
Map
<
String
,
Object
>>>
returnResults
)
{
private
void
saveLabelFromTag
(
List
<
String
>
layer
,
Map
<
String
,
Neo4jSchema
>
neo4jSchemaMap
,
Map
<
String
,
List
<
Map
<
String
,
Object
>>>
returnResults
)
{
for
(
Object
obj
:
returnResults
.
keySet
())
{
for
(
Object
obj
:
returnResults
.
keySet
())
{
String
schemaId
=
(
String
)
obj
;
String
schemaId
=
(
String
)
obj
;
...
...
src/main/java/com/keymobile/metadata/metadataRelation/service/impl/SystemServiceImpl.java
View file @
5f373286
...
@@ -183,11 +183,11 @@ public class SystemServiceImpl implements ISystemService {
...
@@ -183,11 +183,11 @@ public class SystemServiceImpl implements ISystemService {
}
}
}
}
}
}
System
.
out
.
println
(
"排序前:"
);
//
System.out.println("排序前:");
for
(
Neo4jSystem
neo4jSystem:
neo4jSystemList
){
//
for(Neo4jSystem neo4jSystem:neo4jSystemList){
//
System
.
out
.
println
(
"neo4jSystem:"
+
neo4jSystem
.
getName
());
//
System.out.println("neo4jSystem:"+neo4jSystem.getName());
}
//
}
Collections
.
sort
(
neo4jSystemList
,
new
Comparator
<
Neo4jSystem
>()
{
Collections
.
sort
(
neo4jSystemList
,
new
Comparator
<
Neo4jSystem
>()
{
@Override
@Override
public
int
compare
(
Neo4jSystem
o1
,
Neo4jSystem
o2
){
public
int
compare
(
Neo4jSystem
o1
,
Neo4jSystem
o2
){
...
@@ -195,11 +195,11 @@ public class SystemServiceImpl implements ISystemService {
...
@@ -195,11 +195,11 @@ public class SystemServiceImpl implements ISystemService {
return
compare
.
compare
(
o1
.
getName
(),
o2
.
getName
());
return
compare
.
compare
(
o1
.
getName
(),
o2
.
getName
());
}
}
});
});
System
.
out
.
println
(
"排序后:"
);
//
System.out.println("排序后:");
for
(
Neo4jSystem
neo4jSystem:
neo4jSystemList
){
//
for(Neo4jSystem neo4jSystem:neo4jSystemList){
//
System
.
out
.
println
(
"neo4jSystem:"
+
neo4jSystem
.
getName
());
//
System.out.println("neo4jSystem:"+neo4jSystem.getName());
}
//
}
for
(
Neo4jSystem
neo4jSystem:
neo4jSystemList
){
for
(
Neo4jSystem
neo4jSystem:
neo4jSystemList
){
ReturnNode
returnNode
=
new
ReturnNode
();
ReturnNode
returnNode
=
new
ReturnNode
();
returnNode
.
setId
(
neo4jSystem
.
getMetadataId
());
returnNode
.
setId
(
neo4jSystem
.
getMetadataId
());
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment