Skip to content
Projects
Groups
Snippets
Help
This project
Loading...
Sign in / Register
Toggle navigation
N
neo4jRelation
Overview
Overview
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
qiuchaofei
neo4jRelation
Commits
f3008c19
Commit
f3008c19
authored
Jun 14, 2022
by
qiuchaofei
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
1.修改分页获取的数目*2
parent
63cbbd76
Hide whitespace changes
Inline
Side-by-side
Showing
2 changed files
with
19 additions
and
7 deletions
+19
-7
AsyncDataFromMongoToNeo4j.java
...a/metadataRelation/service/AsyncDataFromMongoToNeo4j.java
+8
-1
MetadataServiceImpl.java
...ta/metadataRelation/service/impl/MetadataServiceImpl.java
+11
-6
No files found.
src/main/java/com/keymobile/metadata/metadataRelation/service/AsyncDataFromMongoToNeo4j.java
View file @
f3008c19
...
...
@@ -70,15 +70,22 @@ public class AsyncDataFromMongoToNeo4j {
}
});
//先删除同个环境的数据
long
time0
=
System
.
currentTimeMillis
();
metadataService
.
deleteNodeByCatalogName
(
catalogName
);
long
time1
=
System
.
currentTimeMillis
();
logger
.
info
(
"删除"
+
catalogName
+
"的数据完成,耗时:"
+(
time1
-
time0
));
//先同步元数据,纵向关系,root开始,系统--schema--table--column,系统--etl作业--etl脚本--etlsql。
metadataService
.
syschroMetadataFromMongo
(
catalogName
,
mongoDataList
);
//再同步元数据的横向关系
metadataService
.
syschroTable2EtlJobRelations
(
catalogName
);
//在同步表与表之间的横向关系
metadataService
.
syschroTable2TableRelations
(
catalogName
);
//同步1104报表
//
metadataService.syschro1104Relations(catalogName);
// metadataService.syschro1104Relations(catalogName);
// metadataService.syschroRelationFromMongo(catalogName);
// // 同步其他数据(模型,标准,质量等)
//
...
...
src/main/java/com/keymobile/metadata/metadataRelation/service/impl/MetadataServiceImpl.java
View file @
f3008c19
...
...
@@ -1232,9 +1232,7 @@ public class MetadataServiceImpl implements IMetadataService {
long
columnTime1
=
System
.
currentTimeMillis
();
//专门做字段解析,批量获取字段,进行组装。
saveAllColumnByBatch
(
catalogName
,
compositionRelationList
,
neo4jTableIdMap
);
long
columnTime2
=
System
.
currentTimeMillis
();
logger
.
info
(
"获取字段用时:"
+(
columnTime2
-
columnTime1
));
logger
.
info
(
"创建的system数量:"
+
systemDataMap
.
size
());
logger
.
info
(
"创建的schema:"
+(
objectCountMap
.
get
(
SchemaCountString
)-
1
));
// logger.info("创建的table数量:"+(objectCountMap.get(TableCountString)-1));
...
...
@@ -1306,13 +1304,17 @@ public class MetadataServiceImpl implements IMetadataService {
schem2Table
.
setEnd
(
neo4jTable
);
schem2Table
.
setName
(
"Schema--Table"
);
compositionRelationList
.
add
(
schem2Table
);
tableCount
++;
neo4jTableIdMap
.
put
(
neo4jTable
.
getMetadataId
(),
neo4jTable
);
if
(
tableCount
%
batchSize
==
0
){
neo4jTableRepository
.
saveAll
(
neo4jTableList
);
logger
.
info
(
"同步了:"
+
tableCount
+
"个表。"
+
tableId
);
neo4jTableList
.
clear
();
}
tableCount
++;
}
if
(
tableCursor
!=
null
){
tableCursor
.
close
();
}
if
(
neo4jTableList
.
size
()!=
0
){
...
...
@@ -1383,9 +1385,12 @@ public class MetadataServiceImpl implements IMetadataService {
private
void
saveAllColumnByBatch
(
String
catalogName
,
List
<
CompositionRelation
>
compositionRelationList
,
Map
<
String
,
Neo4jTable
>
neo4jTableIdMap
)
{
long
columnTime1
=
System
.
currentTimeMillis
();
int
columnCount
=
0
;
// objectCountMap.get(ColumnCountString);
String
ColumnPrefix
=
"Catalog,Database,Schema,Table,Column"
;
int
batchSize
=
2
000
;
int
batchSize
=
5
000
;
BasicDBObject
query
=
new
BasicDBObject
();
query
.
put
(
"_class"
,
ColumnPrefix
);
MongoCursor
<
Document
>
columnCursor
=
this
.
mongoTemplate
.
getCollection
(
PREFIX_METADATA_NODE
+
catalogName
).
...
...
@@ -1424,8 +1429,8 @@ public class MetadataServiceImpl implements IMetadataService {
neo4jColumnRepository
.
saveAll
(
neo4jColumnList
);
neo4jColumnList
.
clear
();
}
logger
.
info
(
"创建的column数量:"
+
columnCount
);
long
columnTime2
=
System
.
currentTimeMillis
();
logger
.
info
(
"创建的column数量:"
+
columnCount
+
",用时:"
+(
columnTime2
-
columnTime1
)
);
}
private
void
saveTableDocument
(
List
<
Document
>
tableDocumentList
,
Neo4jSchema
neo4jSchema
,
Map
<
String
,
Integer
>
objectCountMap
,
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment