采集

package.json

@@ -77,6 +77,6 @@
       "last 1 safari version"
     ]
   },
-  "proxy": "http://139.198.127.28:18189",
+  "proxy": "http://192.168.0.111:8189",
   "homepage": "http://myhost/data-govern"
 }

src/model/datasource.js

@@ -40,6 +40,10 @@ export function* getTaskSettingsByDatasource(payload) {
   return yield call(service.getTaskSettingsByDatasource, payload);
 }
 
+export function* getAllTasks() {
+  return yield call(service.getAllTasks);
+}
+
 export function* getTasksByDatasourceId(payload) {
   return yield call(service.getTasksByDatasourceId, payload);
 }

src/service/datasourcemanager.js

@@ -36,6 +36,10 @@ export function getTaskSettingsByDatasource(payload) {
   return GetJSON("/metadataharvester/task/getHarvestingTaskSettings", payload);
 }
 
+export function getAllTasks() {
+  return GetJSON("/metadataharvester/task/findAllHarvestingTasks");
+}
+
 export function getTasksByDatasourceId(payload) {
   return GetJSON("/metadataharvester/task/findHarvestingTasks", payload);
 }

src/view/Manage/MetadataHarvester/Component/Task.jsx

+import React, { useState, useEffect, useContext, useMemo, useReducer } from 'react';
+import { Space, Button, TreeSelect, Select, Pagination } from 'antd';
+
+import { AppContext } from '../../../../App';
+import { dispatch } from '../../../../model';
+import { paginate } from '../../../../util'; 
+import Table from '../../ResizeableTable';
+import UpdateTask from './UpdateTask';
+import env from '../../../../service/samples/env.json';
+
+const FC = (props) => {
+  // const { env } = useContext(AppContext); 
+  const [loadingTasks, setLoadingTasks] = useState(false);
+  const [tasks, setTasks] = useState(undefined);
+  const [pagination, setPagination] = useState({pageNum: 1, pageSize: 20});
+  const [updateTaskParam, setUpdateTaskParam] = useState({
+    visible: false,
+    action: undefined,
+    id: undefined
+  });
+
+  const {pageNum, pageSize} = pagination;
+
+  const columns = [
+    {
+      title: '序号',
+      dataIndex: 'key',
+      render: (text, record, index) => {
+        return (index+1).toString();
+      },
+      width:80,
+    },
+    {
+      title: '任务编号',
+      dataIndex: 'id',
+    },
+    {
+      title: '类型',
+      dataIndex: 'type',
+    },
+    {
+      title: '数据库中文名',
+      dataIndex: 'cnName',
+    },
+    {
+      title: '数据库英文名',
+      dataIndex: 'name',
+    },
+    {
+      title: '抽取Schema',
+      dataIndex: 'schema',
+    },
+    {
+      title: '表白名单',
+      dataIndex: 'tableWhiteList',
+    },
+    {
+      title: '表黑名单',
+      dataIndex: 'tableBlackList',
+    },
+  ]
+
+  useEffect(() => {
+    getAllTasks();
+  }, [])
+
+  const getAllTasks = () => {
+    setLoadingTasks(true);
+    dispatch({
+      type: 'datasource.getAllTasks',
+      callback: data => {
+        setLoadingTasks(false);
+        const newData = [...(data||[])];
+        newData.forEach(item => {
+          item.type = item.target?.type;
+          item.target?.targetParameters?.forEach(param => {
+            if (param.name === 'name') {
+              item.name = param.value;
+            } else if (param.name === 'cnName') {
+              item.cnName = param.value;
+            }
+          });
+          item.targetConfParameters?.forEach(param => {
+            if (param.name === 'schema') {
+              item.schema = param.value;
+            } else if (item.param === 'tableFilterParam') {
+              item.tableWhiteList = param.value;
+            } else if (item.param === 'tableBlacklist') {
+              item.tableBlackList = param.value;
+            }
+          });
+        });
+
+        console.log('new data', newData);
+        setTasks(newData||[]);
+      },
+      error: () => {
+        setLoadingTasks(false);
+      }
+    })
+  }
+
+  const onPaginationChange = (page,size) => {
+    setPagination({pageNum: page, pageSize: size });
+  }
+
+  const onUpdateTaskCancel = (refresh = false) => {
+    setUpdateTaskParam({ visible: false, action: undefined, id: undefined });
+    refresh && getAllTasks();
+  }
+  
+  return (
+    <div>
+      <div className='flex' style={{ justifyContent: 'space-between' }}>
+        <Space>
+          <Button onClick={() => { 
+            setUpdateTaskParam({ visible: true, action: 'add', id: undefined }); 
+          }}>新增任务</Button>
+          <Button>可见列设置</Button>
+          <Button>删除</Button>
+        </Space>
+        <Config onState={(state) => {
+          console.log('state', state);
+        }} />
+      </div>
+      <Table
+        className='mt-3'
+        loading={loadingTasks}
+        columns={columns||[]} 
+        rowKey='id'
+        dataSource={tasks||[]}
+        pagination={false} 
+        sticky
+      />
+      <Pagination
+        size="small"
+        className="text-center m-3"
+        showSizeChanger
+        showQuickJumper
+        onChange={onPaginationChange}
+        onShowSizeChange={onPaginationChange}
+        current={pageNum}
+        pageSize={pageSize}
+        defaultCurrent={1}
+        total={(tasks||[]).length}
+        showTotal={total => `共 ${(tasks||[]).length} 条`}
+      />
+
+      <UpdateTask
+        visible={updateTaskParam.visible}
+        action={updateTaskParam.action}
+        id={updateTaskParam.id}
+        onCancel={onUpdateTaskCancel}
+      />
+    </div>
+  )
+}
+
+export default FC;
+
+export const reducer = (prevState, action) => {
+  const scope = undefined, targetType = undefined, datasourceId = undefined;
+
+  if (action.type === 'selectScope') {
+    const [scope] = action.payload
+    return { ...prevState, scope }
+  } else if (action.type === 'selectTargetType') {
+    const [targetType] = action.payload
+    return { ...prevState, targetType }
+  } else if (action.type === 'selectDatasource') {
+    const [datasourceId] = action.payload
+    return { ...prevState, datasourceId }
+  } else if (action.type === 'init') {
+    return { ...prevState, scope, targetType, datasourceId }
+  }
+
+  return prevState
+}
+
+export function Config({  onState }) {
+  // const { env } = useContext(AppContext);
+  const [treeData, setTreeData] = useState([]);
+  const [treeExpandKeys, setTreeExpandKeys] = useState([]);
+  const [loadingSupportedTargetTypes, setLoadingSupportedTargetTypes] = useState(false);
+  const [supportedTargetTypes, setSupportedTargetTypes] = useState([]);
+  const [loadingDatasources, setLoadingDatasources] = useState(false);
+  const [datasources, setDatasources] = useState([]);
+
+  const [state, dispatchState] = useReducer(reducer, {});
+  const {scope, targetType, datasourceId} = state;
+
+  useEffect(() => {
+    onState?.(state);
+  }, [state])
+
+  useEffect(() => {
+    if (env && env.children) {
+      let newExpandKeys = [];
+      env.children.forEach(domain => {
+        // domain.id = domain.value;
+        domain.value = domain.key;
+        newExpandKeys.push(domain.key);
+
+        (domain.children||[]).forEach(system => {
+          // system.id = system.scopeId;
+          // system.value = system.key;
+        })
+      });
+
+      setTreeData(env.children);
+      setTreeExpandKeys(newExpandKeys);
+      dispatchState('init');
+    } else {
+      setTreeData([]);
+      dispatchState('init');
+    }    
+  }, [env])
+
+  useEffect(() => {
+    getSupportedTargetTypes();
+  }, [])
+
+  useEffect(() => {
+    getDatasources();
+  }, [scope])
+
+  useEffect(() => {
+    dispatchState({type: 'selectDatasource', payload: [undefined]})
+  }, [scope, targetType])
+
+  const _datasources = useMemo(() => {
+    if (datasources && targetType) {
+      return (datasources||[]).filter(item => item.type === targetType);
+    }
+
+    return datasources;
+  }, [datasources, targetType])
+
+  const getSupportedTargetTypes = () => {
+    setLoadingSupportedTargetTypes(true);
+    dispatch({
+      type: 'datasource.getSupportedTargetTypes',
+      callback: data => {
+        setLoadingSupportedTargetTypes(false);
+        setSupportedTargetTypes(data||[]);
+      },
+      error: () => {
+        setLoadingSupportedTargetTypes(false);
+      }
+    })
+  }
+
+  const getDatasources = () => {
+    setLoadingDatasources(true);
+    dispatch({
+      type: 'datasource.getAllDatasources',
+      payload: {
+        namespace: env?.domainId,
+        scope
+      },
+      callback: data => {
+        setLoadingDatasources(false);
+        setDatasources(data);
+      },
+      error: () => {
+        setLoadingDatasources(false);
+      }
+    })
+  }
+
+  const onTreeSelectChange = (value) => {
+    dispatchState({type: 'selectScope', payload: [value]})
+  }
+
+  const onTargetTypeChange = (value) => {
+    dispatchState({type: 'selectTargetType', payload: [value]})
+  }
+
+  const onDatasourceChange = (value) => {
+    dispatchState({type: 'selectDatasource', payload: [value]})
+  }
+
+  return (
+    <Space>
+      <TreeSelect 
+        allowClear
+        style={{width:170}}
+        dropdownMatchSelectWidth={210}
+        listHeight={450}
+        value={scope}
+        treeData={treeData}
+        placeholder="请选择系统"
+        treeExpandedKeys={treeExpandKeys}
+        onTreeExpand={(keys)=>{
+          setTreeExpandKeys(keys);
+        }}
+        onChange={onTreeSelectChange}
+      />
+      <Select 
+        allowClear
+        loading={loadingSupportedTargetTypes}
+        value={targetType} 
+        placeholder='请选择类型'
+        style={{ width: 170 }}
+        onChange={onTargetTypeChange}
+      >
+      {
+        supportedTargetTypes?.map((item, index) => {
+          return (
+            <Select.Option key={index} value={item.targetType}>{item.targetName}</Select.Option>
+          );
+        })
+      }
+      </Select>
+      <Select 
+        allowClear
+        loading={loadingDatasources}
+        value={datasourceId} 
+        placeholder='请选择数据库'
+        style={{ width: 170 }}
+        onChange={onDatasourceChange}
+      >
+      {
+        _datasources?.map((item, index) => {
+          const nameIndex = item.targetParameters?.findIndex(_item => _item.name === 'name');
+          let name = '';
+          if (nameIndex !== -1) {
+            name = item.targetParameters[nameIndex].value;
+          }
+
+          return (
+            <Select.Option key={index} value={item.id}>{name}</Select.Option>
+          );
+        })
+      }
+      </Select>
+    </Space>
+  )
+}
\ No newline at end of file

src/view/Manage/MetadataHarvester/Component/TaskReport.jsx

 import React, { useState, useEffect, useContext, useMemo } from 'react';
-import { Space, DatePicker, Row, Col, Card, Table, Spin, Empty, Typography, Pagination, Input } from 'antd';
+import { Space, DatePicker, Row, Col, Card, Spin, Empty, Typography, Pagination, Input } from 'antd';
 import 'moment/locale/zh-cn';
 import locale from 'antd/es/date-picker/locale/zh_CN';
 
 import { AppContext } from '../../../../App';
 import { dispatch } from '../../../../model';
 import { paginate } from '../../../../util'; 
-// import env from '../../../../service/samples/env.json';
+import Table from '../../ResizeableTable';
+import env from '../../../../service/samples/env.json';
 
 const { RangePicker } = DatePicker;
 const { Meta } = Card;
@@ -18,7 +19,7 @@ function compare(val1, val2) {
 }
 
 const FC = (props) => {
-  const { env } = useContext(AppContext);
+  // const { env } = useContext(AppContext);
   const [rangeValue, setRangeValue] = useState(undefined);
   const [datasources, setDatasources] = useState(undefined);
   const [summaryData, setSummaryData] = useState(undefined);

src/view/Manage/MetadataHarvester/Component/UpdateTask.jsx

+import React, { useEffect, useState } from 'react';
+import { Modal, Checkbox, Row, Col, Divider, Input, Typography, Form, Switch } from 'antd';
+
+import { Config } from './Task';
+import { dispatch } from '../../../../model';
+
+const FC = (props) => {
+  const {visible, onCancel, action, id} = props;
+  const [schemas, setSchemas] = useState([]);
+  const [filterSchemas, setFilterSchemas] = useState([]);
+  const [selectedSchemas, setSelectedSchemas] = useState([]);
+  const [taskSettings, setTaskSettings] = useState({});
+  const [currentTask, setCurrentTask] = useState({});
+  const [keyword, setKeyword] = useState('');
+  const [confirmLoading, setConfirmLoading] = useState(false);
+  const [checkAllValue, setCheckAllValue] = useState(false);
+  const [configState, setConfigState] = useState(undefined);
+
+  const [ form ] = Form.useForm();
+
+  useEffect(() => {
+    if (visible) {
+      if (action==='edit' && id) {
+        getTask();
+      }
+    }
+  }, [visible, action])
+
+  useEffect(() => {
+    if (configState?.datasourceId) {
+      getTaskSettings(configState?.datasourceId);
+    }
+  }, [configState?.datasourceId])
+
+  useEffect(() => {
+    setFilterSchemas((schemas||[]).filter(schema => (schema||'').indexOf(keyword)!==-1));
+  }, [keyword, schemas])
+
+  const getTask = () => {
+    dispatch({
+      type: 'datasource.getTask',
+      payload: {
+        harvestingTaskId: id
+      },
+      callback: data => {
+        setCurrentTask(data);
+        getTaskSettings(data?.target?.id);
+      }
+    })
+  }
+
+  const getTaskSettings = (id) => {
+    setCurrentTask((prevTask) => {
+      dispatch({
+        type: 'datasource.getTaskSettingsByDatasource',
+        payload: {
+          datasourceId: id
+        },
+        callback: data => {
+          setTaskSettings(data);
+
+          let recentSchemas = [];
+          data && (data.targetConfParameters||[]).forEach(item => {
+            if (item.name === 'schema') {
+              setSchemas(item.selectItem||[]);
+              recentSchemas = item.selectItem||[];
+            }
+          })
+  
+          if (action === 'edit') {
+            let _fieldsValue = {};
+            prevTask?.targetConfParameters?.forEach(item => {
+              if (item.name === 'schema') {
+                setSelectedSchemas(item.value?.split(',').filter(value=> recentSchemas.indexOf(value)!==-1));
+              } else {
+                _fieldsValue[item.name||''] = item.value||'';
+              }
+            })
+
+            form.setFieldsValue(_fieldsValue);
+          }
+        }
+      });
+
+      return prevTask;
+    });
+  }
+
+  const onOk = async() => {
+    try {
+      const row = await form.validateFields();
+      //深拷贝
+      let newTask = JSON.parse(JSON.stringify(taskSettings));
+
+      if (action === 'edit') {
+        newTask = {...newTask, ...currentTask};
+      }
+
+      newTask?.targetConfParameters?.forEach(item => {
+        if (item.name === 'schema') {
+          item.value = selectedSchemas.join(',');
+        } else {
+          item.value = row[item.name]||'';
+        }
+      });
+
+      setConfirmLoading(true);
+      dispatch({
+        type: 'datasource.saveTask',
+        payload: {
+          data: newTask
+        },
+        callback: data => {
+          setConfirmLoading(false);
+          reset();
+          onCancel && onCancel(true);
+        },
+        error: () => {
+          setConfirmLoading(false);
+        }
+      })
+
+    } catch (errInfo) {
+      console.log('Validate Failed:', errInfo);
+    }
+  }
+
+  const reset = () => {
+    setSchemas([]);
+    setSelectedSchemas([]);
+    setTaskSettings(undefined);
+    setCurrentTask(undefined);
+    setConfigState(undefined);
+    setKeyword('');
+    setCheckAllValue(false);
+    form.resetFields();
+  }
+
+  const onSearchInputChange = (e) => {
+    setKeyword(e.target.value||'');
+  }
+
+  const onCheckAll =(checked)=>{        
+    setCheckAllValue(checked);
+
+    if (checked) {
+      const _newSelectedSchemas = Array.from(new Set([...selectedSchemas, ...filterSchemas]));;
+
+      setSelectedSchemas(_newSelectedSchemas);
+    } else {
+      setSelectedSchemas(selectedSchemas.filter(schema=>!filterSchemas.includes(schema)));
+    }
+  }
+
+  const onCheckChange = (e) => {
+    if (e.target.checked) {
+      setSelectedSchemas([...selectedSchemas, e.target.value]);
+    } else {
+      const index = selectedSchemas.findIndex(key => key === e.target.value);
+      selectedSchemas.splice(index, 1)
+      setSelectedSchemas([...selectedSchemas]);
+    }
+  }
+
+  const formItemLayout = {
+    labelCol: {
+      xs: { span: 24 },
+      sm: { span: 5 },
+    },
+    wrapperCol: {
+      xs: { span: 24 },
+      sm: { span: 17 },
+    },
+  };
+
+  return (
+    <Modal
+      className='update-task-modal'
+      forceRender
+      title={action==='add'?'新增任务':'修改任务'}
+      visible={visible}
+      width={800}
+      onCancel={() => { 
+        reset();
+        onCancel?.();
+      }}
+      onOk={onOk}
+      confirmLoading={confirmLoading}
+    >
+      <Config onState={(state) => {
+        setConfigState(state);
+      }} />
+      <Divider>{(configState?.targetType==="Neo4jTarget")?'neo4j数据库信息':'schema信息'}</Divider>
+      <div className='d-flex mb-3' style={{ alignItems: 'center' }}>
+        <span className='mr-3'>{(configState?.targetType==="Neo4jTarget")?'neo4j数据库搜索':'schema搜索'}:</span>
+        <Input 
+          placeholder={`请输入${(configState?.targetType==="Neo4jTarget")?'neo4j数据库':'schema'}名称`}
+          allowClear 
+          value={keyword}
+          onChange={onSearchInputChange} 
+          style={{ width: 230 }}
+        />
+         <Switch 
+          checkedChildren="全不选" 
+          unCheckedChildren="全选" 
+          checked={ checkAllValue } 
+          onChange={ onCheckAll } 
+          style={{ marginLeft: 'auto' }}
+        />
+      </div>
+      <div style={{ maxHeight: 300, overflow: 'auto' }}>
+        <Row className='mb-3'>
+        {
+          (filterSchemas||[]).map((schema, index) => {
+            return (
+              <Col className='mt-1' key={index} md={8}>
+                <div className='d-flex'>
+                  <Checkbox checked={ selectedSchemas.indexOf(schema)!==-1 } value={schema||''} onChange={onCheckChange} >
+                  </Checkbox>
+                  <Typography.Paragraph className='ml-1' title={schema||''} ellipsis>
+                  {schema||''}
+                  </Typography.Paragraph>
+                </div>
+              </Col>
+            );
+          })
+        }
+        </Row>
+      </div>
+      <Divider>过滤信息</Divider>
+      <Form {...formItemLayout} form={form}>
+      {
+        taskSettings && (taskSettings.targetConfParameters||[]).filter(item => item.name!=='schema').map((param, index) => {
+          return (
+            <Form.Item 
+              label={param.cnName||''}
+              name={param.name||''}
+              key={index}
+              rules={[{ required: param.required, message: '必填项'}]}
+            >
+            {
+              ( param.show ? <Input placeholder={param.explain||''} /> : <Input.Password placeholder={param.explain||''} visibilityToggle={false} /> )
+            }
+            </Form.Item>
+          )
+        })
+      }
+      </Form>
+    </Modal>
+  );
+}
+
+export default FC;
\ No newline at end of file

src/view/Manage/MetadataHarvester/index.jsx

 import React, { useState } from 'react';
 import { Tabs } from 'antd';
 
+import Task from './Component/Task';
 import TaskReport from './Component/TaskReport';
 
 import './index.less';
@@ -18,7 +19,7 @@ const FC = (props) => {
     <div className='metadata-harvester'>
       <Tabs activeKey={tabKey} onChange={onTabChange}>
         <TabPane tab='采集任务' key='1'>
-        
+          <Task />
         </TabPane>
         <TabPane tab='任务执行情况' key='2'>
           <TaskReport />

src/view/Manage/ResizeableTable/index.jsx

-import { useEffect, useState } from 'react';
+import React, { useEffect, useState, useRef, useMemo } from 'react';
 import { Table } from 'antd';
 import { Resizable } from 'react-resizable';
+import ResizeObserver from 'rc-resize-observer'
+
+const scrollbarWidth = getScrollbarWidth()
 
 const ResizeableHeaderCell = props => {
   const { onResize, width, onClick, ...restProps } = props;
@@ -33,44 +36,119 @@ const ResizeableHeaderCell = props => {
 };
 
 const ResizeableTable = (props) => {
-  const { columns, ...restProps } = props;
-  const [_columns, setColumns] = useState([]);
+  const { columns, ...restProps } = props
 
-  useEffect(() => {
-    setColumns([...columns, { title: '', dataIndex: 'auto' }]);
-  }, [columns])
+  const [tableWidth, setTableWidth] = useState(0)
 
-  const handleResize = index => (e, { size }) => {
-    const nextColumns = [..._columns];
-    nextColumns[index] = {
-      ...nextColumns[index],
-      width: size.width,
-    };
+  const paddingCol = useRef({
+    key: 'padding',
+    width: 0,
+    render: () => undefined
+  })
 
-    setColumns(nextColumns);
+  const handleResize = (index) => (e, { size }) => {
+    setCols((prevCols) => {
+      const nextColumns = [...(prevCols ?? [])];
+      nextColumns[index] = {
+        ...nextColumns[index],
+        width: size.width,
+      };
+      return nextColumns;
+    });
   };
 
-  return (
-    <Table 
-      components={{
-        header: {
-          cell: ResizeableHeaderCell,
-        }
-      }}
-      columns={
-        _columns.map((column, index) => { 
+  const [cols, setCols] = useState();
+
+  useEffect(() => {
+    if (!!columns && tableWidth > 0) {
+      const contentWidth = getWidth(tableWidth)
+      setDefaultWidth(columns, contentWidth)
+      paddingCol.current.width = 0
+
+      const cols = columns
+        .map((col, index) => {
+          const colWidth = col.width ?? 100;
           return {
-            ...column,
-            onHeaderCell: column => ({
+            ...col,  
+            width: colWidth, 
+            ellipsis: true,
+            onHeaderCell: (column: any) => ({
               width: column.width,
               onResize: handleResize(index),
             }),
           };
         })
-      }
-      { ...restProps }
-    />
+      setCols(cols)
+    }
+  }, [columns, tableWidth])
+
+  const cols1 = useMemo(() => !!cols ? [...cols, paddingCol.current] : undefined, [cols])
+
+  return (
+    <ResizeObserver
+      onResize={({ width }) => {
+        setTableWidth(width)
+      }}
+    >
+      <Table 
+        components={{
+          header: {
+            cell: ResizeableHeaderCell,
+          }
+        }}
+        columns={cols1}
+        { ...restProps }
+      />
+    </ResizeObserver>
   );
 }
 
-export default ResizeableTable;
\ No newline at end of file
+export default ResizeableTable;
+
+function getWidth(tableWidth) {
+  // FIXME 判断没有选择列时,32为0
+  return tableWidth - scrollbarWidth - 32 // scrollbar width, checkbox column
+}
+
+function setDefaultWidth(columns, width) {
+  let rowWidth = 0, count = 0
+  for (const col of columns) {
+    if (typeof col.width === 'number') {
+      rowWidth += col.width
+    } else {
+      count++
+    }
+  }
+  if (count > 0) {
+    let defaultW = (rowWidth > width ? 0 : width - rowWidth) / count
+    if (defaultW < 100) {
+      defaultW = 80
+    }
+    for (const col of columns) {
+      if (typeof col.width !== 'number') {
+        col.width = defaultW
+      }
+    }
+  }
+}
+
+export function getScrollbarWidth() {
+  // Creating invisible container
+  const outer = document.createElement('div');
+  outer.style.visibility = 'hidden';
+  outer.style.overflow = 'scroll'; // forcing scrollbar to appear
+  outer.style.msOverflowStyle = 'scrollbar'; // needed for WinJS apps
+  document.body.appendChild(outer);
+
+  // Creating inner element and placing it in the container
+  const inner = document.createElement('div');
+  outer.appendChild(inner);
+
+  // Calculating difference between container's full width and the child width
+  const scrollbarWidth = (outer.offsetWidth - inner.offsetWidth);
+
+  // Removing temporary elements from the DOM
+  outer.parentNode?.removeChild(outer);
+
+  return scrollbarWidth;
+}
\ No newline at end of file