MTK Configuration File Description

Config

Config MTK迁移配置文件.

Field	Type	Description
source	Option	Source Database configuration information.
target	Option	Target Database configuration information.
limit	Limit	Concurrency configuration.
object	Object	Object definition.
dataOnly	bool	Whether only data is migrated.
schemaOnly	bool	Whether only the data structure is migrated.
disableTableDataComp	bool	Disabling table data comparison.
disableCollStatistics	bool	Disabling collection statistics.
reportFile	string	Migration report directory.
debug	bool	debug.
preRun	bool	preRun.
test	bool	test.
disableIgnoreCase	bool	Disables ignoring case queries.
disableSelectPart	bool	Disable the select by partition.
disableFKCons	bool	Disable table foreign key sync.
disableSyncIdxAfterData	bool	disable the synchronization of table data and create this table index immediately
disablePrintMigDataProgress	bool	disable printing migration data progress

Example:

{
  "source": {
    "type": "oracle",
    "connect": {
      "host": "127.0.0.1",
      "user": "system",
      "port": 1521,
      "password": "******",
      "dbName": "orcl"
    }
  },
  "target": {
    "type": "opengauss",
    "connect": {
      "version": "3.0.0",
      "host": "127.0.0.1",
      "user": "gaussdb",
      "port": 26000,
      "password": "******",
      "dbName": "postgres"
    },
    "parameter": {
      "parallelInsert": 1,
      "dropExistingObject": false,
      "truncTable": false,
      "caseSensitive": 0,
      "colKeyWords": {},
      "objKeyWords": {},
      "quoteMark": false,
      "path": "./data",
      "schemaPath": "",
      "dataPath": "",
      "fileType": "",
      "fileSize": "",
      "csvHeader": false,
      "csvNullValue": "",
      "csvFieldDelimiter": ",",
      "csvOptionallyEnclosed": "\"",
      "excludeSysTable": [],
      "remapSchema": {},
      "remapTable": {},
      "remapTablespace": {},
      "enableSyncTabTbsPro": false,
      "enableSyncCompTabPro": false,
      "timeFormat": "HH:MI:SS",
      "dateFormat": "YYYY-MM-DD",
      "dateTimeFormat": "YYYY-MM-DD HH24:MI:SS",
      "noSupportPartTabToNormalTab": false,
      "ignoreDB2PartInclusive": false,
      "igNotSupportIntervalPart": false,
      "igErrorData": false,
      "enableBatchCommit": false,
      "ignoreTabPartition": false,
      "autoAddMaxvaluePart": false,
      "autoAddMySQLAutoIncr": false,
      "autoAddMySQLAutoIncrTabList": null,
      "ignoreNotSupportDefault": false,
      "replaceZeroDate": "",
      "virtualColToNormalCol": false,
      "virtualColConv": {},
      "mySQLSkipErrorDateTimeData": false,
      "ignoreTableDDLCompErr": false,
      "convertPackageMethod": "",
      "convertOracleIntegerToNumeric": false,
      "enableOgBlobClob": false,
      "enableConvertSrid": false,
      "defaultSrid": "4326",
      "seqLastNumAddNum": 0,
      "skipColumnType": {},
      "skipColumnName": {},
      "templateSeqName": "",
      "charAppendEmptyString": false,
      "tableOptions": null,
      "indexOptions": null
    }
  },
  "limit": {
    "parallel": 2,
    "fetchSize": 1000,
    "batchSize": 1000,
    "bufferSize": 8,
    "cpBufferSize": 8,
    "oracleSelectParallel": 2,
    "channelCacheNum": 10000,
    "limit": 0
  },
  "object": {
    "tables": [
      "MTK.TAB1",
      "MTK.TAB111%",
      "MTK1.*"
    ],
    "schemas": [
      "SCHEMA1",
      "SCHEMA2"
    ],
    "excludeTable": {
      "SCHEMA1": [
        "TABLE_SKIP1",
        "TABLE_SKIP1"
      ],
      "SCHEMA2": [
        "TABLE_SKIP1",
        "TABLE_SKIP1"
      ]
    },
    "tableSplit": {
      "SCHEMA1": {
        "TAB_1": [
          "ID \u003c 10000 ",
          "ID \u003c 20000 AND ID \u003e=10000",
          "ID \u003e= 90000"
        ],
        "TAB_2": [
          "ID \u003c 10000 ",
          "ID \u003c 20000 AND ID \u003e=10000",
          "ID \u003e= 90000"
        ]
      },
      "SCHEMA2": {
        "TAB_1": [
          "ID \u003c 10000 ",
          "ID \u003c 20000 AND ID \u003e=10000",
          "ID \u003e= 90000"
        ],
        "TAB_2": [
          "ID \u003c 10000 ",
          "ID \u003c 20000 AND ID \u003e=10000",
          "ID \u003e= 90000"
        ]
      }
    }
  },
  "dataOnly": false,
  "schemaOnly": false,
  "disableTableDataComp": false,
  "disableCollStatistics": false,
  "reportFile": "./report/",
  "debug": false,
  "preRun": false,
  "test": false,
  "disableIgnoreCase": false,
  "disableSelectPart": false,
  "disableFKCons": false,
  "disableSyncIdxAfterData": false,
  "disablePrintMigDataProgress": false
}

source

Type: Option

Desc: Source Database configuration information.

Example:

source database config example

{
  "source": {
    "type": "oracle",
    "connect": {
      "host": "127.0.0.1",
      "user": "system",
      "port": 1521,
      "password": "******",
      "dbName": "orcl"
    }
  }
}

target

Type: Option

Desc: Target Database configuration information.

Example:

target database config example

{
  "target": {
    "type": "opengauss",
    "connect": {
      "version": "3.0.0",
      "host": "127.0.0.1",
      "user": "gaussdb",
      "port": 26000,
      "password": "******",
      "dbName": "postgres"
    },
    "parameter": {
      "parallelInsert": 1,
      "dropExistingObject": false,
      "truncTable": false,
      "caseSensitive": 0,
      "colKeyWords": {},
      "objKeyWords": {},
      "quoteMark": false,
      "path": "./data",
      "schemaPath": "",
      "dataPath": "",
      "fileType": "",
      "fileSize": "",
      "csvHeader": false,
      "csvNullValue": "",
      "csvFieldDelimiter": ",",
      "csvOptionallyEnclosed": "\"",
      "excludeSysTable": [],
      "remapSchema": {},
      "remapTable": {},
      "remapTablespace": {},
      "enableSyncTabTbsPro": false,
      "enableSyncCompTabPro": false,
      "timeFormat": "HH:MI:SS",
      "dateFormat": "YYYY-MM-DD",
      "dateTimeFormat": "YYYY-MM-DD HH24:MI:SS",
      "noSupportPartTabToNormalTab": false,
      "ignoreDB2PartInclusive": false,
      "igNotSupportIntervalPart": false,
      "igErrorData": false,
      "enableBatchCommit": false,
      "ignoreTabPartition": false,
      "autoAddMaxvaluePart": false,
      "autoAddMySQLAutoIncr": false,
      "autoAddMySQLAutoIncrTabList": null,
      "ignoreNotSupportDefault": false,
      "replaceZeroDate": "",
      "virtualColToNormalCol": false,
      "virtualColConv": {},
      "mySQLSkipErrorDateTimeData": false,
      "ignoreTableDDLCompErr": false,
      "convertPackageMethod": "",
      "convertOracleIntegerToNumeric": false,
      "enableOgBlobClob": false,
      "enableConvertSrid": false,
      "defaultSrid": "4326",
      "seqLastNumAddNum": 0,
      "skipColumnType": {},
      "skipColumnName": {},
      "templateSeqName": "",
      "charAppendEmptyString": false,
      "tableOptions": null,
      "indexOptions": null
    }
  }
}

limit

Type: Limit

Desc: Concurrency configuration.

Example:

limit example

{
  "limit": {
    "parallel": 2,
    "fetchSize": 1000,
    "batchSize": 1000,
    "bufferSize": 8,
    "cpBufferSize": 8,
    "oracleSelectParallel": 2,
    "channelCacheNum": 10000,
    "limit": 0
  }
}

object

Type: Object

Desc: Object definition.

Example:

object example

{
  "object": {
    "tables": [
      "MTK.TAB1",
      "MTK.TAB111%",
      "MTK1.*"
    ],
    "schemas": [
      "SCHEMA1",
      "SCHEMA2"
    ],
    "excludeTable": {
      "SCHEMA1": [
        "TABLE_SKIP1",
        "TABLE_SKIP1"
      ],
      "SCHEMA2": [
        "TABLE_SKIP1",
        "TABLE_SKIP1"
      ]
    },
    "tableSplit": {
      "SCHEMA1": {
        "TAB_1": [
          "ID \u003c 10000 ",
          "ID \u003c 20000 AND ID \u003e=10000",
          "ID \u003e= 90000"
        ],
        "TAB_2": [
          "ID \u003c 10000 ",
          "ID \u003c 20000 AND ID \u003e=10000",
          "ID \u003e= 90000"
        ]
      },
      "SCHEMA2": {
        "TAB_1": [
          "ID \u003c 10000 ",
          "ID \u003c 20000 AND ID \u003e=10000",
          "ID \u003e= 90000"
        ],
        "TAB_2": [
          "ID \u003c 10000 ",
          "ID \u003c 20000 AND ID \u003e=10000",
          "ID \u003e= 90000"
        ]
      }
    }
  }
}

dataOnly

Type: bool

Desc: Indicates whether to migrate only data.

Default: false

Option:

true
false

schemaOnly

Type: bool

Desc: Indicates whether to migration only the data structure.

Default: false

Option:

true
false

disableTableDataComp

Type: bool

Desc: Disables the table data comparison function.

After data migration is complete, MTK will count and compare the number of rows in the source and target databases.

Default: false

Option:

true
false

disableCollStatistics

Type: bool

Desc: Disables the collection statistics function.

After data migration is complete, MTK will collect statistics information from the target database.

Default: false

Option:

true
false

reportFile

Type: string

Desc: Indicates the migration report directory.

For MTK of a version earlier than v0.0.18, the migration report is shown in HTML.

For MTK v0.0.18 or later, the migration report is stored in a directory.

debug

Type: bool

Desc: Indicates whether to enable the log debug mode.

Default: false

Option:

true
false

preRun

Type: bool

Desc: pre Run.

Default: false

Option:

true
false

test

Type: bool

Desc: Try migration. Migration parameter limit.limit row data. In this mode, the data can be submitted without any data, and the data will be automatically rolled back. Just to test if the data is inserted normally

Default: false

Option:

true
false

disableIgnoreCase

Type: bool

Desc: Disables ignoring case queries.

Default: false

Option:

true
false

disableSelectPart

Type: bool

Desc: Disable the partition table migration data by partition feature.

Default: false

Option:

true
false

disableFKCons

Type: bool

Desc: Disables synchronization of table foreign key constraints.

Default: false

Option:

true
false

disableSyncIdxAfterData

Type: bool

Desc: disable the synchronization of table data and create this table index immediately

Default: false

Option:

true
false

disablePrintMigDataProgress

Type: bool

Desc: disable printing migration data progress

Default: false

Option:

true
false

Limit

Limit concurrency configuration

Field	Type	Description
parallel	int	Degree of parallelism.
fetchSize	int	Specifies the number of rows to fetch at a time.
batchSize	int	Specify the size of batch insert or copy and Commit.
bufferSize	int	Specifies the buffer size (in MB) when querying or inserting in batches.
cpBufferSize	int	Defines the buffer size (in MB) used in the Copy command.
oracleSelectParallel	int	Add hint `/* +parallel(t,n) */`
limit	int64	Defines how many rows to migrate per table.

Appears in:

Config.limit

Example:

limit example

{
  "parallel": 2,
  "fetchSize": 1000,
  "batchSize": 1000,
  "bufferSize": 8,
  "cpBufferSize": 8,
  "oracleSelectParallel": 2,
  "channelCacheNum": 10000,
  "limit": 0
}

parallel

Type: int

Desc: Indicates the degree of parallelism.

Create tables, constraints, etc. in parallel..

Default: 1

fetchSize

Type: int

Desc: Specifies the row size for one row fetch.

Valid values are 1-50000

Oracle support.

Default: 1000

batchSize

Type: int

Desc: Specify the size of batch insert or Copy and make Commit.

Versions after 2.3.4 began to support batch Commit. The previous version only interacted with data in batches without committing

If the submission fails the program will exit and log the error data to the error file.

If you configure igErrorData the program will not exit and will log the error data to the error file and continue processing the data.

Valid values are 1-50000

Support Oracle, PostgreSQL, openGauss, MySQL.

Batch commit supports PostgreSQL/MogDB,MySQL

Default: 1000

bufferSize

Type: int

Desc: Specifies the cache size (in MB) when querying or bulk inserting.

Valid values are 1-1024

Support PostgreSQL, openGauss, MySQL.

Default: 8 MB

cpBufferSize

Type: int

Desc: Defines the cache size (in MB) used in the Copy command.

Valid values are 1-1024

Support PostgreSQL, openGauss, MySQL

Default: 8 MB

oracleSelectParallel

Type: int

Desc: Add hint /* +parallel(t,n) */ to Oracle query statement

limit

Type: int64

Desc: Define how many rows to migrate per table.

Defining this parameter no longer does table concurrent migrations.

Connect

Connect database connection information

Field	Type	Description
version	string	Database version.
vendor	string	Database issuer.
host	string	Database host
user	string	Database user
port	int	Database port
password	Password	Database User Password
dbName	string	Database name
dsn	string	User-specified connection string.
timeout	Duration	Connection timeout.
charset	string	Database character set.
datCompatibility	string	For openGauss database compatibility mode.
sqlMode	string	For MySQL database `sql_mode`.
clientCharset	string	It is used for database encoding conversion scenarios, and generally does not need to be set.

Appears in:

Option.connect

version

Type: string

Desc: Database version.

No need to specify. Automatic query when connecting to database

Migrating into a file requires manually specifying the version

Example:

version example

{
  "version": "2.1.0"
}

vendor

Type: string

Desc: Database publisher.

No need to specify. Automatic query when connecting to database

Example:

vendor example

{
  "vendor": "MySQL"
}

host

Type: string

Desc: database host

Option:

ip
域名

Example:

host example

{
  "host": "127.0.0.1"
}

user

Type: string

Desc: 数据库用户

Example:

host example

{
  "user": "system"
}

port

Type: int

Desc: database port

Example:

port example

{
  "port": 1521
}

password

Type: Password

Desc: database user password

dbName

Type: string

Desc: Name database

Example:

dbName example

{
  "dbName": "orcl"
}

dsn

Type: string

Desc: User-specified connection string.

No need to specify by default. Reserve functions for special scenarios

timeout

Type: Duration

Desc: connection timeout.

No configuration required

Default: 30s

charset

Type: string

Desc: database character set.

No configuration by default, connect to database query

Option:

gbk
utf8

Example:

charset example

{
  "charset": "gbk"
}

datCompatibility

Type: string

Desc: Compatibility mode for the openGauss database.

By default, no configuration is required, and the database query is connected.

Migrating into a file requires configuration

Option:

Example:

charset example

{
  "datCompatibility": "A"
}

sqlMode

Type: string

Desc: sql_mode for MySQL databases.

By default, no configuration is required, and the database query is connected.

Migrating into a file requires configuration

clientCharset

Type: string

Desc: It is used for database encoding conversion scenarios, and generally does not need to be set.

Such as Oracle ZHS16GBK encoding migrated to openGauss UTF8. Encountered ORA-29275: partial multibyte character.

Option

Option database configuration

Field	Type	Description
type	string	Database type, case insensitive
connect	Connect	Database connection information
parameter	Parameter	Parameter configuration

Appears in:

Config.source
Config.target

Example:

source database config example

{
  "type": "oracle",
  "connect": {
    "host": "127.0.0.1",
    "user": "system",
    "port": 1521,
    "password": "******",
    "dbName": "orcl"
  }
}

target database config example

{
  "type": "opengauss",
  "connect": {
    "version": "3.0.0",
    "host": "127.0.0.1",
    "user": "gaussdb",
    "port": 26000,
    "password": "******",
    "dbName": "postgres"
  },
  "parameter": {
    "parallelInsert": 1,
    "dropExistingObject": false,
    "truncTable": false,
    "caseSensitive": 0,
    "colKeyWords": {},
    "objKeyWords": {},
    "quoteMark": false,
    "path": "./data",
    "schemaPath": "",
    "dataPath": "",
    "fileType": "",
    "fileSize": "",
    "csvHeader": false,
    "csvNullValue": "",
    "csvFieldDelimiter": ",",
    "csvOptionallyEnclosed": "\"",
    "excludeSysTable": [],
    "remapSchema": {},
    "remapTable": {},
    "remapTablespace": {},
    "enableSyncTabTbsPro": false,
    "enableSyncCompTabPro": false,
    "timeFormat": "HH:MI:SS",
    "dateFormat": "YYYY-MM-DD",
    "dateTimeFormat": "YYYY-MM-DD HH24:MI:SS",
    "noSupportPartTabToNormalTab": false,
    "ignoreDB2PartInclusive": false,
    "igNotSupportIntervalPart": false,
    "igErrorData": false,
    "enableBatchCommit": false,
    "ignoreTabPartition": false,
    "autoAddMaxvaluePart": false,
    "autoAddMySQLAutoIncr": false,
    "autoAddMySQLAutoIncrTabList": null,
    "ignoreNotSupportDefault": false,
    "replaceZeroDate": "",
    "virtualColToNormalCol": false,
    "virtualColConv": {},
    "mySQLSkipErrorDateTimeData": false,
    "ignoreTableDDLCompErr": false,
    "convertPackageMethod": "",
    "enableOgBlobClob": false,
    "enableConvertSrid": false,
    "defaultSrid": "4326",
    "seqLastNumAddNum": 0,
    "skipColumnType": {},
    "skipColumnName": {},
    "templateSeqName": "",
    "charAppendEmptyString": false,
    "tableOptions": null,
    "indexOptions": null
  }
}

type

Type: string

Desc: database type, case insensitive

Option:

MySQL
Oracle
Postgres/PostgreSQL
openGauss
MogDB
DB2
sqlServer
file
informix

Example:

type example

{
  "type": "Oracle"
}

connect

Type: Connect

Desc: Database connection information

parameter

Type: Parameter

Desc: parameter configuration