Datax3.0的安裝和基本使用
阿新 • • 發佈：2019-01-06
安裝過程：
1、先解壓datax的安裝包
[root@slave1 datax]# tar -xvf datax.tar.gz 2、個datax的安裝路徑授權
[root@slave1 datax]# chmod -R 775 ./datax3、測試樣例
[[email protected] bin]# python datax.py ../job/job.json
DataX (DATAX-OPENSOURCE-3.0), From Alibaba !
Copyright (C) 2010-2016, Alibaba Group. All Rights Reserved.
2016 以上表示能夠正常使用
配置測試樣例：
下面我們配置一組 從mysql資料庫到另一個mysql資料庫
第一步、建立作業的配置檔案（json格式）
可以通過命令檢視配置模板： python datax.py -r {YOUR_READER} -w {YOUR_WRITER}
[root@slave1 bin]# python datax.py -r mysqlreader -w mysqlwriter
DataX (DATAX-OPENSOURCE-3.0), From Alibaba !
Copyright (C) 2010-2016, Alibaba Group. All Rights Reserved.
Please refer to the mysqlreader document:
https://github.com/alibaba/DataX/blob/master/mysqlreader/doc/mysqlreader.md
Please refer to the mysqlwriter document:
https://github.com/alibaba/DataX/blob/master/mysqlwriter/doc/mysqlwriter.md
Please save the following configuration as a json file and use
python {DATAX_HOME}/bin/datax.py {JSON_FILE_NAME}.json
to run the job.
{
"job": {
"content": [
{
"reader": {
"name": "mysqlreader",
"parameter": {
"column": [],
"connection": [
{
"jdbcUrl": [],
"table": []
}
],
"password": "",
"username": "",
"where": ""
}
},
"writer": {
"name": "mysqlwriter",
"parameter": {
"column": [],
"connection": [
{
"jdbcUrl": "",
"table": []
}
],
"password": "",
"preSql": [],
"session": [],
"username": "",
"writeMode": ""
}
}
}
],
"setting": {
"speed": {
"channel": ""
}
}
}
}第二步、根據配置檔案模板填寫相關選項
命令列印裡面包含對應reader、writer的文件地址，以及配置json樣例，根據json樣例填空完成配置即可。根據模板配置json檔案(mysql2mysql.json)如下
{
"job": {
"content": [
{
"reader": {
"name": "mysqlreader",
"parameter": {
"column": ["store_id","store_type","region_id","store_name","store_number","store_street_address","store_city","store_state","store_postal_code","store_country","store_manager","store_phone","store_fax","first_opened_date","last_remodel_date","store_sqft","grocery_sqft","frozen_sqft","meat_sqft","coffee_bar","video_store","salad_bar","prepared_food","florist"],
"connection": [
{
"jdbcUrl": ["jdbc:mysql://192.168.18.149:3306/saiku"],
"table": ["store"]
}
],
"password": "root",
"username": "root"
}
},
"writer": {
"name": "mysqlwriter",
"parameter": {
"column": ["store_id","store_type","region_id","store_name","store_number","store_street_address","store_city","store_state","store_postal_code","store_country","store_manager","store_phone","store_fax","first_opened_date","last_remodel_date","store_sqft","grocery_sqft","frozen_sqft","meat_sqft","coffee_bar","video_store","salad_bar","prepared_food","florist"],
"connection": [
{
"jdbcUrl": "jdbc:mysql://192.168.18.147:3306/saiku",
"table": ["store"]
}
],
"password": "root",
"username": "root"
}
}
}
],
"setting": {
"speed": {
"channel": "1"
}
}
}
}第三步：啟動DataX
[root@slave1 bin]# python datax.py ./mysql2mysql.json
同步結束，顯示日誌如下：
任務啟動時刻 : 2016-12-10 07:47:16
任務結束時刻 : 2016-12-10 07:47:27
任務總計耗時 : 11s
任務平均流量 : 320B/s
記錄寫入速度 : 2rec/s
讀出記錄總數 : 25
讀寫失敗總數 : 0job的配置
（1）、Job基本配置
Job基本配置定義了一個Job基礎的、框架級別的配置資訊，包括：
{
"job": {
"content": [
{
"reader": {
"name": "",
"parameter": {}
},
"writer": {
"name": "",
"parameter": {}
}
}
],
"setting": {
"speed": {},
"errorLimit": {}
}
}
}（2） Job Setting配置
{
"job": {
"content": [
{
"reader": {
"name": "",
"parameter": {}
},
"writer": {
"name": "",
"parameter": {}
}
}
],
"setting": {
"speed": {
"channel": 1,
"byte": 104857600
},
"errorLimit": {
"record": 10,
"percentage": 0.05
}
}
}
}● job.setting.speed(流量控制)
Job支援使用者對速度的自定義控制，channel的值可以控制同步時的併發數，byte的值可以控制同步時的速度
● job.setting.errorLimit(髒資料控制)
Job支援使用者對於髒資料的自定義監控和告警，包括對髒資料最大記錄數閾值（record值）或者髒資料佔比閾值（percentage值），當Job傳輸過程出現的髒資料大於使用者指定的數量/百分比，DataX Job報錯退出。