datax

datax同步工具的使用

简介：datax是开源的数据同步工具 跨库 同步数据到自己指定的数据库。
连接：(文章转载自) https://blog.csdn.net/weixin_42786042/article/details/104962402

建议使用dataxWeb 更好的可以更好的方便使用

datax 的安装（contos7）

1.检查开启需要的端口

firewall-cmd --add-port=18080/tcp --permanent

firewall-cmd --add-port=18081/tcp --permanent

firewall-cmd --add-port=19999/tcp --permanent

1. 复制到指定文件datax

采坑记录

遇到的问题

    1： 以jar形式linux 后台启动报错 () 注意检查 pom 文件加入这两个,oracle 连接报错我还没找到问题在哪

            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-compiler-plugin</artifactId>
                <configuration>
                    <source>1.8</source>
                    <target>1.8</target>
                </configuration>
            </plugin>
            <plugin>
                <groupId>org.springframework.boot</groupId>
                <artifactId>spring-boot-maven-plugin</artifactId>
                <executions>
                    <execution>
                        <goals>
                            <goal>repackage</goal>
                        </goals>
                    </execution>
                </executions>
            </plugin>

    2：在启动时修改配置的文件路径 （大意可以参照 datax 文件上传到指定文件夹下 赋予可执行权限等）

    
    executor:
    jsonpath: /usr/local/ybDataX/jsons

    #   注释掉的是在本地启动的服务
    #    jsonpath: F:\\temp\\executor\\json\\
    #  pypath: F:\svnCode\CodeGroup\DBsys\datax\bin\datax.py
    pypath: /usr/local/ybDataX/datax/bin/datax.py

    3： 同步json 文件

    {
  "job": {
    "setting": {
      "speed": {
        "channel": 5
      },
      "errorLimit": {
        "record": 0,
        "percentage": 0.02
      }
    },
    "content": [
      {
        "reader": {
          "name": "mysqlreader",
          "parameter": {
            "username": "root",
            "password": "root",
            "connection": [
              {
                "querySql": [  // 我这边使用的是querysql 
                  "SELECT wxdh,sbid,ysbbh,sbmc,bxks,bxsj,jdr,jdsj,gzms,bxjzq,wxzt,jjcd FROM wxxx_djx"
                ],
                "jdbcUrl": [
                  "jdbc:mysql://ip:3306/suwell?serverTimezone=Asia/Shanghai&useLegacyDatetimeCode=false&useSSL=false&nullNamePatternMatchesAll=true&useUnicode=true&characterEncoding=UTF-8\n"
                ]
              }
            ]
          }
        },
        "writer": {
          "name": "oraclewriter",
          "parameter": {
            "username": "root",
            "password": "root",
            "column": [
              "WXDH",
              "SBID",
              "SBBH",
              "SBMC",
              "BXKS",
              "BXSJ",
              "JDR",
              "JDSJ",
              "GZMS",
              "BXJZQ",
              "WXZT",
              "JJCD"
            ],
            "preSql": [  // insert 语句之前执行的动作（我这边是要清除全库再更新进新数据（数据量不多））
              "DELETE WXXX_DJX"
            ],
            "connection": [
              {
                "table": [
                  "WXXX_DJX"
                ],
                "jdbcUrl": "jdbc:oracle:thin:@ip:1521:ORCL"
              }
            ]
          }
        }
      }
    ]
  }
}

增量同步数据 https://segmentfault.com/a/1190000021477420
 querySql : WHERE wcsj >= FROM_UNIXTIME(${lastTime}) AND wcsj < FROM_UNIXTIME(${currentTime})
 增量同步字段：-DlastTime=%s -DcurrentTime=%s
 增量同步json 

 {
  "job": {
    "setting": {
      "speed": {
        "channel": 5
      },
      "errorLimit": {
        "record": 0,
        "percentage": 0.02
      }
    },
    "content": [
      {
        "reader": {
          "name": "mysqlreader",
          "parameter": {
            "username": "**",
            "password": "***",
            "connection": [
              {
                "querySql": [
                  "SELECT wxdh,sbid,ysbbh,sbmc,bxks,bxsj,jdr,jdsj,gzms,bxjzq,jjcd,wcsj FROM wxxx_ywc WHERE wcsj >= FROM_UNIXTIME(${lastTime}) AND wcsj < FROM_UNIXTIME(${currentTime})"
                ],
                "jdbcUrl": [
                  "jdbc:mysql://IP:3306/suwell?serverTimezone=Asia/Shanghai&useLegacyDatetimeCode=false&useSSL=false&nullNamePatternMatchesAll=true&useUnicode=true&characterEncoding=UTF-8\n"
                ]
              }
            ]
          }
        },
        "writer": {
          "name": "oraclewriter",
          "parameter": {
            "username": "**",
            "password": "**",
            "column": [
              "WXDH",
              "SBID",
              "SBBH",
              "SBMC",
              "BXKS",
              "BXSJ",
              "JDR",
              "JDSJ",
              "GZMS",
              "BXJZQ",
              "JJCD",
              "WCSJ"
            ],
            "preSql": [],
            "connection": [
              {
                "table": [
                  "WXXX_YWC"
                ],
                "jdbcUrl": "jdbc:oracle:thin:@**.**.**.**:*:ORCL"
              }
            ]
          }
        }
      }
    ]
  }
}
                 "WXXX_YWC"
                ],
                "jdbcUrl": "jdbc:oracle:thin:@**.**.**.**:*:ORCL"
              }
            ]
          }
        }
      }
    ]
  }
}