四、資料倉庫--ORACLE資料庫卸數實現
阿新 • • 發佈：2019-02-01
5、所用的JAR包package cn.com.dataunload.genektr.tools; import java.io.File; import java.io.FileOutputStream; import java.io.OutputStream; import java.io.PrintStream; public class DataUnloadGenKTR { //用到tables.xml private static final String TABLE_FILE = "res/resources/Tables.xml"; private static final String KTR_FILE_PATH = "res/resources/ktr"; //private static final String TABLE_FILE = "res/resources/cmbanktables.xml"; //private static final String KTR_FILE_PATH = "res/resources/test"; private Tables tables; private OutputTool tool = null; private static final String MODE_ALL = "ALL"; private static final String MODE_ADD = "ADD"; private static final String MODE_DEL = "DEL"; private void start() throws Exception { init(); for( Table table : tables.getTableList() ) { geneTableAllKTR(table); } } private void init() { tables = new Tables(TABLE_FILE); tool = new OutputTool(); } //生成所有表的KTR檔案 private void geneTableAllKTR(Table table) throws Exception { StringBuffer buf = new StringBuffer(); buf.append(xmlHeader()); buf.append(transformation(table, MODE_ALL)); geneKTRFile(table.getTableName(), MODE_ALL, buf); buf = new StringBuffer(); buf.append(xmlHeader()); buf.append(transformation(table, MODE_ADD)); geneKTRFile(table.getTableName(), MODE_ADD, buf); buf = new StringBuffer(); buf.append(xmlHeader()); buf.append(transformation(table, MODE_DEL)); geneKTRFile(table.getTableName(), MODE_DEL, buf); } //StringBuffer xmlHeader()的定義 private StringBuffer xmlHeader() { StringBuffer buf = new StringBuffer(); buf.append("<?xml version=\"1.0\" encoding=\"UTF-8\"?>").append("\n"); return buf; } //StringBuffer transformation(Table table, String mode)的定義 private StringBuffer transformation(Table table, String mode) { StringBuffer buf = new StringBuffer(); buf.append(tool.startLine("transformation")); buf.append(info(table, mode)); buf.append(tool.startLine("notepads")); buf.append(tool.endLine("notepads")); buf.append(connection(table, mode)); buf.append(order(table)); buf.append(tableInput(table, mode)); buf.append(fileOutput(table, mode)); buf.append(tool.startLine("step_error_handling")); buf.append(tool.endLine("step_error_handling")); buf.append(tool.startLine("slave-step-copy-partition-distribution")); buf.append(tool.endLine("slave-step-copy-partition-distribution")); buf.append(tool.textLine("slave_transformation", "N")); buf.append(tool.endLine("transformation")); return buf; } /* * 開始構造ktr檔案的<info>節點，具體可以檢視UE開啟的ktr檔案 */ private StringBuffer info(Table table, String mode) { StringBuffer buf = new StringBuffer(); buf.append(tool.startLine("info")); buf.append(tool.textLine("name", getFullName(table.getTableName(), mode))); buf.append(tool.emptyLine("description")); buf.append(tool.emptyLine("extended_description")); buf.append(tool.emptyLine("trans_version")); buf.append(tool.textLine("trans_type", "Normal")); buf.append(tool.textLine("directory", "/")); buf.append(tool.startLine("parameters")); buf.append(tool.endLine("parameters")); buf.append(tool.startLine("log")); buf.append(tool.startLine("trans-log-table")); buf.append(tool.emptyLine("connection")); buf.append(tool.emptyLine("schema")); buf.append(tool.emptyLine("table")); buf.append(tool.emptyLine("size_limit_lines")); buf.append(tool.emptyLine("interval")); buf.append(tool.emptyLine("timeout_days")); buf.append(tool.field("ID_BATCH")); buf.append(tool.field("CHANNEL_ID")); buf.append(tool.field("TRANSNAME")); buf.append(tool.field("STATUS")); buf.append(tool.fieldSubject("LINES_READ")); buf.append(tool.fieldSubject("LINES_WRITTEN")); buf.append(tool.fieldSubject("LINES_UPDATED")); buf.append(tool.fieldSubject("LINES_INPUT")); buf.append(tool.fieldSubject("LINES_OUTPUT")); buf.append(tool.fieldSubject("LINES_REJECTED")); buf.append(tool.field("ERRORS")); buf.append(tool.field("STARTDATE")); buf.append(tool.field("ENDDATE")); buf.append(tool.field("LOGDATE")); buf.append(tool.field("DEPDATE")); buf.append(tool.field("REPLAYDATE")); buf.append(tool.field("LOG_FIELD")); buf.append(tool.endLine("trans-log-table")); buf.append(tool.startLine("perf-log-table")); buf.append(tool.emptyLine("connection")); buf.append(tool.emptyLine("schema")); buf.append(tool.emptyLine("table")); buf.append(tool.emptyLine("interval")); buf.append(tool.emptyLine("timeout_days")); buf.append(tool.field("ID_BATCH")); buf.append(tool.field("SEQ_NR")); buf.append(tool.field("LOGDATE")); buf.append(tool.field("TRANSNAME")); buf.append(tool.field("STEPNAME")); buf.append(tool.field("STEP_COPY")); buf.append(tool.field("LINES_READ")); buf.append(tool.field("LINES_WRITTEN")); buf.append(tool.field("LINES_UPDATED")); buf.append(tool.field("LINES_INPUT")); buf.append(tool.field("LINES_OUTPUT")); buf.append(tool.field("LINES_REJECTED")); buf.append(tool.field("ERRORS")); buf.append(tool.field("INPUT_BUFFER_ROWS")); buf.append(tool.field("OUTPUT_BUFFER_ROWS")); buf.append(tool.endLine("perf-log-table")); buf.append(tool.startLine("channel-log-table")); buf.append(tool.emptyLine("connection")); buf.append(tool.emptyLine("schema")); buf.append(tool.emptyLine("table")); buf.append(tool.emptyLine("timeout_days")); buf.append(tool.field("ID_BATCH")); buf.append(tool.field("CHANNEL_ID")); buf.append(tool.field("LOG_DATE")); buf.append(tool.field("LOGGING_OBJECT_TYPE")); buf.append(tool.field("OBJECT_NAME")); buf.append(tool.field("OBJECT_COPY")); buf.append(tool.field("REPOSITORY_DIRECTORY")); buf.append(tool.field("FILENAME")); buf.append(tool.field("OBJECT_ID")); buf.append(tool.field("OBJECT_REVISION")); buf.append(tool.field("PARENT_CHANNEL_ID")); buf.append(tool.field("ROOT_CHANNEL_ID")); buf.append(tool.endLine("channel-log-table")); buf.append(tool.startLine("step-log-table")); buf.append(tool.emptyLine("connection")); buf.append(tool.emptyLine("schema")); buf.append(tool.emptyLine("table")); buf.append(tool.emptyLine("timeout_days")); buf.append(tool.field("ID_BATCH")); buf.append(tool.field("CHANNEL_ID")); buf.append(tool.field("LOG_DATE")); buf.append(tool.field("TRANSNAME")); buf.append(tool.field("STEPNAME")); buf.append(tool.field("STEP_COPY")); buf.append(tool.field("LINES_READ")); buf.append(tool.field("LINES_WRITTEN")); buf.append(tool.field("LINES_UPDATED")); buf.append(tool.field("LINES_INPUT")); buf.append(tool.field("LINES_OUTPUT")); buf.append(tool.field("LINES_REJECTED")); buf.append(tool.field("ERRORS")); buf.append(tool.field("LOG_FIELD")); buf.append(tool.endLine("step-log-table")); buf.append(tool.endLine("log")); buf.append(tool.startLine("maxdate")); buf.append(tool.emptyLine("connection")); buf.append(tool.emptyLine("table")); buf.append(tool.emptyLine("field")); buf.append(tool.textLine("offset", "0.0")); buf.append(tool.textLine("maxdiff", "0.0")); buf.append(tool.endLine("maxdate")); buf.append(tool.textLine("size_rowset", "10000")); buf.append(tool.textLine("sleep_time_empty", "50")); buf.append(tool.textLine("sleep_time_full", "50")); buf.append(tool.textLine("unique_connections", "N")); buf.append(tool.textLine("feedback_shown", "Y")); buf.append(tool.textLine("feedback_size", "50000")); buf.append(tool.textLine("using_thread_priorities", "Y")); buf.append(tool.emptyLine("shared_objects_file")); buf.append(tool.textLine("capture_step_performance", "N")); buf.append(tool.textLine("step_performance_capturing_delay", "1000")); buf.append(tool.textLine("step_performance_capturing_size_limit", "100")); buf.append(tool.startLine("dependencies")); buf.append(tool.endLine("dependencies")); buf.append(tool.startLine("partitionschemas")); buf.append(tool.endLine("partitionschemas")); buf.append(tool.startLine("slaveservers")); buf.append(tool.endLine("slaveservers")); buf.append(tool.startLine("clusterschemas")); buf.append(tool.endLine("clusterschemas")); buf.append(tool.textLine("created_user", "-")); buf.append(tool.textLine("created_date", "2014/01/16 19:22:02.123")); buf.append(tool.textLine("modified_user", "-")); buf.append(tool.textLine("modified_date", "2014/01/16 19:22:02.123")); buf.append(tool.endLine("info")); return buf; } /* * 構造ktr檔案的connection節點 * DEL的KTR檔案的UNLOAD_DB_DEL_HOSTNAME還沒有構造， * 暫時統一用UNLOAD_DB_HOSTNAME */ private StringBuffer connection(Table table, String mode) { StringBuffer buf = new StringBuffer(); buf.append(tool.startLine("connection")); buf.append(tool.textLine("name", "ploanlt1")); buf.append(tool.textLine("server", "${UNLOAD_DB_HOSTNAME}")); buf.append(tool.textLine("type", "ORACLE")); buf.append(tool.textLine("access", "Native")); buf.append(tool.textLine("database", "${UNLOAD_DB_SID}")); buf.append(tool.textLine("port", "1521")); buf.append(tool.textLine("username", "${UNLOAD_DB_USERNAME}")); buf.append(tool.textLine("password", "${UNLOAD_DB_PASSWORD}")); buf.append(tool.emptyLine("servername")); buf.append(tool.emptyLine("data_tablespace")); buf.append(tool.emptyLine("index_tablespace")); buf.append(tool.startLine("attributes")); buf.append(tool.startLine("attribute")); buf.append(tool.textLine("code", "FORCE_IDENTIFIERS_TO_LOWERCASE")); buf.append(tool.textLine("attribute", "N")); buf.append(tool.endLine("attribute")); buf.append(tool.startLine("attribute")); buf.append(tool.textLine("code", "FORCE_IDENTIFIERS_TO_UPPERCASE")); buf.append(tool.textLine("attribute", "N")); buf.append(tool.endLine("attribute")); buf.append(tool.startLine("attribute")); buf.append(tool.textLine("code", "IS_CLUSTERED")); buf.append(tool.textLine("attribute", "N")); buf.append(tool.endLine("attribute")); buf.append(tool.startLine("attribute")); buf.append(tool.textLine("code", "PORT_NUMBER")); buf.append(tool.textLine("attribute", "1521")); buf.append(tool.endLine("attribute")); buf.append(tool.startLine("attribute")); buf.append(tool.textLine("code", "QUOTE_ALL_FIELDS")); buf.append(tool.textLine("attribute", "N")); buf.append(tool.endLine("attribute")); buf.append(tool.startLine("attribute")); buf.append(tool.textLine("code", "SUPPORTS_BOOLEAN_DATA_TYPE")); buf.append(tool.textLine("attribute", "N")); buf.append(tool.endLine("attribute")); buf.append(tool.startLine("attribute")); buf.append(tool.textLine("code", "USE_POOLING")); buf.append(tool.textLine("attribute", "N")); buf.append(tool.endLine("attribute")); buf.append(tool.endLine("attributes")); buf.append(tool.endLine("connection")); return buf; } //構造ktr檔案的order節點 private StringBuffer order(Table table) { StringBuffer buf = new StringBuffer(); buf.append(tool.startLine("order")); buf.append(tool.startLine("hop")); buf.append(tool.textLine("from", "INPUT_" + table.getTableName())); buf.append(tool.textLine("to", "OUTPUT_" + table.getTableName())); buf.append(tool.textLine("enabled", "Y")); buf.append(tool.endLine("hop")); buf.append(tool.endLine("order")); return buf; } //構造 表輸入 private StringBuffer tableInput(Table table, String mode) { StringBuffer buf = new StringBuffer(); buf.append(tool.startLine("step")); buf.append(tool.textLine("name", "INPUT_" + table.getTableName())); buf.append(tool.textLine("type", "TableInput")); buf.append(tool.emptyLine("description")); buf.append(tool.textLine("distribute", "Y")); buf.append(tool.textLine("copies", "1")); buf.append(tool.startLine("partitioning")); buf.append(tool.textLine("method", "none")); buf.append(tool.emptyLine("schema_name")); buf.append(tool.endLine("partitioning")); buf.append(tool.textLine("connection", "ploanlt1")); /* if( MODE_ALL.equals(mode) ) { buf.append(tool.textLine("sql", "SELECT * FROM " + table.getTableName() + " WHERE 1 = 1")); } else { buf.append(tool.textLine("sql", "SELECT * FROM " + table.getTableName() + " WHERE LASTDEALDATE = to_date('${UNLOAD_WORKDATE}', 'yyyymmdd')")); } */ /* * 個貸的表中暫時還沒加時間戳，統一用一個SQL語句 * 下面就是填寫的SQL語句 */ buf.append(tool.textLine("sql","SELECT * FROM " + table.getTableName() + " WHERE 1 =1")); buf.append(tool.textLine("limit", "0")); buf.append(tool.emptyLine("lookup")); buf.append(tool.textLine("execute_each_row", "N")); /* if( MODE_ALL.equals(mode) ) { buf.append(tool.textLine("variables_active", "N")); } else { buf.append(tool.textLine("variables_active", "Y")); } */ buf.append(tool.textLine("variables_active","Y")); buf.append(tool.textLine("lazy_conversion_active", "N")); buf.append(tool.emptyLine("cluster_schema")); buf.append(tool.startLine("remotesteps")); buf.append(tool.startLine("input")); buf.append(tool.endLine("input")); buf.append(tool.startLine("output")); buf.append(tool.endLine("output")); buf.append(tool.endLine("remotesteps")); buf.append(tool.startLine("GUI")); buf.append(tool.textLine("xloc", "277")); buf.append(tool.textLine("yloc", "294")); buf.append(tool.textLine("draw", "Y")); buf.append(tool.endLine("GUI")); buf.append(tool.endLine("step")); return buf; } //構造 卸數的 文字檔案輸出 private StringBuffer fileOutput(Table table, String mode) { StringBuffer buf = new StringBuffer(); buf.append(tool.startLine("step")); buf.append(tool.textLine("name", "OUTPUT_" + table.getTableName())); buf.append(tool.textLine("type", "TextFileOutput")); buf.append(tool.emptyLine("description")); buf.append(tool.textLine("distribute", "Y")); buf.append(tool.textLine("copies", "1")); buf.append(tool.startLine("partitioning")); buf.append(tool.textLine("method", "none")); buf.append(tool.emptyLine("schema_name")); buf.append(tool.endLine("partitioning")); buf.append(tool.textLine("separator", "\t")); buf.append(tool.emptyLine("enclosure")); buf.append(tool.textLine("enclosure_forced", "N")); buf.append(tool.textLine("enclosure_fix_disabled", "N")); buf.append(tool.textLine("header", "N")); buf.append(tool.textLine("footer", "N")); buf.append(tool.textLine("format", "Unix")); buf.append(tool.textLine("compression", "None")); buf.append(tool.textLine("encoding", "GBK")); buf.append(tool.emptyLine("endedLine")); buf.append(tool.textLine("fileNameInField", "N")); buf.append(tool.emptyLine("fileNameField")); buf.append(tool.startLine("file")); buf.append(tool.textLine("name", "${UNLOAD_BASE_PATH}/${UNLOAD_WORKDATE}/" + getFullName(table.getTableName(), mode) + ".${UNLOAD_WORKDATE}")); buf.append(tool.textLine("is_command", "N")); buf.append(tool.textLine("do_not_open_new_file_init", "N")); buf.append(tool.emptyLine("extention")); buf.append(tool.textLine("append", "N")); buf.append(tool.textLine("split", "N")); buf.append(tool.textLine("haspartno", "N")); buf.append(tool.textLine("add_date", "N")); buf.append(tool.textLine("add_time", "N")); buf.append(tool.textLine("SpecifyFormat", "N")); buf.append(tool.emptyLine("date_time_format")); buf.append(tool.textLine("add_to_result_filenames", "N")); buf.append(tool.textLine("pad", "N")); buf.append(tool.textLine("fast_dump", "N")); buf.append(tool.textLine("splitevery", "0")); buf.append(tool.endLine("file")); buf.append(tool.startLine("fields")); //開始填寫欄位資訊 for( Column column : table.getColumnList() ) { buf.append(columnField(column)); } //填寫欄位資訊結束 buf.append(tool.endLine("fields")); buf.append(tool.emptyLine("cluster_schema")); buf.append(tool.startLine("remotesteps")); buf.append(tool.startLine("input")); buf.append(tool.endLine("input")); buf.append(tool.startLine("output")); buf.append(tool.endLine("output")); buf.append(tool.endLine("remotesteps")); buf.append(tool.startLine("GUI")); buf.append(tool.textLine("xloc", "511")); buf.append(tool.textLine("yloc", "295")); buf.append(tool.textLine("draw", "Y")); buf.append(tool.endLine("GUI")); buf.append(tool.endLine("step")); return buf; } //定義獲取欄位的函式 private StringBuffer columnField(Column column) { StringBuffer buf = new StringBuffer(); buf.append(tool.startLine("field")); buf.append(tool.textLine("name", column.getName())); //欄位型別以及長度 if( column.getType().startsWith("string") ) { buf.append(tool.textLine("type", "String")); buf.append(tool.emptyLine("format")); } else if( column.getType().startsWith("decimal")||column.getType().startsWith("number") ) { buf.append(tool.textLine("type", "BigNumber")); buf.append(tool.emptyLine("format")); } else if( column.getType().startsWith("int") ) { buf.append(tool.textLine("type", "Integer")); buf.append(tool.textLine("format", "0")); } else if( column.getType().startsWith("date") ) { buf.append(tool.textLine("type", "Date")); if( column.getName().endsWith("TIME") ) { buf.append(tool.textLine("format", "yyyyMMddHHmmss")); } else { buf.append(tool.textLine("format", "yyyyMMdd")); } } else { //System.out.println(table.getTableName()); System.out.println(column.getName() + " : " + column.getType()); } buf.append(tool.emptyLine("currency")); buf.append(tool.emptyLine("decimal")); buf.append(tool.emptyLine("group")); buf.append(tool.emptyLine("nullif")); buf.append(tool.textLine("trim_type", "both")); buf.append(tool.textLine("length", "-1")); buf.append(tool.textLine("precision", "-1")); buf.append(tool.endLine("field")); return buf; } //定義函式，生成KTR檔案 private void geneKTRFile(String tableName, String mode, StringBuffer buf) throws Exception { OutputStream os = new FileOutputStream(new File(KTR_FILE_PATH + "\\" + getFullName(tableName, mode) + ".ktr")); PrintStream ps = new PrintStream(os); ps.print(buf.toString()); ps.close(); os.close(); } //給KTR命名,SPLS_TABLENAME_MODEL.KTR private String getFullName(String tableName, String mode) { return "SPLS_" + tableName + "_" + mode; } //生成KTR檔案主函式 public static void main(String[] args) throws Exception { DataUnloadGenKTR generator = new DataUnloadGenKTR(); generator.start(); } } /*名稱：OutputTool類 *功能：定義上述程式碼中控制輸出的節點函式： startLine(),endLine(),textLine(),emptyLine() */ class OutputTool { private int level; public OutputTool() { level = 0; } private String prefix() { StringBuffer buf = new StringBuffer(); for( int i = 0; i < level; i ++ ) { buf.append("\t"); } return buf.toString(); } //以單獨一行開始一個子節點 public String startLine(String name) { StringBuffer buf = new StringBuffer(prefix()); buf.append("<").append(name).append(">").append("\n"); level ++; return buf.toString(); } //以單獨一行結束一個子節點 public String endLine(String name) { level --; StringBuffer buf = new StringBuffer(prefix()); buf.append("</").append(name).append(">").append("\n"); return buf.toString(); } //在一行內完成一個帶有文字的子節點 public String textLine(String name, String text) { StringBuffer buf = new StringBuffer(prefix()); buf.append("<").append(name).append(">"); buf.append(text); buf.append("</").append(name).append(">").append("\n"); return buf.toString(); } //在一行內完成一個空子節點 public String emptyLine(String name) { StringBuffer buf = new StringBuffer(prefix()); buf.append("<").append(name).append("/>").append("\n"); return buf.toString(); } public String field(String text) { StringBuffer buf = new StringBuffer(); buf.append(startLine("field")); buf.append(textLine("id", "STEP_COPY")); buf.append(textLine("enabled", "Y")); buf.append(textLine("name", "STEP_COPY")); buf.append(endLine("field")); return buf.toString(); } public String fieldSubject(String text) { StringBuffer buf = new StringBuffer(); buf.append(startLine("field")); buf.append(textLine("id", "STEP_COPY")); buf.append(textLine("enabled", "Y")); buf.append(textLine("name", "STEP_COPY")); buf.append(emptyLine("subject")); buf.append(endLine("field")); return buf.toString(); } }