通過資料庫批量向kettle中插入資料，建立trans和job的模板（按照不同的要求需要自行調整）
阿新 • • 發佈：2018-11-02
import psycopg2 # 用來操作資料庫的類 class GPCommand(object): # 類的初始化 def __init__(self): self.hostname = 'XXX.XX.X.XX' self.username = 'XX' self.password = 'XX' self.database = 'XX' def connectGp(self): try: #連結資料庫 #讀取配置利用connect連結資料庫 self.connect = psycopg2.connect( host=self.hostname, user=self.username, password=self.password, dbname=self.database ) #建立一個新的cursor self.cursor = self.connect.cursor() print("connect gp successful."+'\n' + '資料庫連線成功') return ('con_successful') except psycopg2.Error: error = 'Failed to setup Postgres environment.\n{0}'.format(sys.exc_info()) print('connect gp error.'+'\n' + '資料庫連線失敗') return 'con_error'+ error
def insertTrans(self,trans_name,url_name): try: select_url_sql = "SELECT id_directory FROM r_directory WHERE DIRECTORY_NAME = '%s'"%(url_name) self.cursor.execute(select_url_sql) select_url = self.cursor.fetchall() for item in select_url: url_name_id = item[0] schema = 'DW_'+ trans_name[0:3] table_name = trans_name trans_name = 'T_'+ trans_name id_transformation_sql = 'select max(id_transformation) as id_transformation from r_transformation' id_step_sql = 'select max(id_step) as id_step from r_step' id_trans_hop_sql = 'select max(id_trans_hop) as id_trans_hop from r_trans_hop' id_step_attribute_sql = 'select max(id_step_attribute) as id_step_attribute from r_step_attribute' self.cursor.execute(id_step_attribute_sql) id_step_attribute = self.cursor.fetchall() for item in id_step_attribute: max_id_step_attribute = item[0] + 1 self.cursor.execute(id_transformation_sql) id_transformation = self.cursor.fetchall() for item in id_transformation: max_id_transformation = item[0]+1 self.cursor.execute(id_step_sql) id_step = self.cursor.fetchall() for item in id_step: max_id_step = item[0] + 1 self.cursor.execute(id_trans_hop_sql) id_trans_hop = self.cursor.fetchall() for item in id_trans_hop: max_id_trans_hop = item[0] + 1 #插入trans名稱和id insert_name_sql = "insert into r_transformation(id_transformation,id_directory,\"NAME\",trans_status,id_database_log,use_batchid," \ "use_logfield,id_database_maxdate,offset_maxdate,diff_maxdate,created_user,created_date," \ "modified_user,modified_date,size_rowset) values( %d,%d,'%s',0,-1,'t','t',-1,0,0,'admin',now(),'admin',now(),10000)" \ % (max_id_transformation,url_name_id,trans_name) self.cursor.execute(insert_name_sql) #插入trans的step #表輸入 insert_step_sql = "insert into r_step (id_step,id_transformation,\"NAME\",id_step_type,distribute,copies,gui_location_x,gui_location_y," \ "gui_draw,copies_string) values(%d,%d,'表輸入',140,'t',1,300,380,'t',1)" %(max_id_step,max_id_transformation) self.cursor.execute(insert_step_sql) #表輸出 insert_step_sql = "insert into r_step (id_step,id_transformation,\"NAME\",id_step_type,distribute,copies,gui_location_x,gui_location_y," \ "gui_draw,copies_string) values(%d,%d,'表輸出',11,'t',1,624,384,'t',1)" % (max_id_step+1,max_id_transformation) self.cursor.execute(insert_step_sql) #插入step屬性 insert_stepattr_sql = \ "insert into r_step_attribute (id_step_attribute,id_transformation,id_step,nr,code,value_num,value_str) values(%d,%d,%d,0,'PARTITIONING_SCHEMA',0,0);" \ "insert into r_step_attribute (id_step_attribute,id_transformation,id_step,nr,code,value_num,value_str) values(%d,%d,%d,0,'PARTITIONING_METHOD',0,'none');" \ "insert into r_step_attribute (id_step_attribute,id_transformation,id_step,nr,code,value_num,value_str) values(%d,%d,%d,0,'limit',0,0);" \ "insert into r_step_attribute (id_step_attribute,id_transformation,id_step,nr,code,value_num,value_str) values(%d,%d,%d,0,'lookup',0,null);" \ "insert into r_step_attribute (id_step_attribute,id_transformation,id_step,nr,code,value_num,value_str) values(%d,%d,%d,0,'execute_each_row',0,'N');" \ "insert into r_step_attribute (id_step_attribute,id_transformation,id_step,nr,code,value_num,value_str) values(%d,%d,%d,0,'variables_active',0,'Y');" \ "insert into r_step_attribute (id_step_attribute,id_transformation,id_step,nr,code,value_num,value_str) values(%d,%d,%d,0,'lazy_conversion_active',0,'N');" \ "insert into r_step_attribute (id_step_attribute,id_transformation,id_step,nr,code,value_num,value_str) values(%d,%d,%d,0,'cluster_schema',0,null);" \ "insert into r_step_attribute (id_step_attribute,id_transformation,id_step,nr,code,value_num,value_str) values(%d,%d,%d,0,'row_distribution_code',0,null);" \ "insert into r_step_attribute (id_step_attribute,id_transformation,id_step,nr,code,value_num,value_str) values(%d,%d,%d,0,'id_connection',19,null);" \ "insert into r_step_attribute (id_step_attribute,id_transformation,id_step,nr,code,value_num,value_str) values(%d,%d,%d,0,'sql',0,'select 1 as id');" \ "insert into r_step_attribute (id_step_attribute,id_transformation,id_step,nr,code,value_num,value_str) values(%d,%d,%d,0,'id_connection',18,null);" \ "insert into r_step_attribute (id_step_attribute,id_transformation,id_step,nr,code,value_num,value_str) values(%d,%d,%d,0,'schema',0,'%s');" \ "insert into r_step_attribute (id_step_attribute,id_transformation,id_step,nr,code,value_num,value_str) values(%d,%d,%d,0,'table',0,'%s');" \ "insert into r_step_attribute (id_step_attribute,id_transformation,id_step,nr,code,value_num,value_str) values(%d,%d,%d,0,'commit',0,1000);" \ "insert into r_step_attribute (id_step_attribute,id_transformation,id_step,nr,code,value_num,value_str) values(%d,%d,%d,0,'truncate',0,'Y');" \ "insert into r_step_attribute (id_step_attribute,id_transformation,id_step,nr,code,value_num,value_str) values(%d,%d,%d,0,'column_name',0,'id');" \ "insert into r_step_attribute (id_step_attribute,id_transformation,id_step,nr,code,value_num,value_str) values(%d,%d,%d,0,'stream_name',0,'id');" \ "insert into r_step_attribute (id_step_attribute,id_transformation,id_step,nr,code,value_num,value_str) values(%d,%d,%d,0,'PARTITIONING_SCHEMA',0,null);" \ "insert into r_step_attribute (id_step_attribute,id_transformation,id_step,nr,code,value_num,value_str) values(%d,%d,%d,0,'PARTITIONING_METHOD',0,'none');" \ "insert into r_step_attribute (id_step_attribute,id_transformation,id_step,nr,code,value_num,value_str) values(%d,%d,%d,0,'ignore_errors',0,'N');" \ "insert into r_step_attribute (id_step_attribute,id_transformation,id_step,nr,code,value_num,value_str) values(%d,%d,%d,0,'use_batch',0,'Y');" \ "insert into r_step_attribute (id_step_attribute,id_transformation,id_step,nr,code,value_num,value_str) values(%d,%d,%d,0,'specify_fields',0,'Y');" \ "insert into r_step_attribute (id_step_attribute,id_transformation,id_step,nr,code,value_num,value_str) values(%d,%d,%d,0,'partitioning_enabled',0,'N');" \ "insert into r_step_attribute (id_step_attribute,id_transformation,id_step,nr,code,value_num,value_str) values(%d,%d,%d,0,'partitioning_field',0,NULL);" \ "insert into r_step_attribute (id_step_attribute,id_transformation,id_step,nr,code,value_num,value_str) values(%d,%d,%d,0,'partitioning_daily',0,'N');" \ "insert into r_step_attribute (id_step_attribute,id_transformation,id_step,nr,code,value_num,value_str) values(%d,%d,%d,0,'partitioning_monthly',0,'Y');" \ "insert into r_step_attribute (id_step_attribute,id_transformation,id_step,nr,code,value_num,value_str) values(%d,%d,%d,0,'tablename_in_field',0,'N');" \ "insert into r_step_attribute (id_step_attribute,id_transformation,id_step,nr,code,value_num,value_str) values(%d,%d,%d,0,'tablename_field',0,NULL);" \ "insert into r_step_attribute (id_step_attribute,id_transformation,id_step,nr,code,value_num,value_str) values(%d,%d,%d,0,'tablename_in_table',0,'Y');" \ "insert into r_step_attribute (id_step_attribute,id_transformation,id_step,nr,code,value_num,value_str) values(%d,%d,%d,0,'return_keys',0,'N');" \ "insert into r_step_attribute (id_step_attribute,id_transformation,id_step,nr,code,value_num,value_str) values(%d,%d,%d,0,'return_field',0,NULL);" \ "insert into r_step_attribute (id_step_attribute,id_transformation,id_step,nr,code,value_num,value_str) values(%d,%d,%d,0,'cluster_schema',0,null);" \ "insert into r_step_attribute (id_step_attribute,id_transformation,id_step,nr,code,value_num,value_str) values(%d,%d,%d,0,'row_distribution_code',0,null); " \ "" \ "insert into r_trans_hop (id_trans_hop,id_transformation,id_step_from,id_step_to,enabled) values (%d,%d,%d,%d,'t');"\ % (max_id_step_attribute,max_id_transformation,max_id_step, max_id_step_attribute+1,max_id_transformation,max_id_step, max_id_step_attribute+2,max_id_transformation,max_id_step, max_id_step_attribute+3,max_id_transformation,max_id_step, max_id_step_attribute+4,max_id_transformation,max_id_step, max_id_step_attribute+5,max_id_transformation, max_id_step, max_id_step_attribute+6,max_id_transformation,max_id_step, max_id_step_attribute+7,max_id_transformation,max_id_step, max_id_step_attribute+8,max_id_transformation,max_id_step, max_id_step_attribute+9,max_id_transformation,max_id_step, max_id_step_attribute+10,max_id_transformation,max_id_step, max_id_step_attribute+11,max_id_transformation, max_id_step+1, max_id_step_attribute+12,max_id_transformation, max_id_step+1,schema, max_id_step_attribute+13,max_id_transformation, max_id_step+1,table_name, max_id_step_attribute+14,max_id_transformation, max_id_step+1, max_id_step_attribute+15,max_id_transformation, max_id_step + 1, max_id_step_attribute+16,max_id_transformation, max_id_step+1, max_id_step_attribute+17,max_id_transformation, max_id_step+1, max_id_step_attribute+18,max_id_transformation, max_id_step+1, max_id_step_attribute+19,max_id_transformation, max_id_step + 1, max_id_step_attribute+20,max_id_transformation, max_id_step+1, max_id_step_attribute+21,max_id_transformation, max_id_step+1, max_id_step_attribute+22,max_id_transformation, max_id_step+1, max_id_step_attribute+23,max_id_transformation, max_id_step + 1, max_id_step_attribute+24,max_id_transformation, max_id_step+1, max_id_step_attribute+25,max_id_transformation, max_id_step+1, max_id_step_attribute+26,max_id_transformation, max_id_step+1, max_id_step_attribute+27,max_id_transformation, max_id_step + 1, max_id_step_attribute+28,max_id_transformation, max_id_step+1, max_id_step_attribute+29,max_id_transformation, max_id_step+1, max_id_step_attribute+30,max_id_transformation, max_id_step+1, max_id_step_attribute+31,max_id_transformation, max_id_step + 1, max_id_step_attribute+32,max_id_transformation, max_id_step+1, max_id_step_attribute+33,max_id_transformation, max_id_step+1, max_id_trans_hop,max_id_transformation,max_id_step,max_id_step+1) self.cursor.execute(insert_stepattr_sql) self.connect.commit() return 'insert_success' except psycopg2.Error: error = 'Failed to setup Postgres environment.\n{0}'.format(sys.exc_info()) return 'insert_failed' + error def insertJobs(self,tablename,url_name): try: select_url_sql = "SELECT id_directory FROM r_directory WHERE DIRECTORY_NAME = '%s'" % (url_name) self.cursor.execute(select_url_sql) select_url = self.cursor.fetchall() for item in select_url: url_name_id = item[0] job_name = 'J_'+ tablename id_job_sql = 'select max(id_job) as id_job from r_job' id_jobentry_sql = 'select max(id_jobentry) as id_jobentry from r_jobentry ' id_jobentry_copy_sql = 'select max(id_jobentry_copy) as id_jobentry_copy from r_jobentry_copy' id_jobentry_attribute_sql = "select max(id_jobentry_attribute) as id_jobentry_attribute from r_jobentry_attribute" self.cursor.execute(id_jobentry_attribute_sql) id_jobentry_attribute = self.cursor.fetchall() for item in id_jobentry_attribute: max_id_jobentry_attribute = item[0] + 1 self.cursor.execute(id_job_sql) id_job = self.cursor.fetchall() for item in id_job: max_id_job = item[0]+1 self.cursor.execute(id_jobentry_sql) id_jobentry = self.cursor.fetchall() for item in id_jobentry: max_id_jobentry = item[0] + 1 self.cursor.execute(id_jobentry_copy_sql) id_jobentry_copy = self.cursor.fetchall() for item in id_jobentry_copy: max_id_jobentry_copy = item[0] + 1 # 插入job的id，目錄id，名稱，建立時間等資訊 insert_name_sql = "insert into r_job(id_job,id_directory,\"NAME\",job_status,id_database_log,created_date,modified_user," \ "modified_date,use_batch_id,pass_batch_id,use_logfield) " \ "values(%d,%d,'%s',0,-1,now(),'admin',now(),'t','f','t'); " \ % (max_id_job,url_name_id,job_name) self.cursor.execute(insert_name_sql) #向job中插入trans insert_trans_sql = "insert into r_jobentry (id_jobentry,id_job,id_jobentry_type,\"NAME\") values(%d,%d,7,'START');" \ "insert into r_jobentry (id_jobentry,id_job,id_jobentry_type,\"NAME\") values(%d,%d,48,'T_EDW_PUB_ETL_INCREMENT_TIME_GET');" \ "insert into r_jobentry (id_jobentry,id_job,id_jobentry_type,\"NAME\") values(%d,%d,48,'%s');" \ "insert into r_jobentry (id_jobentry,id_job,id_jobentry_type,\"NAME\") values(%d,%d,48,'T_EDW_PUB_ETL_INCREMENT_TIME_UPDATE');"\ %(max_id_jobentry,max_id_job,max_id_jobentry+1,max_id_job,max_id_jobentry+2,max_id_job,'T_'+tablename,max_id_jobentry+3,max_id_job) self.cursor.execute(insert_trans_sql) #插入trans的顯示座標 insert_loc_sql = \ "insert into r_jobentry_copy (id_jobentry_copy,id_jobentry,id_job,id_jobentry_type,nr,gui_location_x,gui_location_y,gui_draw,parallel) values (%d,%d,%d,7,0,224,288,'t','f');" \ "insert into r_jobentry_copy (id_jobentry_copy,id_jobentry,id_job,id_jobentry_type,nr,gui_location_x,gui_location_y,gui_draw,parallel) values (%d,%d,%d,48,0,384,192,'t','f');" \ "insert into r_jobentry_copy (id_jobentry_copy,id_jobentry,id_job,id_jobentry_type,nr,gui_location_x,gui_location_y,gui_draw,parallel) values (%d,%d,%d,48,0,608,304,'t','f');" \ "insert into r_jobentry_copy (id_jobentry_copy,id_jobentry,id_job,id_jobentry_type,nr,gui_location_x,gui_location_y,gui_draw,parallel) values (%d,%d,%d,48,0,768,208,'t','f');" \ %(max_id_jobentry_copy,max_id_jobentry_copy,max_id_job, max_id_jobentry_copy+1, max_id_jobentry_copy+1, max_id_job, max_id_jobentry_copy+2, max_id_jobentry_copy+2, max_id_job, max_id_jobentry_copy+3, max_id_jobentry_copy+3, max_id_job) self.cursor.execute(insert_loc_sql) #插入trans的屬性 start insert_start_sql = "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'start',0,'Y');" \ "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'dummy',0,'N');" \ "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'repeat',0,'N');" \ "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'schedulerType',0,null);" \ "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'intervalSeconds',0,null);" \ "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'intervalMinutes',60,null);" \ "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'hour',12,null);" \ "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'minutes',0,null);" \ "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'weekDay',1,null);" \ "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'dayOfMonth',1,null);"\ %(max_id_jobentry_attribute,max_id_job,max_id_jobentry, max_id_jobentry_attribute+1,max_id_job,max_id_jobentry, max_id_jobentry_attribute+2,max_id_job,max_id_jobentry, max_id_jobentry_attribute+3,max_id_job,max_id_jobentry, max_id_jobentry_attribute+4,max_id_job,max_id_jobentry, max_id_jobentry_attribute+5,max_id_job,max_id_jobentry, max_id_jobentry_attribute+6,max_id_job,max_id_jobentry, max_id_jobentry_attribute+7,max_id_job,max_id_jobentry, max_id_jobentry_attribute+8,max_id_job,max_id_jobentry, max_id_jobentry_attribute+9,max_id_job,max_id_jobentry) self.cursor.execute(insert_start_sql) # 插入get時間的sql insert_get_sql ="insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'specification_method',0,'rep_name');"\ "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'trans_object_id',0,null);"\ "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'name',0,'T_EDW_PUB_ETL_INCREMENT_TIME_GET');"\ "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'dir_path',0,'/GSDW_EDW_PUB');"\ "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'file_name',0,null);"\ "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'arg_from_previous',0,'N');"\ "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'params_from_previous',0,'N');"\ "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'exec_per_row',0,'N');"\ "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'clear_rows',0,'N');"\ "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'clear_files',0,'N');"\ "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'set_logfile',0,'N');"\ "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'add_date',0,'N');"\ "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'add_time',0,'N');"\ "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'logfile',0,null);"\ "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'logext',0,null);"\ "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'loglevel',0,'Basic');"\ "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'cluster',0,'N');"\ "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'slave_server_name',0,null);"\ "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'set_append_logfile',0,'N');"\ "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'wait_until_finished',0,'Y');"\ "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'follow_abort_remote',0,'N');"\ "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'create_parent_folder',0,'N');"\ "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'logging_remote_work',0,'N');"\ "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'pass_all_parameters',0,'Y');"\ %(max_id_jobentry_attribute+10,max_id_job,max_id_jobentry+1, max_id_jobentry_attribute+11,max_id_job,max_id_jobentry+1, max_id_jobentry_attribute+12,max_id_job,max_id_jobentry+1, max_id_jobentry_attribute+13,max_id_job,max_id_jobentry+1, max_id_jobentry_attribute+14,max_id_job,max_id_jobentry+1, max_id_jobentry_attribute+15,max_id_job,max_id_jobentry+1, max_id_jobentry_attribute+16,max_id_job,max_id_jobentry+1, max_id_jobentry_attribute+17,max_id_job,max_id_jobentry+1, max_id_jobentry_attribute+18,max_id_job,max_id_jobentry+1, max_id_jobentry_attribute+19,max_id_job,max_id_jobentry+1, max_id_jobentry_attribute+20,max_id_job,max_id_jobentry+1, max_id_jobentry_attribute+21,max_id_job,max_id_jobentry+1, max_id_jobentry_attribute+22,max_id_job,max_id_jobentry+1, max_id_jobentry_attribute+23,max_id_job,max_id_jobentry+1, max_id_jobentry_attribute+24,max_id_job,max_id_jobentry+1, max_id_jobentry_attribute+28,max_id_job,max_id_jobentry+1, max_id_jobentry_attribute+26,max_id_job,max_id_jobentry+1, max_id_jobentry_attribute+27,max_id_job,max_id_jobentry+1, max_id_jobentry_attribute+28,max_id_job,max_id_jobentry+1, max_id_jobentry_attribute+29,max_id_job,max_id_jobentry+1, max_id_jobentry_attribute+30,max_id_job,max_id_jobentry+1, max_id_jobentry_attribute+31,max_id_job,max_id_jobentry+1, max_id_jobentry_attribute+32,max_id_job,max_id_jobentry+1, max_id_jobentry_attribute+33,max_id_job,max_id_jobentry+1) self.cursor.execute(insert_get_sql) #插入要執行的trans insert_exec_trans_sql = "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'name',0,'%s');"\ "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'dir_path',0,'%s');"\ "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'specification_method',0,'rep_name');"\ "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'trans_object_id',0,null);"\ "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'file_name',0,null);"\ "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'arg_from_previous',0,'N');"\ "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'params_from_previous',0,'N');"\ "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'exec_per_row',0,'N');"\ "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'clear_rows',0,'N');"\ "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'clear_files',0,'N');"\ "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'set_logfile',0,'N');"\ "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'add_date',0,'N');"\ "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'add_time',0,'N');"\ "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'logfile',0,null);"\ "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'logext',0,null);"\ "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'loglevel',0,'Basic');"\ "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'cluster',0,'N');"\ "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'slave_server_name',0,null);"\ "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'set_append_logfile',0,'N');"\ "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'wait_until_finished',0,'Y');"\ "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'follow_abort_remote',0,'N');"\ "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'create_parent_folder',0,'N');"\ "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'logging_remote_work',0,'N');"\ "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'pass_all_parameters',0,'Y');"\ %(max_id_jobentry_attribute+81,max_id_job,max_id_jobentry+2,'T_' + tablename, max_id_jobentry_attribute+34,max_id_job, max_id_jobentry + 2,'/'+ url_name, max_id_jobentry_attribute+35,max_id_job,max_id_jobentry+2, max_id_jobentry_attribute+36,max_id_job,max_id_jobentry+2, max_id_jobentry_attribute+37,max_id_job,max_id_jobentry+2, max_id_jobentry_attribute+38,max_id_job,max_id_jobentry+2, max_id_jobentry_attribute+39,max_id_job, max_id_jobentry + 2, max_id_jobentry_attribute+40,max_id_job,max_id_jobentry+2, max_id_jobentry_attribute+41,max_id_job,max_id_jobentry+2, max_id_jobentry_attribute+42,max_id_job,max_id_jobentry+2, max_id_jobentry_attribute+43,max_id_job,max_id_jobentry+2, max_id_jobentry_attribute+44,max_id_job, max_id_jobentry + 2, max_id_jobentry_attribute+45,max_id_job,max_id_jobentry+2, max_id_jobentry_attribute+46,max_id_job,max_id_jobentry+2, max_id_jobentry_attribute+47,max_id_job,max_id_jobentry+2, max_id_jobentry_attribute+48,max_id_job,max_id_jobentry+2, max_id_jobentry_attribute+49,max_id_job, max_id_jobentry + 2, max_id_jobentry_attribute+50,max_id_job,max_id_jobentry+2, max_id_jobentry_attribute+51,max_id_job,max_id_jobentry+2, max_id_jobentry_attribute+52,max_id_job,max_id_jobentry+2, max_id_jobentry_attribute+53,max_id_job,max_id_jobentry+2, max_id_jobentry_attribute+54,max_id_job, max_id_jobentry + 2, max_id_jobentry_attribute+55,max_id_job,max_id_jobentry+2, max_id_jobentry_attribute+56,max_id_job,max_id_jobentry+2,) self.cursor.execute(insert_exec_trans_sql) #插入update時間的sql insert_update_sql = "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'specification_method',0,'rep_name');"\ "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'trans_object_id',0,null);"\ "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'name',0,'T_EDW_PUB_ETL_INCREMENT_TIME_UPDATE');"\ "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'dir_path',0,'/GSDW_EDW_PUB');"\ "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'file_name',0,null);"\ "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'arg_from_previous',0,'N');"\ "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'params_from_previous',0,'N');"\ "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'exec_per_row',0,'N');"\ "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'clear_rows',0,'N');"\ "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'clear_files',0,'N');"\ "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'set_logfile',0,'N');"\ "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'add_date',0,'N');"\ "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'add_time',0,'N');"\ "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'logfile',0,null);"\ "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'logext',0,null);"\ "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'loglevel',0,'Basic');"\ "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'cluster',0,'N');"\ "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'slave_server_name',0,null);"\ "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'set_append_logfile',0,'N');"\ "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'wait_until_finished',0,'Y');"\ "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'follow_abort_remote',0,'N');"\ "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'create_parent_folder',0,'N');"\ "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'logging_remote_work',0,'N');"\ "insert into r_jobentry_attribute (id_jobentry_attribute,id_job,id_jobentry,nr,code,value_num,value_str) values(%d,%d,%d,0,'pass_all_parameters',0,'Y');" \ % (max_id_jobentry_attribute+57,max_id_job, max_id_jobentry + 3, max_id_jobentry_attribute+58,max_id_job, max_id_jobentry + 3, max_id_jobentry_attribute+59,max_id_job, max_id_jobentry + 3, max_id_jobentry_attribute+60,max_id_job, max_id_jobentry + 3, max_id_jobentry_attribute+61,max_id_job, max_id_jobentry + 3, max_id_jobentry_attribute+62,max_id_job, max_id_jobentry + 3, max_id_jobentry_attribute+63,max_id_job, max_id_jobentry + 3, max_id_jobentry_attribute+64,max_id_job, max_id_jobentry + 3, max_id_jobentry_attribute+65,max_id_job,max_id_jobentry + 3, max_id_jobentry_attribute+66,max_id_job, max_id_jobentry + 3, max_id_jobentry_attribute+67,max_id_job,max_id_jobentry + 3, max_id_jobentry_attribute+68,max_id_job, max_id_jobentry + 3, max_id_jobentry_attribute+69,max_id_job, max_id_jobentry + 3, max_id_jobentry_attribute+70,max_id_job, max_id_jobentry + 3, max_id_jobentry_attribute+71,max_id_job,max_id_jobentry + 3, max_id_jobentry_attribute+72,max_id_job, max_id_jobentry + 3, max_id_jobentry_attribute+73,max_id_job,max_id_jobentry + 3, max_id_jobentry_attribute+74,max_id_job, max_id_jobentry + 3, max_id_jobentry_attribute+75,max_id_job, max_id_jobentry + 3, max_id_jobentry_attribute+76,max_id_job, max_id_jobentry + 3, max_id_jobentry_attribute+77,max_id_job,max_id_jobentry + 3, max_id_jobentry_attribute+78,max_id_job, max_id_jobentry + 3, max_id_jobentry_attribute+79,max_id_job,max_id_jobentry + 3, max_id_jobentry_attribute+80,max_id_job, max_id_jobentry + 3) self.cursor.execute(insert_update_sql) #插入job的連線線 insert_hop_sql ="insert into r_job_hop (id_job_hop,id_job,id_jobentry_copy_from,id_jobentry_copy_to,enabled,evaluation,unconditional) values (%d,%d,%d,%d,'t','t','t');"\ "insert into r_job_hop (id_job_hop,id_job,id_jobentry_copy_from,id_jobentry_copy_to,enabled,evaluation,unconditional) values (%d,%d,%d,%d,'t','t','t');"\ "insert into r_job_hop (id_job_hop,id_job,id_jobentry_copy_from,id_jobentry_copy_to,enabled,evaluation,unconditional) values (%d,%d,%d,%d,'t','t','t');"\ %(max_id_jobentry_attribute+82,max_id_job,max_id_jobentry,max_id_jobentry+1, max_id_jobentry_attribute+83,max_id_job, max_id_jobentry+1, max_id_jobentry + 2, max_id_jobentry_attribute+84,max_id_job, max_id_jobentry+2, max_id_jobentry + 3) self.cursor.execute(insert_hop_sql) self.connect.commit() return 'insert_success' except psycopg2.Error: error = 'Failed to setup Postgres environment.\n{0}'.format(sys.exc_info()) return 'insert_failed' + error #關閉資料庫 def closeMysql(self): self.cursor.close() self.connect.close() print("資料庫已關閉")
def main(): gpCommand = GPCommand() connect_result = gpCommand.connectGp() #1、建立trans，輸入建立的trans的名稱 #輸入所有的trans名稱，迴圈名稱，向資料庫中插入資料 table_names = ['STG_ZHUDONG_TEST'] #目錄名稱 url_name = 'TAMP' for item_trans_name in table_names: #先新建trans gpCommand.insertTrans(item_trans_name,url_name) #再建立job gpCommand.insertJobs(item_trans_name,url_name) # 最後一定要要把資料關閉 gpCommand.closeMysql() main()