hive 動態分割槽及load和insert用法
hive load用法:
load data local inpath '/home/data/stg_activity_antirush_apply.txt'
overwrite into table stg_activity_antirush_apply;
關鍵字local 不加資料從hdfs上載入,如果加local資料從本地載入;
關鍵字overwrite 為覆蓋載入,會覆蓋掉原有的資料;
動態分割槽設定
也可以在配置檔案中修改
--SET hive.exec.dynamic.partition=true;
--SET hive.exec.dynamic.partition.mode=nonstrict;
--SET hive.exec.max.dynamic.partitions.pernode = 1000;
--SET hive.exec.max.dynamic.partitions=1000;
<property>
<name>hive.exec.dynamic.partition</name>
<value>true</value>
<description>Whether or not to allow dynamic partitions in DML/DDL.</description>
</property>
<property>
<name>hive.exec.dynamic.partition.mode</name>
<value>nonstrict</value>
<description>
In strict mode, the user must specify at least one static partition
in case the user accidentally overwrites all partitions.
In nonstrict mode all partitions are allowed to be dynamic.
</description>
</property>
<property>
<name>hive.exec.max.dynamic.partitions</name>
<value>100000</value>
<description>Maximum number of dynamic partitions allowed to be created in total.</description>
</property>
<property>
<name>hive.exec.max.dynamic.partitions.pernode</name>
<value>10000</value>
<description>Maximum number of dynamic partitions allowed to be created in each mapper/reducer node.</description>
</property>
<property>
<name>hive.exec.max.created.files</name>
<value>100000</value>
<description>Maximum number of HDFS files created by all mappers/reducers in a MapReduce job.</description>
</property>
insert overwrite table dw_activity_antirush_apply PARTITION(DateID = ${etlDate}) --手動分割槽
select * from tablename;
insert into table dw_activity_antirush_apply PARTITION(DateID = ${etlDate}) --手動分割槽
select * from tablename;
insert overwrite table dw_activity_antirush_apply PARTITION(DateID) --動態分割槽
select *, column_name as dateid from tablename;
INSERT OVERWRITE會覆蓋表或分割槽中已存在的資料
INSERT INTO以追加資料的方式插入到表或分割槽,原有資料不會刪除
Insert可以插入表或分割槽,如果表是分割槽表,則Insert時需要指定插入到哪個分割槽