0DWS層之使用者行為寬表

1）為什麼要建寬表

需求目標，把每個使用者單日的行為聚合起來組成一張多列寬表，以便之後關聯使用者維度資訊後進行，不同角度的統計分析。

1 建立使用者行為寬表

hive (gmall)>
drop table if exists dws_user_action;
create external table dws_user_action 
(   
    user_id          string      comment '使用者 id',
    order_count     bigint      comment '下單次數 ',
    order_amount    
 
decimal(16,2)  comment '下單金額 ',
    payment_count   bigint      comment '支付次數',
    payment_amount  decimal(16,2) comment '支付金額 '
) COMMENT '每日使用者行為寬表'
PARTITIONED BY (`dt` string)
stored as parquet
location '/warehouse/gmall/dws/dws_user_action/'
tblproperties ("parquet.compression"="snappy");

2 使用者行為資料

寬表匯入指令碼

1）在/root/bin目錄下建立指令碼dws_db_wide.sh

[root@hadoop102 bin]$ vim dws_db_wide.sh

在指令碼中填寫如下內容

#!/bin/bash

# 定義變數方便修改
APP=gmall

# 如果是輸入的日期按照取輸入日期；如果沒輸入日期取當前時間的前一天
if [ -n "$1" ] ;then
    do_date=$1
else 
    do_date=`date -d "-1 day" +%F`  
fi 

sql="
with  
tmp_order as
(
    select 
        user_id, 
        count(
 
*)  order_count,
        sum(oi.total_amount) order_amount
    from "$APP".dwd_order_info oi
    where date_format(oi.create_time,'yyyy-MM-dd')='$do_date'
    group by user_id
) ,
tmp_payment as
(
    select
        user_id, 
        sum(pi.total_amount) payment_amount, 
        count(*) payment_count 
    from "$APP".dwd_payment_info pi 
    where date_format(pi.payment_time,'yyyy-MM-dd')='$do_date'
    group by user_id
)
insert overwrite table "$APP".dws_user_action partition(dt='$do_date')
select
    user_actions.user_id,
    sum(user_actions.order_count),
    sum(user_actions.order_amount),
    sum(user_actions.payment_count),
    sum(user_actions.payment_amount)
from 
(
    select
        user_id,
        order_count,
        order_amount,
        0 payment_count,
        0 payment_amount
    from tmp_order

    union all
    select
        user_id,
        0 order_count,
        0 order_amount,
        payment_count,
        payment_amount
    from tmp_payment
 ) user_actions
group by user_id;
"

beeline -u "jdbc:hive2://hadoop102:10000/" -n hive -e "$sql"

2）增加指令碼執行許可權

[root@hadoop102 bin]#chmod 777 dws_db_wide.sh

3）增加指令碼執行許可權

[root@hadoop102 bin]#dws_db_wide.sh 2019-02-10