1. 程式人生 > >Hive-JSON 原始資料對映 Hive 表結構

Hive-JSON 原始資料對映 Hive 表結構

參考文件 

https://cwiki.apache.org/confluence/display/Hive/LanguageManual+DDL#LanguageManualDDL-RowFormats&SerDe

 

建立 HIVE-JSON 表

    -- 新增依賴 jar
    ADD JAR /data/tmp/liwei/hive-hcatalog-core-2.1.1.jar;


    DROP TABLE IF EXISTS `tmp.liwei_json_table`;
    CREATE TABLE IF NOT EXISTS tmp.liwei_json_table (
        `data_param`      map<string, string>,
        `account_param`   map<string, string>,
        `role_param`      map<string, string>,
        `event`           map<string, string>
    )
    comment 'liwei_json_table'
    ROW FORMAT SERDE 'org.apache.hive.hcatalog.data.JsonSerDe'
    STORED AS TEXTFILE
    ;


    --報錯  return code 1 from org.apache.hadoop.hive.ql.exec.ddltask. cannot validate serde: org.apache.hive.hcatalog.data.jsonserde
    --解決方案 ,新增對應 jar ,並對新增的 jar 授執行許可權

建立測試 JSON 檔案資料

新增測試JSON資料檔案 hive_json_table.json(需要壓縮為一行)
{
    "data_param": {
      "game_key_s": "sbkcq",
      "game_version_name_s": "1.23",
      "game_version_code_s": "1.23b",
      "data_from_s": "s",
      "create_unix": "1501641267",
      "platform_s": "ios",
      "region_s": "101",
      "server_s": "1011",
      "data_unix": "1501641267",
      "index": "event",
      "type": "progression_8_l",
      "data_date": "2017-08-02",
      "data_time": "10:34:27",
      "data_date_partition": "20170802"
    },
    "account_param": {
      "use_id_s": "1100",
      "account_id_s": "1",
      "account_name_s": "lw",
      "channel_s": "baidu"
    },
    "role_param": {
      "role_id_s": "baidu|110023|1|1011",
      "role_name_s": "傻大姐",
      "level_i": "90",
      "role_class_s": "FaShi",
      "role_race_s": "RZ",
      "fighting_i": "201289",
      "remain_currency_l": "198",
      "vip_level_i": 2
    },
    "event": {
      "t0": "world",
      "t1": "world_z1",
      "t2": "world_z1-2",
      "t3": "world_z1-2-killbos",
      "t4": "difficult",
      "t5": "succeed",
      "r_l": "123000"
    }
  }

{"data_param":{"game_key_s":"sbkcq","game_version_name_s":"1.23","game_version_code_s":"1.23b","data_from_s":"s","create_unix":"1501641267","platform_s":"ios","region_s":"101","server_s":"1011","data_unix":"1501641267","index":"event","type":"progression_8_l","data_date":"2017-08-02","data_time":"10:34:27",
"data_date_partition":"20170802"},"account_param":{"use_id_s":"1100","account_id_s":"1","account_name_s":"lw","channel_s":"baidu"},"role_param":{"role_id_s":"baidu|110023|1|1011","role_name_s":"傻大姐","level_i":"90","role_class_s":"FaShi","role_race_s":"RZ","fighting_i":"201289","remain_currency_l":"198","vip_level_i":2},"event":{"t0":"world","t1":"world_z1","t2":"world_z1-2","t3":"world_z1-2-killbos","t4":"difficult","t5":"succeed","r_l":"123000"}}

上傳資料到 HDFS

hadoop fs -D speed.limit.kb=10240 -copyFromLocal  /data/tmp/liwei/hive_tmp/hive_json_table.json /user/hive/warehouse/tmp.db/liwei_json_table/

執行測試查詢操作

hive> select * from tmp.liwei_json_table;
OK
{"game_key_s":"sbkcq","game_version_name_s":"1.23","game_version_code_s":"1.23b","data_from_s":"s","create_unix":"1501641267","platform_s":"ios","region_s":"101","server_s":"1011","data_unix":"1501641267","index":"event","type":"progression_8_l","data_date":"2017-08-02","data_time":"10:34:27","data_date_partition":"20170802"}{"use_id_s":"1100","account_id_s":"1","account_name_s":"lw","channel_s":"baidu"}	{"role_id_s":"baidu|110023|1|1011","role_name_s":"傻大姐","level_i":"90","role_class_s":"FaShi","role_race_s":"RZ","fighting_i":"201289","remain_currency_l":"198","vip_level_i":"2"}	{"t0":"world","t1":"world_z1","t2":"world_z1-2","t3":"world_z1-2-killbos","t4":"difficult","t5":"succeed","r_l":"123000"}
Time taken: 1.292 seconds, Fetched: 1 row(s)
hive>
    >
    >
    > select data_param from tmp.liwei_json_table;
OK
{"game_key_s":"sbkcq","game_version_name_s":"1.23","game_version_code_s":"1.23b","data_from_s":"s","create_unix":"1501641267","platform_s":"ios","region_s":"101","server_s":"1011","data_unix":"1501641267","index":"event","type":"progression_8_l","data_date":"2017-08-02","data_time":"10:34:27","data_date_partition":"20170802"}
Time taken: 0.121 seconds, Fetched: 1 row(s)
hive> select data_param["game_key_s"] from tmp.liwei_json_table;
OK
sbkcq