Mysql和Hive實現列轉行,類似reduceByKey操作
阿新 • • 發佈:2021-09-28
給的資料如下圖:
需要實現的結果(需要對label去重):
mysql:
建表和準備資料
CREATE TABLE `comp_label` ( `compName` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci DEFAULT NULL, `label` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci DEFAULT NULL ) ENGINE = InnoDB CHARACTER SET = latin1 COLLATE = latin1_swedish_ci ROW_FORMAT =View CodeCompact; INSERT INTO `comp_label` VALUES ('視野數科', 'FourthCompany'); INSERT INTO `comp_label` VALUES ('視野數科', 'PrepareThirdCompany'); INSERT INTO `comp_label` VALUES ('視野數科', 'PrepareThirdCompany'); INSERT INTO `comp_label` VALUES ('視野數科', 'SYBI10000'); INSERT INTO `comp_label` VALUES ('天眼查', 'SYBB10000');
實現:
SELECT compName,GROUP_CONCAT(DISTINCT label) label FROM comp_label GROUP BY compName
hive:
建表和準備資料
CREATE TABLE `default.comp_label`( `compName` string, `label` string) ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' WITH SERDEPROPERTIES ( 'field.delim'='\u0001View Code', 'line.delim'='\n', 'serialization.format'='r') STORED AS INPUTFORMAT 'org.apache.hadoop.mapred.TextInputFormat' OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'; INSERT INTO `default.comp_label` VALUES ('視野數科', 'FourthCompany'),('視野數科', 'PrepareThirdCompany'), ('視野數科', 'PrepareThirdCompany'), ('視野數科', 'SYBI10000'),('天眼查', 'SYBB10000');
實現:
SELECT compName,concat_ws(',',collect_set(label)) AS label FROM default.comp_label GROUP BY compName
hive如果不去重
SELECT compName,concat_ws(',',collect_list(label)) AS label FROM default.comp_label GROUP BY compNameView Code