1. 程式人生 > 其它 >Mysql和Hive實現列轉行,類似reduceByKey操作

Mysql和Hive實現列轉行,類似reduceByKey操作

給的資料如下圖:

需要實現的結果(需要對label去重):

mysql:

建表和準備資料

CREATE TABLE `comp_label`  (
  `compName` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci DEFAULT NULL,
  `label` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci DEFAULT NULL
) ENGINE = InnoDB CHARACTER SET = latin1 COLLATE = latin1_swedish_ci ROW_FORMAT =
Compact; INSERT INTO `comp_label` VALUES ('視野數科', 'FourthCompany'); INSERT INTO `comp_label` VALUES ('視野數科', 'PrepareThirdCompany'); INSERT INTO `comp_label` VALUES ('視野數科', 'PrepareThirdCompany'); INSERT INTO `comp_label` VALUES ('視野數科', 'SYBI10000'); INSERT INTO `comp_label` VALUES ('天眼查', 'SYBB10000');
View Code

實現:

SELECT compName,GROUP_CONCAT(DISTINCT label) label FROM comp_label GROUP BY compName

hive:

建表和準備資料

CREATE TABLE `default.comp_label`(
  `compName` string,
  `label` string)
ROW FORMAT SERDE 
  'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' 
WITH SERDEPROPERTIES ( 
  'field.delim'='\u0001
', 'line.delim'='\n', 'serialization.format'='r') STORED AS INPUTFORMAT 'org.apache.hadoop.mapred.TextInputFormat' OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'; INSERT INTO `default.comp_label` VALUES ('視野數科', 'FourthCompany'),('視野數科', 'PrepareThirdCompany'), ('視野數科', 'PrepareThirdCompany'), ('視野數科', 'SYBI10000'),('天眼查', 'SYBB10000');
View Code

實現:

SELECT compName,concat_ws(',',collect_set(label)) AS label FROM default.comp_label GROUP BY compName

hive如果不去重

SELECT compName,concat_ws(',',collect_list(label)) AS label FROM default.comp_label GROUP BY compName
View Code