问题
brokerLoad将一个以day字段为动态分区字段的hive表
导入到doris也是以day字段为动态分区字段的表
结果报错
type:LOAD_RUN_FAIL; msg:errCode = 2, detailMessage = ParseError : Invalid column selected day
doris是不支持导入动态分区hive表嘛
问题解决
问了社区大佬,发现MysqL连接doris后在终端使用help broker load
就能看到有个example显示
到COLUMNS FROM PATH AS (day)
关键词语法
随后建立如下任务
LOAD
LABEL alipay_bill_2021_0915_3
(
DATA INFILE("hdfs://hadoop/user/data/hive_db/data_ods.db/hive_alipay_bill/*/*")
INTO TABLE alipay_bill
COLUMNS TERMINATED BY "\\x01"
FORMAT AS "orc"
(business_id,appid,finance_id,mch_id,time,produce_name,other_account,income,pay,balance,channel,type,note,huabei_number)
COLUMNS FROM PATH AS (day)
SET
(day=day,business_id=business_id,appid=appid,finance_id=finance_id,mch_id=mch_id,time=time,produce_name=produce_name,other_account=other_account,income=income,pay=pay,balance=balance,channel=channel,type=type,note=note,huabei_number=huabei_number)
where 1=1
)
WITH broKER "hdfs_broker"
(
"dfs.nameservices"="hadoop",
"dfs.ha.namenodes.eadhadoop" = "nn1,nn2",
"dfs.namenode.rpc-address.eadhadoop.nn1" = "222:8000",
"dfs.namenode.rpc-address.eadhadoop.nn2" = "117:8000",
"dfs.client.failover.proxy.provider" = "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider",
"hadoop.security.authentication" = "kerberos",
"kerberos_principal" = "ddd.COM",
"kerberos_keytab_content" = "BQHIININ1111URAAE="
)
PROPERTIES
(
"timeout"="1200",
"max_filter_ratio"="0.1"
);
即可