问题描述
我正在尝试使用直接运行程序上Apache Beam上的BigQuery I / O连接器在现有BigQuery数据集上执行查询，在该数据库中我试图检查表是否存在于数据集中。代码如下所示：
def check_table(pipeline,table_name):
exists = (
pipeline
| "Verifying BQ table existence in dataset"
>> beam.io.Read(beam.io.BigQuerySource(
query="SELECT EXISTS (SELECT * FROM `test.__TABLES__` WHERE table_id = '{}')".format(table_name),use_standard_sql=True
))
)
return exists
WARNING:apache_beam.io.gcp.bigquery_tools:Dataset xyz:temp_dataset_95f0fd1f2fe845c486d935c7bbb25f37 does not exist so we will create it as temporary with location=unkNown
ERROR:apache_beam.runners.direct.executor:Exception at bundle <apache_beam.runners.direct.bundle_factory._Bundle object at 0x10f680c40>,due to an exception.
Traceback (most recent call last):
File "/Users/souvikdey/.pyenv/versions/3.8.5/envs/athena-venv/lib/python3.8/site-packages/apache_beam/io/gcp/bigquery_tools.py",line 652,in create_temporary_dataset
self.client.datasets.Get(
File "/Users/souvikdey/.pyenv/versions/3.8.5/envs/athena-venv/lib/python3.8/site-packages/apache_beam/io/gcp/internal/clients/bigquery/bigquery_v2_client.py",line 115,in Get
return self._RunMethod(
File "/Users/souvikdey/.pyenv/versions/3.8.5/envs/athena-venv/lib/python3.8/site-packages/apitools/base/py/base_api.py",line 731,in _RunMethod
return self.ProcessHttpResponse(method_config,http_response,request)
File "/Users/souvikdey/.pyenv/versions/3.8.5/envs/athena-venv/lib/python3.8/site-packages/apitools/base/py/base_api.py",line 737,in ProcessHttpResponse
self.__ProcessHttpResponse(method_config,request))
File "/Users/souvikdey/.pyenv/versions/3.8.5/envs/athena-venv/lib/python3.8/site-packages/apitools/base/py/base_api.py",line 603,in __ProcessHttpResponse
raise exceptions.HttpError.Fromresponse(
apitools.base.py.exceptions.HttpNotFoundError: HttpError accessing <https://bigquery.googleapis.com/bigquery/v2/projects/xyz/datasets/temp_dataset_95f0fd1f2fe845c486d935c7bbb25f37?alt=json>: response: <{'vary': 'Origin,X-Origin,Referer','content-type': 'application/json; charset=UTF-8','date': 'Wed,30 Sep 2020 13:37:28 GMT','server': 'ESF','cache-control': 'private','x-xss-protection': '0','x-frame-options': 'SAMEORIGIN','x-content-type-options': 'nosniff','alt-svc': 'h3-Q050=":443"; ma=2592000,h3-29=":443"; ma=2592000,h3-27=":443"; ma=2592000,h3-T051=":443"; ma=2592000,h3-T050=":443"; ma=2592000,h3-Q046=":443"; ma=2592000,h3-Q043=":443"; ma=2592000,quic=":443"; ma=2592000; v="46,43"','transfer-encoding': 'chunked','status': '404','content-length': '374','-content-encoding': 'gzip'}>,content <{
"error": {
"code": 404,"message": "Not found: Dataset xyz:temp_dataset_95f0fd1f2fe845c486d935c7bbb25f37","errors": [
{
"message": "Not found: Dataset xyz:temp_dataset_95f0fd1f2fe845c486d935c7bbb25f37","domain": "global","reason": "notFound"
}
],"status": "NOT_FOUND"
}
}
>
During handling of the above exception,another exception occurred:
Traceback (most recent call last):
File "/Users/souvikdey/.pyenv/versions/3.8.5/envs/athena-venv/lib/python3.8/site-packages/apache_beam/io/gcp/bigquery_tools.py",line 573,in get_or_create_dataset
dataset = self.client.datasets.Get(
File "/Users/souvikdey/.pyenv/versions/3.8.5/envs/athena-venv/lib/python3.8/site-packages/apache_beam/io/gcp/internal/clients/bigquery/bigquery_v2_client.py",30 Sep 2020 13:37:29 GMT',another exception occurred:
Traceback (most recent call last):
File "/Users/souvikdey/.pyenv/versions/3.8.5/envs/athena-venv/lib/python3.8/site-packages/apache_beam/runners/direct/executor.py",line 377,in call
self.attempt_call(
File "/Users/souvikdey/.pyenv/versions/3.8.5/envs/athena-venv/lib/python3.8/site-packages/apache_beam/runners/direct/executor.py",line 423,in attempt_call
result = evaluator.finish_bundle()
File "/Users/souvikdey/.pyenv/versions/3.8.5/envs/athena-venv/lib/python3.8/site-packages/apache_beam/runners/direct/transform_evaluator.py",line 375,in finish_bundle
with self._source.reader() as reader:
File "/Users/souvikdey/.pyenv/versions/3.8.5/envs/athena-venv/lib/python3.8/site-packages/apache_beam/io/gcp/bigquery_tools.py",line 1134,in __enter__
self.client.create_temporary_dataset(
File "/Users/souvikdey/.pyenv/versions/3.8.5/envs/athena-venv/lib/python3.8/site-packages/apache_beam/utils/retry.py",line 236,in wrapper
return fun(*args,**kwargs)
File "/Users/souvikdey/.pyenv/versions/3.8.5/envs/athena-venv/lib/python3.8/site-packages/apache_beam/io/gcp/bigquery_tools.py",line 668,in create_temporary_dataset
self.get_or_create_dataset(project_id,dataset_id,location=location)
File "/Users/souvikdey/.pyenv/versions/3.8.5/envs/athena-venv/lib/python3.8/site-packages/apache_beam/utils/retry.py",line 586,in get_or_create_dataset
response = self.client.datasets.Insert(request)
File "/Users/souvikdey/.pyenv/versions/3.8.5/envs/athena-venv/lib/python3.8/site-packages/apache_beam/io/gcp/internal/clients/bigquery/bigquery_v2_client.py",line 141,in Insert
return self._RunMethod(
File "/Users/souvikdey/.pyenv/versions/3.8.5/envs/athena-venv/lib/python3.8/site-packages/apitools/base/py/base_api.py",in __ProcessHttpResponse
raise exceptions.HttpError.Fromresponse( apitools.base.py.exceptions.HttpForbiddenError: HttpError accessing <https://bigquery.googleapis.com/bigquery/v2/projects/xyz/datasets?alt=json>: response: <{'vary': 'Origin,'status': '403','content-length': '376',content <{
"error": {
"code": 403,"message": "Access Denied: BigQuery BigQuery: Location unkNown is not yet publicly available.","errors": [
{
"message": "Access Denied: BigQuery BigQuery: Location unkNown is not yet publicly available.","reason": "accessDenied"
}
],"status": "PERMISSION_DENIED"
}
}
我错过了什么吗？另外，为什么location=unkNown呢？我已经向服务帐户提供了BigQuery Admin角色以进行授权。
解决方法
暂无找到可以解决该程序问题的有效方法，小编努力寻找整理中！
如果你已经找到好的解决方法，欢迎将解决方案带上本链接一起发送给小编。
小编邮箱:dio#foxmail.com (将#修改为@）