问题描述
我希望能够使用 dbplyr 执行与此等效的操作。
library(magrittr)
library(tidyverse)
library(bigrquery)
library(dbplyr)
#>
#> Attaching package: 'dbplyr'
#> The following objects are masked from 'package:dplyr':
#>
#> ident,sql
bq_deauth()
bq_auth()
bq_conn = dbConnect(
bigquery(),project = "eacri-genomics"
)
df = tibble(
chr = c(1,1,2,3),start = c(0,10,12,5,1),end = c(2,11,15,8,3)
)
df
#> # A tibble: 6 x 3
#> chr start end
#> <dbl> <dbl> <dbl>
#> 1 1 0 2
#> 2 1 10 11
#> 3 1 12 15
#> 4 2 0 1
#> 5 2 5 8
#> 6 3 1 3
df %>%
rowwise() %>% mutate(range = list(seq(start,end)))
#> # A tibble: 6 x 4
#> # Rowwise:
#> chr start end range
#> <dbl> <dbl> <dbl> <list>
#> 1 1 0 2 <int [3]>
#> 2 1 10 11 <int [2]>
#> 3 1 12 15 <int [4]>
#> 4 2 0 1 <int [2]>
#> 5 2 5 8 <int [4]>
#> 6 3 1 3 <int [3]>
df %>%
rowwise() %>% mutate(range = list(seq(start,end))) %>%
unnest(range)
#> # A tibble: 18 x 4
#> chr start end range
#> <dbl> <dbl> <dbl> <int>
#> 1 1 0 2 0
#> 2 1 0 2 1
#> 3 1 0 2 2
#> 4 1 10 11 10
#> 5 1 10 11 11
#> 6 1 12 15 12
#> 7 1 12 15 13
#> 8 1 12 15 14
#> 9 1 12 15 15
#> 10 2 0 1 0
#> 11 2 0 1 1
#> 12 2 5 8 5
#> 13 2 5 8 6
#> 14 2 5 8 7
#> 15 2 5 8 8
#> 16 3 1 3 1
#> 17 3 1 3 2
#> 18 3 1 3 3
dbWriteTable(
bq_conn,name = "balter.range_test",value = df,overwrite = T
)
df_bq = tbl(bq_conn,"balter.range_test")
df_bq %>%
rowwise() %>% mutate(range = list(seq(start,end)))
#> Error in UseMethod("rowwise"): no applicable method for 'rowwise' applied to an object of class "c('tbl_BigQueryConnection','tbl_dbi','tbl_sql','tbl_lazy','tbl')"
df_bq %>%
mutate(range = generate_series(start,end))
#> Error: Job 'eacri-genomics.job_dnmwBmRtY9j0heyY1AtmwEtgblGp.US' Failed
#> x Function not found: generate_series at [1:31] [invalidQuery]
由 reprex package (v1.0.0) 于 2021 年 2 月 18 日创建
解决方法
在尝试使用 dbplyr 执行此操作之前，值得首先考虑您使用的数据库是否支持具有列表/数组类型的列。这是您的 range <list> 列所必需的。
我怀疑 (1) 这个特性在许多数据库中并不常见/广泛支持，并且 (2) dbplyr 目前没有提供直接的翻译。 （例如，请参阅以下两个问题：one 和 two）。
但由于您的序列只是一个数字范围，您可以通过连接完成同样的事情：
df = tibble(
chr = c(1,1,2,3),start = c(0,10,12,5,1),end = c(2,11,15,8,3)
)
whole_range = tibble(range = -100:100)
# need start < min(df$start) and end > max(df$end)
dbWriteTable(conn,name = "df",value = df,overwrite = T)
dbWriteTable(conn,name = "whole_range",value = whole_range,overwrite = T)
remote_df = tbl(conn,"df")
remote_whole_range = tbl(conn,"whole_range")
# create dummy columns to join
remote_df = remote_df %>% mutate(ones = 1)
remote_whole_range = remote_whole_range %>% mutate(ones = 1)
# join and filter
remote_output = inner_join(remote_df,remote_whole_range,by = "ones") %>%
filter(start <= range,range <= end)
我发现了一个不是规范的 postgresql 函数，但在 BigQuery 中可以使用：generate_array。
library(magrittr)
library(tidyverse)
library(dbplyr)
#>
#> Attaching package: 'dbplyr'
#> The following objects are masked from 'package:dplyr':
#>
#> ident,sql
library(bigrquery)
library(DBI)
library(RPostgres)
library(RPostgreSQL)
bq_deauth()
bq_auth(email="ariel.balter@gmail.com")
bq_conn = dbConnect(
bigquery(),project = "elite-magpie-257717",dataset = "test_dataset"
)
df = tibble(
chr = c(1,3)
)
df %>%
rowwise() %>% mutate(range = list(seq(start,end)))
#> # A tibble: 6 x 4
#> # Rowwise:
#> chr start end range
#> <dbl> <dbl> <dbl> <list>
#> 1 1 0 2 <int [3]>
#> 2 1 10 11 <int [2]>
#> 3 1 12 15 <int [4]>
#> 4 2 0 1 <int [2]>
#> 5 2 5 8 <int [4]>
#> 6 3 1 3 <int [3]>
df %>%
rowwise() %>% mutate(range = list(seq(start,end))) %>%
unnest(range)
#> # A tibble: 18 x 4
#> chr start end range
#> <dbl> <dbl> <dbl> <int>
#> 1 1 0 2 0
#> 2 1 0 2 1
#> 3 1 0 2 2
#> 4 1 10 11 10
#> 5 1 10 11 11
#> 6 1 12 15 12
#> 7 1 12 15 13
#> 8 1 12 15 14
#> 9 1 12 15 15
#> 10 2 0 1 0
#> 11 2 0 1 1
#> 12 2 5 8 5
#> 13 2 5 8 6
#> 14 2 5 8 7
#> 15 2 5 8 8
#> 16 3 1 3 1
#> 17 3 1 3 2
#> 18 3 1 3 3
dbWriteTable(
bq_conn,name = "test_dataset.range_test",overwrite = T
)
#> Auto-refreshing stale OAuth token.
df_bq = tbl(bq_conn,"test_dataset.range_test")
df_bq %>%
mutate(range = generate_array(start,end,1))
#> # Source: lazy query [?? x 4]
#> # Database: BigQueryConnection
#> end start chr range
#> <int> <int> <int> <list>
#> 1 2 0 1 <dbl [3]>
#> 2 11 10 1 <dbl [2]>
#> 3 15 12 1 <dbl [4]>
#> 4 1 0 2 <dbl [2]>
#> 5 8 5 2 <dbl [4]>
#> 6 3 1 3 <dbl [3]>
由 reprex package (v1.0.0) 于 2021 年 2 月 19 日创建