The goal of this vignette is explain how to use ResamplingSameOtherSizesCV for various kinds of cross-validation.

Comparing training on Same/Other/All subsets

N <- 2100
abs.x <- 70
set.seed(2)
x.vec <- runif(N, -abs.x, abs.x)
str(x.vec)
#>  num [1:2100] -44.1 28.3 10.3 -46.5 62.1 ...
library(data.table)
(task.dt <- data.table(
  x=x.vec,
  y = sin(x.vec)+rnorm(N,sd=0.5)))
#>               x           y
#>           <num>       <num>
#>    1: -44.11648 -0.40781530
#>    2:  28.33237 -0.08520601
#>    3:  10.26569 -1.23266284
#>    4: -46.47273 -1.36225125
#>    5:  62.13751 -1.33779346
#>   ---                      
#> 2096:  60.83765 -0.10678010
#> 2097:  55.71469 -0.92403513
#> 2098:  14.31045  1.04519820
#> 2099:  27.18008  1.67815828
#> 2100:  23.67202 -0.26881102
library(animint2)
ggplot()+
  geom_point(aes(
    x, y),
    shape=1,
    data=task.dt)+
  coord_equal()

Above we see a scatterplot of the simulated data. The goal of the learning algorithm will be to predict y from x.

The code below assigns three test groups to the randomly simulated data.

atomic.group.size <- 2
task.dt[, agroup := rep(seq(1, N/atomic.group.size), each=atomic.group.size)][]
#>               x           y agroup
#>           <num>       <num>  <int>
#>    1: -44.11648 -0.40781530      1
#>    2:  28.33237 -0.08520601      1
#>    3:  10.26569 -1.23266284      2
#>    4: -46.47273 -1.36225125      2
#>    5:  62.13751 -1.33779346      3
#>   ---                             
#> 2096:  60.83765 -0.10678010   1048
#> 2097:  55.71469 -0.92403513   1049
#> 2098:  14.31045  1.04519820   1049
#> 2099:  27.18008  1.67815828   1050
#> 2100:  23.67202 -0.26881102   1050
task.dt[, random_group := rep(
  rep(c("A","B","B","C","C","C","C"), each=atomic.group.size),
  l=.N
)][]
#>               x           y agroup random_group
#>           <num>       <num>  <int>       <char>
#>    1: -44.11648 -0.40781530      1            A
#>    2:  28.33237 -0.08520601      1            A
#>    3:  10.26569 -1.23266284      2            B
#>    4: -46.47273 -1.36225125      2            B
#>    5:  62.13751 -1.33779346      3            B
#>   ---                                          
#> 2096:  60.83765 -0.10678010   1048            C
#> 2097:  55.71469 -0.92403513   1049            C
#> 2098:  14.31045  1.04519820   1049            C
#> 2099:  27.18008  1.67815828   1050            C
#> 2100:  23.67202 -0.26881102   1050            C
table(group.tab <- task.dt$random_group)
#> 
#>    A    B    C 
#>  300  600 1200

The output above shows the number of rows in each random group. Below we define a task using that group,

reg.task <- mlr3::TaskRegr$new(
  "sin", task.dt, target="y")
reg.task$col_roles$subset <- "random_group"
reg.task$col_roles$group <- "agroup"
reg.task$col_roles$stratum <- "random_group"
reg.task$col_roles$feature <- "x"
str(reg.task$col_roles)
#> List of 8
#>  $ feature: chr "x"
#>  $ target : chr "y"
#>  $ name   : chr(0) 
#>  $ order  : chr(0) 
#>  $ stratum: chr "random_group"
#>  $ group  : chr "agroup"
#>  $ weight : chr(0) 
#>  $ subset : chr "random_group"

Below we define cross-validation.

same_other_sizes_cv <- mlr3resampling::ResamplingSameOtherSizesCV$new()
same_other_sizes_cv$instantiate(reg.task)
same_other_sizes_cv$instance$iteration.dt
#>     test.subset train.subsets groups test.fold                        test
#>          <char>        <char>  <int>     <int>                      <list>
#>  1:           A           all    700         1       43,44,57,58,71,72,...
#>  2:           B           all    700         1        3, 4, 5, 6,17,18,...
#>  3:           C           all    700         1       23,24,25,26,37,38,...
#>  4:           A           all    700         2        1, 2,15,16,29,30,...
#>  5:           B           all    700         2       33,34,47,48,61,62,...
#>  6:           C           all    700         2       13,14,21,22,35,36,...
#>  7:           A           all    700         3  99,100,155,156,169,170,...
#>  8:           B           all    700         3       19,20,45,46,75,76,...
#>  9:           C           all    700         3        7, 8, 9,10,11,12,...
#> 10:           A         other    600         1       43,44,57,58,71,72,...
#> 11:           B         other    500         1        3, 4, 5, 6,17,18,...
#> 12:           C         other    300         1       23,24,25,26,37,38,...
#> 13:           A         other    600         2        1, 2,15,16,29,30,...
#> 14:           B         other    500         2       33,34,47,48,61,62,...
#> 15:           C         other    300         2       13,14,21,22,35,36,...
#> 16:           A         other    600         3  99,100,155,156,169,170,...
#> 17:           B         other    500         3       19,20,45,46,75,76,...
#> 18:           C         other    300         3        7, 8, 9,10,11,12,...
#> 19:           A          same    100         1       43,44,57,58,71,72,...
#> 20:           B          same    200         1        3, 4, 5, 6,17,18,...
#> 21:           C          same    400         1       23,24,25,26,37,38,...
#> 22:           A          same    100         2        1, 2,15,16,29,30,...
#> 23:           B          same    200         2       33,34,47,48,61,62,...
#> 24:           C          same    400         2       13,14,21,22,35,36,...
#> 25:           A          same    100         3  99,100,155,156,169,170,...
#> 26:           B          same    200         3       19,20,45,46,75,76,...
#> 27:           C          same    400         3        7, 8, 9,10,11,12,...
#>     test.subset train.subsets groups test.fold                        test
#>                     train  seed n.train.groups iteration
#>                    <list> <int>          <int>     <int>
#>  1:  1, 2, 7, 8, 9,10,...     1            700         1
#>  2:  1, 2, 7, 8, 9,10,...     1            700         2
#>  3:  1, 2, 7, 8, 9,10,...     1            700         3
#>  4:       3,4,5,6,7,8,...     1            700         4
#>  5:       3,4,5,6,7,8,...     1            700         5
#>  6:       3,4,5,6,7,8,...     1            700         6
#>  7:       1,2,3,4,5,6,...     1            700         7
#>  8:       1,2,3,4,5,6,...     1            700         8
#>  9:       1,2,3,4,5,6,...     1            700         9
#> 10:  7, 8, 9,10,11,12,...     1            600        10
#> 11:  1, 2, 7, 8, 9,10,...     1            500        11
#> 12:  1, 2,15,16,19,20,...     1            300        12
#> 13:       3,4,5,6,7,8,...     1            600        13
#> 14:  7, 8, 9,10,11,12,...     1            500        14
#> 15:  3, 4, 5, 6,17,18,...     1            300        15
#> 16:  3, 4, 5, 6,13,14,...     1            600        16
#> 17:  1, 2,13,14,15,16,...     1            500        17
#> 18:       1,2,3,4,5,6,...     1            300        18
#> 19:  1, 2,15,16,29,30,...     1            100        19
#> 20: 19,20,33,34,45,46,...     1            200        20
#> 21:  7, 8, 9,10,11,12,...     1            400        21
#> 22: 43,44,57,58,71,72,...     1            100        22
#> 23:  3, 4, 5, 6,17,18,...     1            200        23
#> 24:  7, 8, 9,10,11,12,...     1            400        24
#> 25:  1, 2,15,16,29,30,...     1            100        25
#> 26:  3, 4, 5, 6,17,18,...     1            200        26
#> 27: 13,14,21,22,23,24,...     1            400        27
#>                     train  seed n.train.groups iteration

So using the K-fold cross-validation, we will do one train/test split for each row of the table above. There is one row for each combination of test subset (A/B/C), train subset (same/other/all), and test fold (1/2/3).

We compute and plot the results using the code below,

(reg.learner.list <- list(
  mlr3::LearnerRegrFeatureless$new()))
#> [[1]]
#> <LearnerRegrFeatureless:regr.featureless>: Featureless Regression Learner
#> * Model: -
#> * Parameters: robust=FALSE
#> * Packages: mlr3, stats
#> * Predict Types:  [response], se
#> * Feature Types: logical, integer, numeric, character, factor, ordered,
#>   POSIXct
#> * Properties: featureless, importance, missings, selected_features
if(requireNamespace("rpart")){
  reg.learner.list$rpart <- mlr3::LearnerRegrRpart$new()
}
(same.other.grid <- mlr3::benchmark_grid(
  reg.task,
  reg.learner.list,
  same_other_sizes_cv))
#>      task          learner          resampling
#>    <char>           <char>              <char>
#> 1:    sin regr.featureless same_other_sizes_cv
#> 2:    sin       regr.rpart same_other_sizes_cv
##if(require(future))plan("multisession")
lgr::get_logger("mlr3")$set_threshold("warn")
(same.other.result <- mlr3::benchmark(
  same.other.grid, store_models = TRUE))
#> <BenchmarkResult> of 54 rows with 2 resampling runs
#>  nr task_id       learner_id       resampling_id iters warnings errors
#>   1     sin regr.featureless same_other_sizes_cv    27        0      0
#>   2     sin       regr.rpart same_other_sizes_cv    27        0      0
same.other.score <- mlr3resampling::score(same.other.result)
same.other.score[, n.train := sapply(train, length)]
same.other.score[1]
#>    test.subset train.subsets groups test.fold                  test
#>         <char>        <char>  <int>     <int>                <list>
#> 1:           A           all    700         1 43,44,57,58,71,72,...
#>                    train  seed n.train.groups iteration
#>                   <list> <int>          <int>     <int>
#> 1:  1, 2, 7, 8, 9,10,...     1            700         1
#>                                   uhash    nr           task task_id
#>                                  <char> <int>         <list>  <char>
#> 1: 6bf9fd54-7d6a-4bd0-85fc-9c1437d80f1e     1 <TaskRegr:sin>     sin
#>                                      learner       learner_id
#>                                       <list>           <char>
#> 1: <LearnerRegrFeatureless:regr.featureless> regr.featureless
#>                      resampling       resampling_id       prediction regr.mse
#>                          <list>              <char>           <list>    <num>
#> 1: <ResamplingSameOtherSizesCV> same_other_sizes_cv <PredictionRegr> 0.819815
#>      algorithm n.train
#>         <char>   <int>
#> 1: featureless    1400

ggplot()+
  geom_point(aes(
    regr.mse, train.subsets, color=algorithm),
    shape=1,
    data=same.other.score)+
  geom_text(aes(
    Inf, train.subsets,
    label=sprintf("n.train=%d ", n.train)),
    hjust=1,
    vjust=1.5,
    shape=1,
    data=same.other.score[algorithm=="featureless" & test.fold==1])+
  facet_grid(. ~ test.subset, labeller=label_both, scales="free")+
  scale_x_log10(
    "Mean squared prediction error (test set)")


same.other.wide <- dcast(
  same.other.score,
  algorithm + test.subset + train.subsets ~ .,
  list(mean, sd),
  value.var="regr.mse")
ggplot()+
  geom_segment(aes(
    regr.mse_mean+regr.mse_sd, train.subsets,
    xend=regr.mse_mean-regr.mse_sd, yend=train.subsets,
    color=algorithm),
    shape=1,
    data=same.other.wide)+
  geom_point(aes(
    regr.mse_mean, train.subsets, color=algorithm),
    shape=1,
    data=same.other.wide)+
  geom_text(aes(
    Inf, train.subsets,
    label=sprintf("n.train=%d ", n.train)),
    hjust=1,
    vjust=1.5,
    shape=1,
    data=same.other.score[algorithm=="featureless" & test.fold==1])+
  facet_grid(. ~ test.subset, labeller=label_both, scales="free")+
  scale_x_log10(
    "Mean squared prediction error (test set)")

The figures above show a test subset in each panel, the train subsets on the y axis, the test error on the x axis, the two different algorithms are shown in two different colors. We can clearly see that

Overall in the plot above, all tends to have less prediction error than same, which suggests that the subsets are similar (and indeed there are iid in this simulation).

Below we visualize test error as a function of train size.

ggplot()+
  geom_line(aes(
    n.train, regr.mse,
    color=algorithm,
    subset=paste(algorithm, test.fold)),
    data=same.other.score)+
  geom_label(aes(
    n.train, regr.mse,
    color=algorithm,
    label=train.subsets),
    data=same.other.score)+
  facet_grid(. ~ test.subset, labeller=label_both, scales="free")+
  scale_y_log10(
    "Mean squared prediction error (test set)")

Downsample to see how many train data are required for good accuracy overall

In the previous section we defined a task using the subset role, which means that the different values in that column will be used to define different subsets for training/testing using same/other/all CV. In contrast, below we define a task without the subset role, which means that we will not have separate CV iterations for same/other/all (full data is treated as one subset / train subset is same).

task.no.subset <- mlr3::TaskRegr$new(
  "sin", task.dt, target="y")
task.no.subset$col_roles$group <- "agroup"
task.no.subset$col_roles$stratum <- "random_group"
task.no.subset$col_roles$feature <- "x"
str(task.no.subset$col_roles)
#> List of 7
#>  $ feature: chr "x"
#>  $ target : chr "y"
#>  $ name   : chr(0) 
#>  $ order  : chr(0) 
#>  $ stratum: chr "random_group"
#>  $ group  : chr "agroup"
#>  $ weight : chr(0)

Below we define cross-validation, and we set the sizes to 5 so we can see what happens when we have have train sets that are 5 sizes smaller than the full train set size.

same_other_sizes_cv <- mlr3resampling::ResamplingSameOtherSizesCV$new()
same_other_sizes_cv$param_set$values$sizes <- 5
same_other_sizes_cv$instantiate(task.no.subset)
same_other_sizes_cv$instance$iteration.dt
#>     test.subset train.subsets groups test.fold                  test
#>          <char>        <char>  <int>     <int>                <list>
#>  1:        full          same    700         1  3, 4,13,14,15,16,...
#>  2:        full          same    700         1  3, 4,13,14,15,16,...
#>  3:        full          same    700         1  3, 4,13,14,15,16,...
#>  4:        full          same    700         1  3, 4,13,14,15,16,...
#>  5:        full          same    700         1  3, 4,13,14,15,16,...
#>  6:        full          same    700         1  3, 4,13,14,15,16,...
#>  7:        full          same    700         2  1, 2,17,18,21,22,...
#>  8:        full          same    700         2  1, 2,17,18,21,22,...
#>  9:        full          same    700         2  1, 2,17,18,21,22,...
#> 10:        full          same    700         2  1, 2,17,18,21,22,...
#> 11:        full          same    700         2  1, 2,17,18,21,22,...
#> 12:        full          same    700         2  1, 2,17,18,21,22,...
#> 13:        full          same    700         3  5, 6, 7, 8, 9,10,...
#> 14:        full          same    700         3  5, 6, 7, 8, 9,10,...
#> 15:        full          same    700         3  5, 6, 7, 8, 9,10,...
#> 16:        full          same    700         3  5, 6, 7, 8, 9,10,...
#> 17:        full          same    700         3  5, 6, 7, 8, 9,10,...
#> 18:        full          same    700         3  5, 6, 7, 8, 9,10,...
#>                           train  seed n.train.groups iteration
#>                          <list> <int>          <int>     <int>
#>  1: 565,566,583,584,743,744,...     1             21         1
#>  2: 133,134,171,172,305,306,...     1             43         2
#>  3:       77,78,93,94,95,96,...     1             87         3
#>  4:        7, 8,25,26,29,30,...     1            175         4
#>  5:        1, 2, 7, 8,17,18,...     1            350         5
#>  6:             1,2,5,6,7,8,...     1            700         6
#>  7:  39, 40,109,110,285,286,...     1             21         7
#>  8:       29,30,37,38,39,40,...     1             43         8
#>  9:       29,30,37,38,39,40,...     1             87         9
#> 10:        3, 4,13,14,23,24,...     1            175        10
#> 11:        3, 4,13,14,19,20,...     1            350        11
#> 12:             3,4,5,6,7,8,...     1            700        12
#> 13: 209,210,397,398,519,520,...     1             21        13
#> 14: 139,140,209,210,343,344,...     1             43        14
#> 15:        1, 2,17,18,43,44,...     1             87        15
#> 16:        1, 2,17,18,37,38,...     1            175        16
#> 17:        1, 2,13,14,17,18,...     1            350        17
#> 18:        1, 2, 3, 4,13,14,...     1            700        18

So using the K-fold cross-validation, we will do one train/test split for each row of the table above. There is one row for each combination of n.train.groups (full train set size + 5 smaller sizes), and test fold (1/2/3).

We compute and plot the results using the code below,

(reg.learner.list <- list(
  mlr3::LearnerRegrFeatureless$new()))
#> [[1]]
#> <LearnerRegrFeatureless:regr.featureless>: Featureless Regression Learner
#> * Model: -
#> * Parameters: robust=FALSE
#> * Packages: mlr3, stats
#> * Predict Types:  [response], se
#> * Feature Types: logical, integer, numeric, character, factor, ordered,
#>   POSIXct
#> * Properties: featureless, importance, missings, selected_features
if(requireNamespace("rpart")){
  reg.learner.list$rpart <- mlr3::LearnerRegrRpart$new()
}
(same.other.grid <- mlr3::benchmark_grid(
  task.no.subset,
  reg.learner.list,
  same_other_sizes_cv))
#>      task          learner          resampling
#>    <char>           <char>              <char>
#> 1:    sin regr.featureless same_other_sizes_cv
#> 2:    sin       regr.rpart same_other_sizes_cv
##if(require(future))plan("multisession")
lgr::get_logger("mlr3")$set_threshold("warn")
(same.other.result <- mlr3::benchmark(
  same.other.grid, store_models = TRUE))
#> <BenchmarkResult> of 36 rows with 2 resampling runs
#>  nr task_id       learner_id       resampling_id iters warnings errors
#>   1     sin regr.featureless same_other_sizes_cv    18        0      0
#>   2     sin       regr.rpart same_other_sizes_cv    18        0      0
same.other.score <- mlr3resampling::score(same.other.result)
same.other.score[, n.train := sapply(train, length)]
same.other.score[1]
#>    test.subset train.subsets groups test.fold                  test
#>         <char>        <char>  <int>     <int>                <list>
#> 1:        full          same    700         1  3, 4,13,14,15,16,...
#>                          train  seed n.train.groups iteration
#>                         <list> <int>          <int>     <int>
#> 1: 565,566,583,584,743,744,...     1             21         1
#>                                   uhash    nr           task task_id
#>                                  <char> <int>         <list>  <char>
#> 1: 9610f56c-73f2-4e34-b0f9-d397db3f1726     1 <TaskRegr:sin>     sin
#>                                      learner       learner_id
#>                                       <list>           <char>
#> 1: <LearnerRegrFeatureless:regr.featureless> regr.featureless
#>                      resampling       resampling_id       prediction  regr.mse
#>                          <list>              <char>           <list>     <num>
#> 1: <ResamplingSameOtherSizesCV> same_other_sizes_cv <PredictionRegr> 0.7650132
#>      algorithm n.train
#>         <char>   <int>
#> 1: featureless      42

ggplot()+
  geom_line(aes(
    n.train, regr.mse,
    color=algorithm,
    subset=paste(algorithm, test.fold)),
    data=same.other.score)+
  geom_point(aes(
    n.train, regr.mse,
    color=algorithm),
    data=same.other.score)+
  facet_grid(. ~ test.subset, labeller=label_both, scales="free")+
  scale_x_log10(
    "Number of train rows",
    breaks=unique(same.other.score$n.train))+
  scale_y_log10(
    "Mean squared prediction error (test set)")

From the plot above, it looks like about 700 rows is enough to get minimal test error, using the rpart learner.

Downsample to sizes of other sets

N <- 600
abs.x <- 20
set.seed(1)
x.vec <- sort(runif(N, -abs.x, abs.x))
str(x.vec)
#>  num [1:600] -19.9 -19.9 -19.7 -19.6 -19.6 ...
library(data.table)
(task.dt <- data.table(
  x=x.vec,
  y = sin(x.vec)+rnorm(N,sd=0.5)))
#>              x          y
#>          <num>      <num>
#>   1: -19.92653 -0.4336887
#>   2: -19.92269 -1.4023484
#>   3: -19.67486  0.2509134
#>   4: -19.55856 -0.8428921
#>   5: -19.55402  0.1794473
#>  ---                     
#> 596:  19.70736  0.7497818
#> 597:  19.74997  0.3178435
#> 598:  19.75656  1.3950030
#> 599:  19.83862 -0.2086586
#> 600:  19.84309  0.5748863
library(animint2)
ggplot()+
  geom_point(aes(
    x, y),
    shape=1,
    data=task.dt)+
  coord_equal()

atomic.subset.size <- 2
task.dt[, agroup := rep(seq(1, N/atomic.subset.size), each=atomic.subset.size)][]
#>              x          y agroup
#>          <num>      <num>  <int>
#>   1: -19.92653 -0.4336887      1
#>   2: -19.92269 -1.4023484      1
#>   3: -19.67486  0.2509134      2
#>   4: -19.55856 -0.8428921      2
#>   5: -19.55402  0.1794473      3
#>  ---                            
#> 596:  19.70736  0.7497818    298
#> 597:  19.74997  0.3178435    299
#> 598:  19.75656  1.3950030    299
#> 599:  19.83862 -0.2086586    300
#> 600:  19.84309  0.5748863    300
task.dt[, random_subset := rep(
  rep(c("A","B","B","B"), each=atomic.subset.size),
  l=.N
)][]
#>              x          y agroup random_subset
#>          <num>      <num>  <int>        <char>
#>   1: -19.92653 -0.4336887      1             A
#>   2: -19.92269 -1.4023484      1             A
#>   3: -19.67486  0.2509134      2             B
#>   4: -19.55856 -0.8428921      2             B
#>   5: -19.55402  0.1794473      3             B
#>  ---                                          
#> 596:  19.70736  0.7497818    298             B
#> 597:  19.74997  0.3178435    299             B
#> 598:  19.75656  1.3950030    299             B
#> 599:  19.83862 -0.2086586    300             B
#> 600:  19.84309  0.5748863    300             B
table(subset.tab <- task.dt$random_subset)
#> 
#>   A   B 
#> 150 450

reg.task <- mlr3::TaskRegr$new(
  "sin", task.dt, target="y")
reg.task$col_roles$subset <- "random_subset"
reg.task$col_roles$group <- "agroup"
reg.task$col_roles$stratum <- "random_subset"
reg.task$col_roles$feature <- "x"
same_other_sizes_cv <- mlr3resampling::ResamplingSameOtherSizesCV$new()

In the previous section we analyzed prediction accuracy of same/other/all, which corresponds to keeping sizes parameter at default of -1. The main difference in this section is that we change sizes to 0, which means to down-sample same/other/all, so we can see if there is an effect for sample size (there should be for iid problems with intermediate difficulty). We set sizes to 0 in the next line:

same_other_sizes_cv$param_set$values$sizes <- 0
same_other_sizes_cv$instantiate(reg.task)
same_other_sizes_cv$instance$it
#>     test.subset train.subsets groups test.fold                  test
#>          <char>        <char>  <int>     <int>                <list>
#>  1:           A           all    200         1  1, 2,49,50,57,58,...
#>  2:           A           all    200         1  1, 2,49,50,57,58,...
#>  3:           A           all    200         1  1, 2,49,50,57,58,...
#>  4:           B           all    200         1 19,20,31,32,37,38,...
#>  5:           B           all    200         1 19,20,31,32,37,38,...
#>  6:           B           all    200         1 19,20,31,32,37,38,...
#>  7:           A           all    200         2 17,18,41,42,89,90,...
#>  8:           A           all    200         2 17,18,41,42,89,90,...
#>  9:           A           all    200         2 17,18,41,42,89,90,...
#> 10:           B           all    200         2       3,4,5,6,7,8,...
#> 11:           B           all    200         2       3,4,5,6,7,8,...
#> 12:           B           all    200         2       3,4,5,6,7,8,...
#> 13:           A           all    200         3  9,10,25,26,33,34,...
#> 14:           A           all    200         3  9,10,25,26,33,34,...
#> 15:           A           all    200         3  9,10,25,26,33,34,...
#> 16:           B           all    200         3 15,16,21,22,23,24,...
#> 17:           B           all    200         3 15,16,21,22,23,24,...
#> 18:           B           all    200         3 15,16,21,22,23,24,...
#> 19:           A         other    150         1  1, 2,49,50,57,58,...
#> 20:           A         other    150         1  1, 2,49,50,57,58,...
#> 21:           B         other     50         1 19,20,31,32,37,38,...
#> 22:           A         other    150         2 17,18,41,42,89,90,...
#> 23:           A         other    150         2 17,18,41,42,89,90,...
#> 24:           B         other     50         2       3,4,5,6,7,8,...
#> 25:           A         other    150         3  9,10,25,26,33,34,...
#> 26:           A         other    150         3  9,10,25,26,33,34,...
#> 27:           B         other     50         3 15,16,21,22,23,24,...
#> 28:           A          same     50         1  1, 2,49,50,57,58,...
#> 29:           B          same    150         1 19,20,31,32,37,38,...
#> 30:           B          same    150         1 19,20,31,32,37,38,...
#> 31:           A          same     50         2 17,18,41,42,89,90,...
#> 32:           B          same    150         2       3,4,5,6,7,8,...
#> 33:           B          same    150         2       3,4,5,6,7,8,...
#> 34:           A          same     50         3  9,10,25,26,33,34,...
#> 35:           B          same    150         3 15,16,21,22,23,24,...
#> 36:           B          same    150         3 15,16,21,22,23,24,...
#>     test.subset train.subsets groups test.fold                  test
#>                     train  seed n.train.groups iteration
#>                    <list> <int>          <int>     <int>
#>  1:  5, 6, 9,10,15,16,...     1             50         1
#>  2:       3,4,5,6,7,8,...     1            150         2
#>  3:       3,4,5,6,7,8,...     1            200         3
#>  4:  3, 4, 7, 8,15,16,...     1             50         4
#>  5:       3,4,5,6,7,8,...     1            150         5
#>  6:       3,4,5,6,7,8,...     1            200         6
#>  7:  1, 2,35,36,39,40,...     1             50         7
#>  8:  1, 2, 9,10,19,20,...     1            150         8
#>  9:  1, 2, 9,10,15,16,...     1            200         9
#> 10: 19,20,63,64,73,74,...     1             50        10
#> 11:  1, 2, 9,10,15,16,...     1            150        11
#> 12:  1, 2, 9,10,15,16,...     1            200        12
#> 13: 29,30,37,38,49,50,...     1             50        13
#> 14:  5, 6,11,12,13,14,...     1            150        14
#> 15:       1,2,3,4,5,6,...     1            200        15
#> 16: 13,14,29,30,49,50,...     1             50        16
#> 17:       1,2,3,4,5,6,...     1            150        17
#> 18:       1,2,3,4,5,6,...     1            200        18
#> 19: 15,16,21,22,55,56,...     1             50        19
#> 20:       3,4,5,6,7,8,...     1            150        20
#> 21:  9,10,17,18,25,26,...     1             50        21
#> 22: 15,16,19,20,23,24,...     1             50        22
#> 23: 15,16,19,20,21,22,...     1            150        23
#> 24:  1, 2, 9,10,25,26,...     1             50        24
#> 25: 11,12,19,20,27,28,...     1             50        25
#> 26:       3,4,5,6,7,8,...     1            150        26
#> 27:  1, 2,17,18,41,42,...     1             50        27
#> 28:  9,10,17,18,25,26,...     1             50        28
#> 29: 59,60,63,64,75,76,...     1             50        29
#> 30:       3,4,5,6,7,8,...     1            150        30
#> 31:  1, 2, 9,10,25,26,...     1             50        31
#> 32: 23,24,37,38,51,52,...     1             50        32
#> 33: 15,16,19,20,21,22,...     1            150        33
#> 34:  1, 2,17,18,41,42,...     1             50        34
#> 35: 11,12,19,20,45,46,...     1             50        35
#> 36:       3,4,5,6,7,8,...     1            150        36
#>                     train  seed n.train.groups iteration
(reg.learner.list <- list(
  mlr3::LearnerRegrFeatureless$new()))
#> [[1]]
#> <LearnerRegrFeatureless:regr.featureless>: Featureless Regression Learner
#> * Model: -
#> * Parameters: robust=FALSE
#> * Packages: mlr3, stats
#> * Predict Types:  [response], se
#> * Feature Types: logical, integer, numeric, character, factor, ordered,
#>   POSIXct
#> * Properties: featureless, importance, missings, selected_features
if(requireNamespace("rpart")){
  reg.learner.list$rpart <- mlr3::LearnerRegrRpart$new()
}
(same.other.grid <- mlr3::benchmark_grid(
  reg.task,
  reg.learner.list,
  same_other_sizes_cv))
#>      task          learner          resampling
#>    <char>           <char>              <char>
#> 1:    sin regr.featureless same_other_sizes_cv
#> 2:    sin       regr.rpart same_other_sizes_cv
##if(require(future))plan("multisession")
lgr::get_logger("mlr3")$set_threshold("warn")
(same.other.result <- mlr3::benchmark(
  same.other.grid, store_models = TRUE))
#> <BenchmarkResult> of 72 rows with 2 resampling runs
#>  nr task_id       learner_id       resampling_id iters warnings errors
#>   1     sin regr.featureless same_other_sizes_cv    36        0      0
#>   2     sin       regr.rpart same_other_sizes_cv    36        0      0
same.other.score <- mlr3resampling::score(same.other.result)
same.other.score[1]
#>    test.subset train.subsets groups test.fold                  test
#>         <char>        <char>  <int>     <int>                <list>
#> 1:           A           all    200         1  1, 2,49,50,57,58,...
#>                    train  seed n.train.groups iteration
#>                   <list> <int>          <int>     <int>
#> 1:  5, 6, 9,10,15,16,...     1             50         1
#>                                   uhash    nr           task task_id
#>                                  <char> <int>         <list>  <char>
#> 1: b956f97e-1f5b-43c4-b232-ddafed3a8347     1 <TaskRegr:sin>     sin
#>                                      learner       learner_id
#>                                       <list>           <char>
#> 1: <LearnerRegrFeatureless:regr.featureless> regr.featureless
#>                      resampling       resampling_id       prediction  regr.mse
#>                          <list>              <char>           <list>     <num>
#> 1: <ResamplingSameOtherSizesCV> same_other_sizes_cv <PredictionRegr> 0.5814085
#>      algorithm
#>         <char>
#> 1: featureless

The plot below shows the same results (no down-sampling) as if we did sizes=-1 (like in the previous section.

ggplot()+
  geom_point(aes(
    regr.mse, train.subsets, color=algorithm),
    shape=1,
    data=same.other.score[groups==n.train.groups])+
  facet_grid(. ~ test.subset, labeller=label_both)

The plots below compare all six train subsets (including three down-sampled), and it it is clear there is an effect for sample size.

same.other.score[, subset.N := paste(train.subsets, n.train.groups)][]
#>     test.subset train.subsets groups test.fold                  test
#>          <char>        <char>  <int>     <int>                <list>
#>  1:           A           all    200         1  1, 2,49,50,57,58,...
#>  2:           A           all    200         1  1, 2,49,50,57,58,...
#>  3:           A           all    200         1  1, 2,49,50,57,58,...
#>  4:           B           all    200         1 19,20,31,32,37,38,...
#>  5:           B           all    200         1 19,20,31,32,37,38,...
#>  6:           B           all    200         1 19,20,31,32,37,38,...
#>  7:           A           all    200         2 17,18,41,42,89,90,...
#>  8:           A           all    200         2 17,18,41,42,89,90,...
#>  9:           A           all    200         2 17,18,41,42,89,90,...
#> 10:           B           all    200         2       3,4,5,6,7,8,...
#> 11:           B           all    200         2       3,4,5,6,7,8,...
#> 12:           B           all    200         2       3,4,5,6,7,8,...
#> 13:           A           all    200         3  9,10,25,26,33,34,...
#> 14:           A           all    200         3  9,10,25,26,33,34,...
#> 15:           A           all    200         3  9,10,25,26,33,34,...
#> 16:           B           all    200         3 15,16,21,22,23,24,...
#> 17:           B           all    200         3 15,16,21,22,23,24,...
#> 18:           B           all    200         3 15,16,21,22,23,24,...
#> 19:           A         other    150         1  1, 2,49,50,57,58,...
#> 20:           A         other    150         1  1, 2,49,50,57,58,...
#> 21:           B         other     50         1 19,20,31,32,37,38,...
#> 22:           A         other    150         2 17,18,41,42,89,90,...
#> 23:           A         other    150         2 17,18,41,42,89,90,...
#> 24:           B         other     50         2       3,4,5,6,7,8,...
#> 25:           A         other    150         3  9,10,25,26,33,34,...
#> 26:           A         other    150         3  9,10,25,26,33,34,...
#> 27:           B         other     50         3 15,16,21,22,23,24,...
#> 28:           A          same     50         1  1, 2,49,50,57,58,...
#> 29:           B          same    150         1 19,20,31,32,37,38,...
#> 30:           B          same    150         1 19,20,31,32,37,38,...
#> 31:           A          same     50         2 17,18,41,42,89,90,...
#> 32:           B          same    150         2       3,4,5,6,7,8,...
#> 33:           B          same    150         2       3,4,5,6,7,8,...
#> 34:           A          same     50         3  9,10,25,26,33,34,...
#> 35:           B          same    150         3 15,16,21,22,23,24,...
#> 36:           B          same    150         3 15,16,21,22,23,24,...
#> 37:           A           all    200         1  1, 2,49,50,57,58,...
#> 38:           A           all    200         1  1, 2,49,50,57,58,...
#> 39:           A           all    200         1  1, 2,49,50,57,58,...
#> 40:           B           all    200         1 19,20,31,32,37,38,...
#> 41:           B           all    200         1 19,20,31,32,37,38,...
#> 42:           B           all    200         1 19,20,31,32,37,38,...
#> 43:           A           all    200         2 17,18,41,42,89,90,...
#> 44:           A           all    200         2 17,18,41,42,89,90,...
#> 45:           A           all    200         2 17,18,41,42,89,90,...
#> 46:           B           all    200         2       3,4,5,6,7,8,...
#> 47:           B           all    200         2       3,4,5,6,7,8,...
#> 48:           B           all    200         2       3,4,5,6,7,8,...
#> 49:           A           all    200         3  9,10,25,26,33,34,...
#> 50:           A           all    200         3  9,10,25,26,33,34,...
#> 51:           A           all    200         3  9,10,25,26,33,34,...
#> 52:           B           all    200         3 15,16,21,22,23,24,...
#> 53:           B           all    200         3 15,16,21,22,23,24,...
#> 54:           B           all    200         3 15,16,21,22,23,24,...
#> 55:           A         other    150         1  1, 2,49,50,57,58,...
#> 56:           A         other    150         1  1, 2,49,50,57,58,...
#> 57:           B         other     50         1 19,20,31,32,37,38,...
#> 58:           A         other    150         2 17,18,41,42,89,90,...
#> 59:           A         other    150         2 17,18,41,42,89,90,...
#> 60:           B         other     50         2       3,4,5,6,7,8,...
#> 61:           A         other    150         3  9,10,25,26,33,34,...
#> 62:           A         other    150         3  9,10,25,26,33,34,...
#> 63:           B         other     50         3 15,16,21,22,23,24,...
#> 64:           A          same     50         1  1, 2,49,50,57,58,...
#> 65:           B          same    150         1 19,20,31,32,37,38,...
#> 66:           B          same    150         1 19,20,31,32,37,38,...
#> 67:           A          same     50         2 17,18,41,42,89,90,...
#> 68:           B          same    150         2       3,4,5,6,7,8,...
#> 69:           B          same    150         2       3,4,5,6,7,8,...
#> 70:           A          same     50         3  9,10,25,26,33,34,...
#> 71:           B          same    150         3 15,16,21,22,23,24,...
#> 72:           B          same    150         3 15,16,21,22,23,24,...
#>     test.subset train.subsets groups test.fold                  test
#>                     train  seed n.train.groups iteration
#>                    <list> <int>          <int>     <int>
#>  1:  5, 6, 9,10,15,16,...     1             50         1
#>  2:       3,4,5,6,7,8,...     1            150         2
#>  3:       3,4,5,6,7,8,...     1            200         3
#>  4:  3, 4, 7, 8,15,16,...     1             50         4
#>  5:       3,4,5,6,7,8,...     1            150         5
#>  6:       3,4,5,6,7,8,...     1            200         6
#>  7:  1, 2,35,36,39,40,...     1             50         7
#>  8:  1, 2, 9,10,19,20,...     1            150         8
#>  9:  1, 2, 9,10,15,16,...     1            200         9
#> 10: 19,20,63,64,73,74,...     1             50        10
#> 11:  1, 2, 9,10,15,16,...     1            150        11
#> 12:  1, 2, 9,10,15,16,...     1            200        12
#> 13: 29,30,37,38,49,50,...     1             50        13
#> 14:  5, 6,11,12,13,14,...     1            150        14
#> 15:       1,2,3,4,5,6,...     1            200        15
#> 16: 13,14,29,30,49,50,...     1             50        16
#> 17:       1,2,3,4,5,6,...     1            150        17
#> 18:       1,2,3,4,5,6,...     1            200        18
#> 19: 15,16,21,22,55,56,...     1             50        19
#> 20:       3,4,5,6,7,8,...     1            150        20
#> 21:  9,10,17,18,25,26,...     1             50        21
#> 22: 15,16,19,20,23,24,...     1             50        22
#> 23: 15,16,19,20,21,22,...     1            150        23
#> 24:  1, 2, 9,10,25,26,...     1             50        24
#> 25: 11,12,19,20,27,28,...     1             50        25
#> 26:       3,4,5,6,7,8,...     1            150        26
#> 27:  1, 2,17,18,41,42,...     1             50        27
#> 28:  9,10,17,18,25,26,...     1             50        28
#> 29: 59,60,63,64,75,76,...     1             50        29
#> 30:       3,4,5,6,7,8,...     1            150        30
#> 31:  1, 2, 9,10,25,26,...     1             50        31
#> 32: 23,24,37,38,51,52,...     1             50        32
#> 33: 15,16,19,20,21,22,...     1            150        33
#> 34:  1, 2,17,18,41,42,...     1             50        34
#> 35: 11,12,19,20,45,46,...     1             50        35
#> 36:       3,4,5,6,7,8,...     1            150        36
#> 37:  5, 6, 9,10,15,16,...     1             50         1
#> 38:       3,4,5,6,7,8,...     1            150         2
#> 39:       3,4,5,6,7,8,...     1            200         3
#> 40:  3, 4, 7, 8,15,16,...     1             50         4
#> 41:       3,4,5,6,7,8,...     1            150         5
#> 42:       3,4,5,6,7,8,...     1            200         6
#> 43:  1, 2,35,36,39,40,...     1             50         7
#> 44:  1, 2, 9,10,19,20,...     1            150         8
#> 45:  1, 2, 9,10,15,16,...     1            200         9
#> 46: 19,20,63,64,73,74,...     1             50        10
#> 47:  1, 2, 9,10,15,16,...     1            150        11
#> 48:  1, 2, 9,10,15,16,...     1            200        12
#> 49: 29,30,37,38,49,50,...     1             50        13
#> 50:  5, 6,11,12,13,14,...     1            150        14
#> 51:       1,2,3,4,5,6,...     1            200        15
#> 52: 13,14,29,30,49,50,...     1             50        16
#> 53:       1,2,3,4,5,6,...     1            150        17
#> 54:       1,2,3,4,5,6,...     1            200        18
#> 55: 15,16,21,22,55,56,...     1             50        19
#> 56:       3,4,5,6,7,8,...     1            150        20
#> 57:  9,10,17,18,25,26,...     1             50        21
#> 58: 15,16,19,20,23,24,...     1             50        22
#> 59: 15,16,19,20,21,22,...     1            150        23
#> 60:  1, 2, 9,10,25,26,...     1             50        24
#> 61: 11,12,19,20,27,28,...     1             50        25
#> 62:       3,4,5,6,7,8,...     1            150        26
#> 63:  1, 2,17,18,41,42,...     1             50        27
#> 64:  9,10,17,18,25,26,...     1             50        28
#> 65: 59,60,63,64,75,76,...     1             50        29
#> 66:       3,4,5,6,7,8,...     1            150        30
#> 67:  1, 2, 9,10,25,26,...     1             50        31
#> 68: 23,24,37,38,51,52,...     1             50        32
#> 69: 15,16,19,20,21,22,...     1            150        33
#> 70:  1, 2,17,18,41,42,...     1             50        34
#> 71: 11,12,19,20,45,46,...     1             50        35
#> 72:       3,4,5,6,7,8,...     1            150        36
#>                     train  seed n.train.groups iteration
#>                                    uhash    nr           task task_id
#>                                   <char> <int>         <list>  <char>
#>  1: b956f97e-1f5b-43c4-b232-ddafed3a8347     1 <TaskRegr:sin>     sin
#>  2: b956f97e-1f5b-43c4-b232-ddafed3a8347     1 <TaskRegr:sin>     sin
#>  3: b956f97e-1f5b-43c4-b232-ddafed3a8347     1 <TaskRegr:sin>     sin
#>  4: b956f97e-1f5b-43c4-b232-ddafed3a8347     1 <TaskRegr:sin>     sin
#>  5: b956f97e-1f5b-43c4-b232-ddafed3a8347     1 <TaskRegr:sin>     sin
#>  6: b956f97e-1f5b-43c4-b232-ddafed3a8347     1 <TaskRegr:sin>     sin
#>  7: b956f97e-1f5b-43c4-b232-ddafed3a8347     1 <TaskRegr:sin>     sin
#>  8: b956f97e-1f5b-43c4-b232-ddafed3a8347     1 <TaskRegr:sin>     sin
#>  9: b956f97e-1f5b-43c4-b232-ddafed3a8347     1 <TaskRegr:sin>     sin
#> 10: b956f97e-1f5b-43c4-b232-ddafed3a8347     1 <TaskRegr:sin>     sin
#> 11: b956f97e-1f5b-43c4-b232-ddafed3a8347     1 <TaskRegr:sin>     sin
#> 12: b956f97e-1f5b-43c4-b232-ddafed3a8347     1 <TaskRegr:sin>     sin
#> 13: b956f97e-1f5b-43c4-b232-ddafed3a8347     1 <TaskRegr:sin>     sin
#> 14: b956f97e-1f5b-43c4-b232-ddafed3a8347     1 <TaskRegr:sin>     sin
#> 15: b956f97e-1f5b-43c4-b232-ddafed3a8347     1 <TaskRegr:sin>     sin
#> 16: b956f97e-1f5b-43c4-b232-ddafed3a8347     1 <TaskRegr:sin>     sin
#> 17: b956f97e-1f5b-43c4-b232-ddafed3a8347     1 <TaskRegr:sin>     sin
#> 18: b956f97e-1f5b-43c4-b232-ddafed3a8347     1 <TaskRegr:sin>     sin
#> 19: b956f97e-1f5b-43c4-b232-ddafed3a8347     1 <TaskRegr:sin>     sin
#> 20: b956f97e-1f5b-43c4-b232-ddafed3a8347     1 <TaskRegr:sin>     sin
#> 21: b956f97e-1f5b-43c4-b232-ddafed3a8347     1 <TaskRegr:sin>     sin
#> 22: b956f97e-1f5b-43c4-b232-ddafed3a8347     1 <TaskRegr:sin>     sin
#> 23: b956f97e-1f5b-43c4-b232-ddafed3a8347     1 <TaskRegr:sin>     sin
#> 24: b956f97e-1f5b-43c4-b232-ddafed3a8347     1 <TaskRegr:sin>     sin
#> 25: b956f97e-1f5b-43c4-b232-ddafed3a8347     1 <TaskRegr:sin>     sin
#> 26: b956f97e-1f5b-43c4-b232-ddafed3a8347     1 <TaskRegr:sin>     sin
#> 27: b956f97e-1f5b-43c4-b232-ddafed3a8347     1 <TaskRegr:sin>     sin
#> 28: b956f97e-1f5b-43c4-b232-ddafed3a8347     1 <TaskRegr:sin>     sin
#> 29: b956f97e-1f5b-43c4-b232-ddafed3a8347     1 <TaskRegr:sin>     sin
#> 30: b956f97e-1f5b-43c4-b232-ddafed3a8347     1 <TaskRegr:sin>     sin
#> 31: b956f97e-1f5b-43c4-b232-ddafed3a8347     1 <TaskRegr:sin>     sin
#> 32: b956f97e-1f5b-43c4-b232-ddafed3a8347     1 <TaskRegr:sin>     sin
#> 33: b956f97e-1f5b-43c4-b232-ddafed3a8347     1 <TaskRegr:sin>     sin
#> 34: b956f97e-1f5b-43c4-b232-ddafed3a8347     1 <TaskRegr:sin>     sin
#> 35: b956f97e-1f5b-43c4-b232-ddafed3a8347     1 <TaskRegr:sin>     sin
#> 36: b956f97e-1f5b-43c4-b232-ddafed3a8347     1 <TaskRegr:sin>     sin
#> 37: dc2721cf-a2b3-4505-9fd7-9c7e0c4b546b     2 <TaskRegr:sin>     sin
#> 38: dc2721cf-a2b3-4505-9fd7-9c7e0c4b546b     2 <TaskRegr:sin>     sin
#> 39: dc2721cf-a2b3-4505-9fd7-9c7e0c4b546b     2 <TaskRegr:sin>     sin
#> 40: dc2721cf-a2b3-4505-9fd7-9c7e0c4b546b     2 <TaskRegr:sin>     sin
#> 41: dc2721cf-a2b3-4505-9fd7-9c7e0c4b546b     2 <TaskRegr:sin>     sin
#> 42: dc2721cf-a2b3-4505-9fd7-9c7e0c4b546b     2 <TaskRegr:sin>     sin
#> 43: dc2721cf-a2b3-4505-9fd7-9c7e0c4b546b     2 <TaskRegr:sin>     sin
#> 44: dc2721cf-a2b3-4505-9fd7-9c7e0c4b546b     2 <TaskRegr:sin>     sin
#> 45: dc2721cf-a2b3-4505-9fd7-9c7e0c4b546b     2 <TaskRegr:sin>     sin
#> 46: dc2721cf-a2b3-4505-9fd7-9c7e0c4b546b     2 <TaskRegr:sin>     sin
#> 47: dc2721cf-a2b3-4505-9