Use difference to cancel out the resample-to-resample effect.
compare_lm <- rsq_estimates %>%
mutate(difference = `with splines` - `no splines`)
lm(difference ~ 1, data = compare_lm) %>%
tidy(conf.int = TRUE) %>%
select(estimate, p.value, starts_with("conf"))