Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .Rbuildignore
Original file line number Diff line number Diff line change
Expand Up @@ -7,3 +7,5 @@
^README.Rmd$
^docs$
^\.github$
^[.]?air[.]toml$
^\.vscode$
5 changes: 5 additions & 0 deletions .vscode/extensions.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
{
"recommendations": [
"Posit.air-vscode"
]
}
10 changes: 10 additions & 0 deletions .vscode/settings.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
{
"[r]": {
"editor.formatOnSave": true,
"editor.defaultFormatter": "Posit.air-vscode"
},
"[quarto]": {
"editor.formatOnSave": true,
"editor.defaultFormatter": "quarto.quarto"
}
}
7 changes: 4 additions & 3 deletions R/functions.R
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@

number_of_files <- function(bucket_name) {
chk::chk_string(bucket_name)

Expand Down Expand Up @@ -28,6 +27,8 @@ ask_to_overwrite <- function(directory, ask) {
return(TRUE)
}

usethis::ui_yeah(paste0("The folder is not empty. Files may be overwritten. ",
"Proceed?"))
usethis::ui_yeah(paste0(
"The folder is not empty. Files may be overwritten. ",
"Proceed?"
))
}
18 changes: 10 additions & 8 deletions R/rwa-download-files.R
Original file line number Diff line number Diff line change
Expand Up @@ -60,14 +60,16 @@
#' }
#' @export
#'
rwa_download_files <- function(file_list,
directory,
bucket_name,
silent = FALSE,
ask = TRUE,
aws_access_key_id = Sys.getenv("AWS_ACCESS_KEY_ID"),
aws_secret_access_key = Sys.getenv("AWS_SECRET_ACCESS_KEY"),
region = Sys.getenv("AWS_REGION", "ca-central-1")) {
rwa_download_files <- function(
file_list,
directory,
bucket_name,
silent = FALSE,
ask = TRUE,
aws_access_key_id = Sys.getenv("AWS_ACCESS_KEY_ID"),
aws_secret_access_key = Sys.getenv("AWS_SECRET_ACCESS_KEY"),
region = Sys.getenv("AWS_REGION", "ca-central-1")
) {
chk::chk_character(file_list)
chk::chk_dir(directory)
chk::chk_string(bucket_name)
Expand Down
64 changes: 39 additions & 25 deletions R/rwa-list-files.R
Original file line number Diff line number Diff line change
Expand Up @@ -56,13 +56,15 @@
#' )
#' }
#' @export
rwa_list_files <- function(bucket_name,
max_request_size = 1000,
pattern = ".*",
silent = FALSE,
aws_access_key_id = Sys.getenv("AWS_ACCESS_KEY_ID"),
aws_secret_access_key = Sys.getenv("AWS_SECRET_ACCESS_KEY"),
region = Sys.getenv("AWS_REGION", "ca-central-1")) {
rwa_list_files <- function(
bucket_name,
max_request_size = 1000,
pattern = ".*",
silent = FALSE,
aws_access_key_id = Sys.getenv("AWS_ACCESS_KEY_ID"),
aws_secret_access_key = Sys.getenv("AWS_SECRET_ACCESS_KEY"),
region = Sys.getenv("AWS_REGION", "ca-central-1")
) {
chk::chk_string(bucket_name)
chk::chk_whole_number(max_request_size)
chk::chk_gt(max_request_size, value = 0)
Expand All @@ -74,24 +76,31 @@ rwa_list_files <- function(bucket_name,
input_max_request <- max_request_size
start_after <- ""
all_keys <- c()
s3 <- paws::s3(config = list(
credentials = list(creds = list(
access_key_id = aws_access_key_id,
secret_access_key = aws_secret_access_key
)),
region = region
))

s3 <- paws::s3(
config = list(
credentials = list(
creds = list(
access_key_id = aws_access_key_id,
secret_access_key = aws_secret_access_key
)
),
region = region
)
)

while (max_request_size > 0) {
key_list <- s3$list_objects_v2(
Bucket = bucket_name,
MaxKeys = max_request_size,
StartAfter = start_after
)
key_names <- vapply(key_list$Contents, function(x) {
x$Key
}, "")
key_names <- vapply(
key_list$Contents,
function(x) {
x$Key
},
""
)
all_keys <- c(all_keys, key_names)
start_after <- key_names[length(key_names)]
max_request_size <- max_request_size - 1000
Expand All @@ -103,19 +112,24 @@ rwa_list_files <- function(bucket_name,
usethis::ui_info(msg)

if (input_max_request == no_all_keys) {
usethis::ui_warn(paste("{usethis::ui_path('Max_request_size')} matches",
"retrieved files from AWS. Think about",
"increasing",
"{usethis::ui_path('Max_request_size')}",
"as there could be more files present"))
usethis::ui_warn(paste(
"{usethis::ui_path('Max_request_size')} matches",
"retrieved files from AWS. Think about",
"increasing",
"{usethis::ui_path('Max_request_size')}",
"as there could be more files present"
))
}
}

filter_files <- grep(pattern, all_keys, value = TRUE)

if (!silent) {
msg <- paste(length(filter_files), "files returned after",
"{usethis::ui_path('pattern')} is applied")
msg <- paste(
length(filter_files),
"files returned after",
"{usethis::ui_path('pattern')} is applied"
)
usethis::ui_info(msg)
}

Expand Down
36 changes: 17 additions & 19 deletions R/rwa-list-su-files.R
Original file line number Diff line number Diff line change
Expand Up @@ -24,18 +24,20 @@
#' extension = "xlsx", year = 2021)
#' }
#' @export
rwa_list_su_files <- function(bucket_name,
data_type = NULL,
year = NULL,
month = NULL,
day = NULL,
file_name = NULL,
file_extension = NULL,
max_request_size = 1000,
silent = FALSE,
aws_access_key_id = Sys.getenv("AWS_ACCESS_KEY_ID"),
aws_secret_access_key = Sys.getenv("AWS_SECRET_ACCESS_KEY"),
region = Sys.getenv("AWS_REGION", "ca-central-1")) {
rwa_list_su_files <- function(
bucket_name,
data_type = NULL,
year = NULL,
month = NULL,
day = NULL,
file_name = NULL,
file_extension = NULL,
max_request_size = 1000,
silent = FALSE,
aws_access_key_id = Sys.getenv("AWS_ACCESS_KEY_ID"),
aws_secret_access_key = Sys.getenv("AWS_SECRET_ACCESS_KEY"),
region = Sys.getenv("AWS_REGION", "ca-central-1")
) {
chk::chk_null_or(data_type, vld = chk::vld_string)
chk::chk_null_or(year, vld = chk::vld_whole_number)
chk::chk_range(year, range = c(1900, 2100))
Expand All @@ -59,13 +61,9 @@ rwa_list_su_files <- function(bucket_name,
month <- pad_dates(month)

date <- paste(year, month, day, sep = "-")
regex_pattern <- paste(data_type,
date,
file_name,
file_extension,
sep = ".*"
)
file_list <- rwa_list_files(bucket_name,
regex_pattern <- paste(data_type, date, file_name, file_extension, sep = ".*")
file_list <- rwa_list_files(
bucket_name,
pattern = regex_pattern,
max_request_size = max_request_size,
silent = silent,
Expand Down
34 changes: 20 additions & 14 deletions R/rwa-upload-files.R
Original file line number Diff line number Diff line change
Expand Up @@ -54,13 +54,15 @@
#' }
#'
#' @export
rwa_upload_files <- function(file_list,
directory = ".",
bucket_name,
bucket_path = "",
aws_access_key_id = Sys.getenv("AWS_ACCESS_KEY_ID"),
aws_secret_access_key = Sys.getenv("AWS_SECRET_ACCESS_KEY"),
region = Sys.getenv("AWS_REGION", "ca-central-1")) {
rwa_upload_files <- function(
file_list,
directory = ".",
bucket_name,
bucket_path = "",
aws_access_key_id = Sys.getenv("AWS_ACCESS_KEY_ID"),
aws_secret_access_key = Sys.getenv("AWS_SECRET_ACCESS_KEY"),
region = Sys.getenv("AWS_REGION", "ca-central-1")
) {
chk::chk_character(file_list)
chk::chk_null_or(directory, vld = chk::vld_dir)
chk::chk_string(bucket_name)
Expand All @@ -69,13 +71,17 @@ rwa_upload_files <- function(file_list,
chk::chk_string(aws_secret_access_key)
chk::chk_string(region)

s3 <- paws::s3(config = list(
credentials = list(creds = list(
access_key_id = aws_access_key_id,
secret_access_key = aws_secret_access_key
)),
region = region
))
s3 <- paws::s3(
config = list(
credentials = list(
creds = list(
access_key_id = aws_access_key_id,
secret_access_key = aws_secret_access_key
)
),
region = region
)
)

for (file in file_list) {
local_path <- paste0(directory, "/", file)
Expand Down
7 changes: 7 additions & 0 deletions air.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
[format]
line-width = 80
indent-width = 2
indent-style = "space"
line-ending = "auto"
persistent-line-breaks = true
default-exclude = true
35 changes: 31 additions & 4 deletions scripts/build.R
Original file line number Diff line number Diff line change
@@ -1,10 +1,37 @@
roxygen2md::roxygen2md()
styler::style_pkg(filetype = c("R", "Rmd"))
lintr::lint_package()
styler::style_pkg(
scope = "line_breaks",
filetype = c("R", "Rmd")
)

devtools::test()
lintr::lint_package(
linters = lintr::linters_with_defaults(
line_length_linter = lintr::line_length_linter(1000),
object_name_linter = lintr::object_name_linter(regexes = ".*")
)
)

roxygen2md::roxygen2md()
devtools::document()

devtools::build_readme()

# Note: Only use pkgdown to build a documentation website for public facing packages
pkgdown::build_home()
pkgdown::build_reference()
pkgdown::build_site()
browseURL("docs/index.html")

devtools::test()
devtools::check()

# Test files that have less then 100% coverage
covr:::tally_coverage(covr::package_coverage()) |>
dplyr::summarize(
percent_coverage = mean(value > 0) * 100,
.by = filename
) |>
dplyr::filter(percent_coverage < 100) |>
dplyr::arrange(percent_coverage)

# Report for all test files
covr::report(covr::package_coverage())
Loading
Loading