diff --git a/R/RSocrata.R b/R/RSocrata.R index ebc554d..45fac3b 100644 --- a/R/RSocrata.R +++ b/R/RSocrata.R @@ -366,13 +366,19 @@ read.socrata <- function(url, app_token = NULL, email = NULL, password = NULL, # PAGE through data and combine # if user limit is provided do not page # if no limit $provided, loop until all data is paged - while (nrow(page) > 0 & !limitProvided) { - query <- paste(validUrl, if(is.null(parsedUrl$query)) {'?'} else {"&"}, - '$limit=50000&$offset=', nrow(result), sep='') - response <- getResponse(query, email, password) - page <- getContentAsDataFrame(response) - result <- rbind.fill(result, page) # accumulate - } + + # If an offset has been used, and limit has not been provided, skip the paging and return a message + if(!is.null(parsedUrl$query$`$offset`) & !limitProvided){ + message(paste('Returning results from offset of', format(parsedUrl$query$`$offset`, big.mark=","))) + } else { + while (nrow(page) > 0 & !limitProvided) { + query <- paste(validUrl, if(is.null(parsedUrl$query)) {'?'} else {"&"}, + '$limit=50000&$offset=', nrow(result), sep='') + response <- getResponse(query, email, password) + page <- getContentAsDataFrame(response) + result <- rbind.fill(result, page) # accumulate + } + } if (is.null(dataTypes)) { warning("Dates and currency fields will be converted to character") } else { diff --git a/tests/testthat/test-all.R b/tests/testthat/test-all.R index 137c65f..800fd53 100644 --- a/tests/testthat/test-all.R +++ b/tests/testthat/test-all.R @@ -550,3 +550,14 @@ test_that("getContentAsDataFrame does not get caught in infinite loop", { expect_equal("data.frame", class(df), label="class") }) + +context("User-defined offset") + +test_that("User can specify offset", { + + rows_hist = 108000 # rows of data previously read + df <- read.socrata(url = paste0("https://data.cambridgema.gov/resource/gxzm-dpwp.csv?$offset=", rows_hist)) + + expect_equal("data.frame", class(df), label="class") +}) +