1 |
.onLoad <- function(lib, pkg) { |
|
2 | ! |
if (Sys.getenv("RECEPTIVITI_URL") == "") Sys.setenv(RECEPTIVITI_URL = "https://api.receptiviti.com/") |
3 |
} |
|
4 | ||
5 |
#' Receptiviti API |
|
6 |
#' |
|
7 |
#' The main function to access the \href{https://www.receptiviti.com}{Receptiviti} API. |
|
8 |
#' |
|
9 |
#' @param text A character vector with text to be processed, path to a directory containing files, or a vector of file paths. |
|
10 |
#' If a single path to a directory, each file is collapsed to a single text. If a path to a file or files, |
|
11 |
#' each line or row is treated as a separate text, unless \code{collapse_lines} is \code{TRUE} (in which case, |
|
12 |
#' files will be read in as part of bundles at processing time, as is always the case when a directory). |
|
13 |
#' Use \code{files} to more reliably enter files, or \code{dir} to more reliably specify a directory. |
|
14 |
#' @param output Path to a \code{.csv} file to write results to. If this already exists, set \code{overwrite} to \code{TRUE} |
|
15 |
#' to overwrite it. |
|
16 |
#' @param id Vector of unique IDs the same length as \code{text}, to be included in the results. |
|
17 |
#' @param text_column,id_column Column name of text/id, if \code{text} is a matrix-like object, or a path to a csv file. |
|
18 |
#' @param files A list of file paths, as alternate entry to \code{text}. |
|
19 |
#' @param dir A directory to search for files in, as alternate entry to \code{text}. |
|
20 |
#' @param file_type File extension to search for, if \code{text} is the path to a directory containing files to be read in. |
|
21 |
#' @param encoding Encoding of file(s) to be read in. If not specified, this will be detected, which can fail, |
|
22 |
#' resulting in mis-encoded characters; for best (and fasted) results, specify encoding. |
|
23 |
#' @param return_text Logical; if \code{TRUE}, \code{text} is included as the first column of the result. |
|
24 |
#' @param api_args A list of additional arguments to pass to the API (e.g., \code{list(sallee_mode = "sparse")}). Defaults to the |
|
25 |
#' \code{receptiviti.api_args} option. |
|
26 |
#' @param frameworks A vector of frameworks to include results from. Texts are always scored with all available framework -- |
|
27 |
#' this just specifies what to return. Defaults to \code{all}, to return all scored frameworks. Can be set by the |
|
28 |
#' \code{receptiviti.frameworks} option (e.g., \code{options(receptiviti.frameworks = c("liwc", "sallee"))}). |
|
29 |
#' @param framework_prefix Logical; if \code{FALSE}, will remove the framework prefix from column names, which may result in duplicates. |
|
30 |
#' If this is not specified, and 1 framework is selected, or \code{as_list} is \code{TRUE}, will default to remove prefixes. |
|
31 |
#' @param as_list Logical; if \code{TRUE}, returns a list with frameworks in separate entries. |
|
32 |
#' @param bundle_size Number of texts to include in each request; between 1 and 1,000. |
|
33 |
#' @param bundle_byte_limit Memory limit (in bytes) of each bundle, under \code{1e7} (10 MB, which is the API's limit). |
|
34 |
#' May need to be lower than the API's limit, depending on the system's requesting library. |
|
35 |
#' @param collapse_lines Logical; if \code{TRUE}, and \code{text} contains paths to files, each file is treated as a single text. |
|
36 |
#' @param retry_limit Maximum number of times each request can be retried after hitting a rate limit. |
|
37 |
#' @param overwrite Logical; if \code{TRUE}, will overwrite an existing \code{output} file. |
|
38 |
#' @param compress Logical; if \code{TRUE}, will save as an \code{xz}-compressed file. |
|
39 |
#' @param make_request Logical; if \code{FALSE}, a request is not made. This could be useful if you want to be sure and |
|
40 |
#' load from one of the caches, but aren't sure that all results exist there; it will error out if it encounters |
|
41 |
#' texts it has no other source for. |
|
42 |
#' @param text_as_paths Logical; if \code{TRUE}, ensures \code{text} is treated as a vector of file paths. Otherwise, this will be |
|
43 |
#' determined if there are no \code{NA}s in \code{text} and every entry is under 500 characters long. |
|
44 |
#' @param cache Path to a directory in which to save unique results for reuse; defaults to |
|
45 |
#' \code{Sys.getenv(}\code{"RECEPTIVITI_CACHE")}. See the Cache section for details. |
|
46 |
#' @param cache_overwrite Logical; if \code{TRUE}, will write results to the cache without reading from it. This could be used |
|
47 |
#' if you want fresh results to be cached without clearing the cache. |
|
48 |
#' @param cache_format Format of the cache database; see \code{\link[arrow]{FileFormat}}. |
|
49 |
#' Defaults to \code{Sys.getenv(}\code{"RECEPTIVITI_CACHE_FORMAT")}. |
|
50 |
#' @param clear_cache Logical; if \code{TRUE}, will clear any existing files in the cache. Use \code{cache_overwrite} if |
|
51 |
#' you want fresh results without clearing or disabling the cache. Use \code{cache = FALSE} to disable the cache. |
|
52 |
#' @param request_cache Logical; if \code{FALSE}, will always make a fresh request, rather than using the response |
|
53 |
#' from a previous identical request. |
|
54 |
#' @param cores Number of CPU cores to split bundles across, if there are multiple bundles. See the Parallelization section. |
|
55 |
#' @param use_future Logical; if \code{TRUE}, uses a \code{future} back-end to process bundles, in which case, |
|
56 |
#' parallelization can be controlled with the \code{\link[future]{plan}} function (e.g., \code{plan("multisession")} |
|
57 |
#' to use multiple cores); this is required to see progress bars when using multiple cores. See the Parallelization section. |
|
58 |
#' @param in_memory Logical; if \code{FALSE}, will write bundles to temporary files, and only load them as they are being requested. |
|
59 |
#' @param clear_scratch_cache Logical; if \code{FALSE}, will preserve the bundles written when \code{in_memory} is \code{TRUE}, after |
|
60 |
#' the request has been made. |
|
61 |
#' @param verbose Logical; if \code{TRUE}, will show status messages. |
|
62 |
#' @param key API Key; defaults to \code{Sys.getenv("RECEPTIVITI_KEY")}. |
|
63 |
#' @param secret API Secret; defaults to \code{Sys.getenv("RECEPTIVITI_SECRET")}. |
|
64 |
#' @param url API URL; defaults to \code{Sys.getenv("RECEPTIVITI_URL")}, which defaults to |
|
65 |
#' \code{"https://api.receptiviti.com/"}. |
|
66 |
#' @param version API version; defaults to \code{Sys.getenv("RECEPTIVITI_VERSION")}, which defaults to |
|
67 |
#' \code{"v1"}. |
|
68 |
#' @param endpoint API endpoint (path name after the version); defaults to \code{Sys.getenv("RECEPTIVITI_ENDPOINT")}, |
|
69 |
#' which defaults to \code{"framework"}. |
|
70 |
#' @param include_headers Logical; if \code{TRUE}, \code{receptiviti_status}'s verbose message will include |
|
71 |
#' the HTTP headers. |
|
72 |
#' |
|
73 |
#' @returns A \code{data.frame} with columns for \code{text} (if \code{return_text} is \code{TRUE}; the originally entered text), |
|
74 |
#' \code{id} (if one was provided), \code{text_hash} (the MD5 hash of the text), a column each for relevant entries in \code{api_args}, |
|
75 |
#' and scores from each included framework (e.g., \code{summary.word_count} and \code{liwc.i}). If \code{as_list} is \code{TRUE}, |
|
76 |
#' returns a list with a named entry containing such a \code{data.frame} for each framework. |
|
77 |
#' |
|
78 |
#' @section Cache: |
|
79 |
#' If the \code{cache} argument is specified, results for unique texts are saved in an |
|
80 |
#' \href{https://arrow.apache.org}{Arrow} database in the cache location |
|
81 |
#' (\code{Sys.getenv(}\code{"RECEPTIVITI_CACHE")}), and are retrieved with subsequent requests. |
|
82 |
#' This ensures that the exact same texts are not re-sent to the API. |
|
83 |
#' This does, however, add some processing time and disc space usage. |
|
84 |
#' |
|
85 |
#' If \code{cache} is \code{TRUE}, a default directory (\code{receptiviti_cache}) will be looked for |
|
86 |
#' in the system's temporary directory (which is usually the parent of \code{tempdir()}). |
|
87 |
#' If this does not exist, you will be asked if it should be created. |
|
88 |
#' |
|
89 |
#' The primary cache is checked when each bundle is processed, and existing results are loaded at |
|
90 |
#' that time. When processing many bundles in parallel, and many results have been cached, |
|
91 |
#' this can cause the system to freeze and potentially crash. |
|
92 |
#' To avoid this, limit the number of cores, or disable parallel processing. |
|
93 |
#' |
|
94 |
#' The \code{cache_format} arguments (or the \code{RECEPTIVITI_CACHE_FORMAT} environment variable) can be used to adjust the format of the cache. |
|
95 |
#' |
|
96 |
#' You can use the cache independently with \code{open_database(Sys.getenv("RECEPTIVITI_CACHE"))}. |
|
97 |
#' |
|
98 |
#' You can also set the \code{clear_cache} argument to \code{TRUE} to clear the cache before it is used again, which may be useful |
|
99 |
#' if the cache has gotten big, or you know new results will be returned. Even if a cached result exists, it will be |
|
100 |
#' reprocessed if it does not have all of the variables of new results, but this depends on there being at least 1 uncached |
|
101 |
#' result. If, for instance, you add a framework to your account and want to reprocess a previously processed set of texts, |
|
102 |
#' you would need to first clear the cache. |
|
103 |
#' |
|
104 |
#' Either way, duplicated texts within the same call will only be sent once. |
|
105 |
#' |
|
106 |
#' The \code{request_cache} argument controls a more temporary cache of each bundle request. This is cleared when the |
|
107 |
#' R session ends. You might want to set this to \code{FALSE} if a new framework becomes available on your account |
|
108 |
#' and you want to process a set of text you already processed in the current R session without restarting. |
|
109 |
#' |
|
110 |
#' Another temporary cache is made when \code{in_memory} is \code{FALSE}, which is the default when processing |
|
111 |
#' in parallel (when \code{cores} is over \code{1} or \code{use_future} is \code{TRUE}). This contains |
|
112 |
#' a file for each unique bundle, which is read in as needed by the parallel workers. |
|
113 |
#' |
|
114 |
#' @section Parallelization: |
|
115 |
#' \code{text}s are split into bundles based on the \code{bundle_size} argument. Each bundle represents |
|
116 |
#' a single request to the API, which is why they are limited to 1000 texts and a total size of 10 MB. |
|
117 |
#' When there is more than one bundle and either \code{cores} is greater than 1 or \code{use_future} is \code{TRUE} (and you've |
|
118 |
#' externally specified a \code{\link[future]{plan}}), bundles are processed by multiple cores. |
|
119 |
#' |
|
120 |
#' If you have texts spread across multiple files, they can be most efficiently processed in parallel |
|
121 |
#' if each file contains a single text (potentially collapsed from multiple lines). If files contain |
|
122 |
#' multiple texts (i.e., \code{collapse_lines = FALSE}), then texts need to be read in before bundling |
|
123 |
#' in order to ensure bundles are under the length limit. |
|
124 |
#' |
|
125 |
#' Whether processing in serial or parallel, progress bars can be specified externally with |
|
126 |
#' \code{\link[progressr]{handlers}}; see examples. |
|
127 |
#' @examples |
|
128 |
#' \dontrun{ |
|
129 |
#' |
|
130 |
#' # check that the API is available, and your credentials work |
|
131 |
#' receptiviti_status() |
|
132 |
#' |
|
133 |
#' # score a single text |
|
134 |
#' single <- receptiviti("a text to score") |
|
135 |
#' |
|
136 |
#' # score multiple texts, and write results to a file |
|
137 |
#' multi <- receptiviti(c("first text to score", "second text"), "filename.csv") |
|
138 |
#' |
|
139 |
#' # score many texts in separate files |
|
140 |
#' ## defaults to look for .txt files |
|
141 |
#' file_results <- receptiviti(dir = "./path/to/txt_folder") |
|
142 |
#' |
|
143 |
#' ## could be .csv |
|
144 |
#' file_results <- receptiviti( |
|
145 |
#' dir = "./path/to/csv_folder", |
|
146 |
#' text_column = "text", file_type = "csv" |
|
147 |
#' ) |
|
148 |
#' |
|
149 |
#' # score many texts from a file, with a progress bar |
|
150 |
#' ## set up cores and progress bar (only necessary if you want the progress bar) |
|
151 |
#' future::plan("multisession") |
|
152 |
#' progressr::handlers(global = TRUE) |
|
153 |
#' progressr::handlers("progress") |
|
154 |
#' |
|
155 |
#' ## make request |
|
156 |
#' results <- receptiviti( |
|
157 |
#' "./path/to/largefile.csv", |
|
158 |
#' text_column = "text", use_future = TRUE |
|
159 |
#' ) |
|
160 |
#' } |
|
161 |
#' @importFrom curl new_handle curl_fetch_memory curl_fetch_disk |
|
162 |
#' @importFrom jsonlite toJSON fromJSON read_json |
|
163 |
#' @importFrom utils object.size |
|
164 |
#' @importFrom digest digest |
|
165 |
#' @importFrom parallel detectCores makeCluster clusterExport parLapplyLB parLapply stopCluster |
|
166 |
#' @importFrom progressr progressor |
|
167 |
#' @importFrom stringi stri_enc_detect |
|
168 |
#' @export |
|
169 | ||
170 |
receptiviti <- function(text, output = NULL, id = NULL, text_column = NULL, id_column = NULL, files = NULL, dir = NULL, |
|
171 |
file_type = "txt", encoding = NULL, return_text = FALSE, api_args = getOption("receptiviti.api_args", list()), |
|
172 |
frameworks = getOption("receptiviti.frameworks", "all"), framework_prefix = TRUE, as_list = FALSE, |
|
173 |
bundle_size = 1000, bundle_byte_limit = 75e5, collapse_lines = FALSE, retry_limit = 50, clear_cache = FALSE, |
|
174 |
clear_scratch_cache = TRUE, request_cache = TRUE, cores = detectCores() - 1, use_future = FALSE, |
|
175 |
in_memory = TRUE, verbose = FALSE, overwrite = FALSE, compress = FALSE, make_request = TRUE, |
|
176 |
text_as_paths = FALSE, cache = Sys.getenv("RECEPTIVITI_CACHE"), cache_overwrite = FALSE, |
|
177 |
cache_format = Sys.getenv("RECEPTIVITI_CACHE_FORMAT", "parquet"), key = Sys.getenv("RECEPTIVITI_KEY"), |
|
178 |
secret = Sys.getenv("RECEPTIVITI_SECRET"), url = Sys.getenv("RECEPTIVITI_URL"), |
|
179 |
version = Sys.getenv("RECEPTIVITI_VERSION"), endpoint = Sys.getenv("RECEPTIVITI_ENDPOINT")) { |
|
180 |
# check input |
|
181 | 66x |
final_res <- text_hash <- bin <- NULL |
182 | 66x |
if (!is.null(output)) { |
183 | ! |
if (!file.exists(output) && file.exists(paste0(output, ".xz"))) output <- paste0(output, ".xz") |
184 | 1x |
if (!overwrite && file.exists(output)) stop("output file already exists; use overwrite = TRUE to overwrite it", call. = FALSE) |
185 |
} |
|
186 | 65x |
if (isTRUE(cache)) { |
187 | 2x |
if (!requireNamespace("arrow", quietly = TRUE)) { |
188 | ! |
stop("install the `arrow` package to enable the cache", call. = FALSE) |
189 |
} |
|
190 | 2x |
temp <- dirname(tempdir()) |
191 | ! |
if (basename(temp) == "working_dir") temp <- dirname(dirname(temp)) |
192 | 2x |
cache <- paste0(temp, "/receptiviti_cache") |
193 | 2x |
if (!dir.exists(cache)) { |
194 | ! |
if (interactive() && !isFALSE(getOption("receptiviti.cache_prompt")) && |
195 | ! |
grepl("^(?:[Yy1]|$)", readline("Do you want to establish a default cache? [Y/n] "))) { |
196 | ! |
dir.create(cache, FALSE) |
197 |
} else { |
|
198 | ! |
options(receptiviti.cache_prompt = FALSE) |
199 | ! |
cache <- FALSE |
200 |
} |
|
201 |
} |
|
202 |
} |
|
203 | 65x |
if (use_future && !requireNamespace("future.apply", quietly = TRUE)) { |
204 | ! |
stop("install the `future.apply` package to use future", call. = FALSE) |
205 |
} |
|
206 | 65x |
st <- proc.time()[[3]] |
207 | 65x |
text_as_dir <- FALSE |
208 | 65x |
if (missing(text)) { |
209 | 3x |
if (!is.null(dir)) { |
210 | ! |
if (!dir.exists(dir)) stop("entered dir does not exist", call. = FALSE) |
211 | 1x |
text <- dir |
212 | 1x |
text_as_dir <- TRUE |
213 | 2x |
} else if (!is.null(files)) { |
214 | 1x |
text <- files |
215 | 1x |
text_as_paths <- TRUE |
216 |
} else { |
|
217 | 1x |
stop("enter text as the first argument, or use files or dir", call. = FALSE) |
218 |
} |
|
219 |
} |
|
220 | 64x |
if (text_as_paths) { |
221 | 1x |
if (anyNA(text)) stop("NAs are not allowed in text when being treated as file paths", call. = FALSE) |
222 | 1x |
if (!all(file.exists(text))) stop("not all of the files in text exist", call. = FALSE) |
223 |
} |
|
224 | 62x |
read_in <- FALSE |
225 | 62x |
handle_encoding <- function(file) { |
226 | 212x |
if (is.null(encoding)) { |
227 | 210x |
unlist(stringi::stri_enc_detect(readBin(file, "raw", file.size(file)))[[1]])[[1]] |
228 |
} else { |
|
229 | 2x |
encoding |
230 |
} |
|
231 |
} |
|
232 | 62x |
if (text_as_dir || text_as_paths || (is.character(text) && !anyNA(text) && all(nchar(text) < 500))) { |
233 | 45x |
if (text_as_dir || length(text) == 1 && dir.exists(text)) { |
234 | ! |
if (verbose) message("reading in texts from directory: ", text, " (", round(proc.time()[[3]] - st, 4), ")") |
235 | 5x |
text_as_paths <- TRUE |
236 | 5x |
text <- normalizePath(list.files(text, file_type, full.names = TRUE), "/", FALSE) |
237 |
} |
|
238 | 45x |
if (text_as_paths || all(file.exists(text))) { |
239 | 15x |
text_as_paths <- collapse_lines |
240 | 15x |
if (!collapse_lines) { |
241 | ! |
if (verbose) message("reading in texts from file list (", round(proc.time()[[3]] - st, 4), ")") |
242 | 12x |
if (missing(id_column)) names(text) <- if (length(id) != length(text)) text else id |
243 | 13x |
if (all(grepl("\\.csv", text, TRUE))) { |
244 | 2x |
if (is.null(text_column)) stop("text appears to point to csv files, but text_column was not specified", call. = FALSE) |
245 | 4x |
read_in <- TRUE |
246 | 4x |
text <- unlist(lapply(text, function(f) { |
247 | 4x |
d <- tryCatch(arrow::read_csv_arrow(f, read_options = arrow::CsvReadOptions$create( |
248 | 4x |
encoding = handle_encoding(f) |
249 | 4x |
), col_select = c(text_column, id_column)), error = function(e) NULL) |
250 | 1x |
if (is.null(d)) stop("failed to read in file ", f, call. = FALSE) |
251 | 3x |
if (!is.null(id_column) && id_column %in% colnames(d)) { |
252 | 1x |
structure(d[, text_column, drop = TRUE], names = d[, id_column, drop = TRUE]) |
253 |
} else { |
|
254 | 2x |
d[, text_column, drop = TRUE] |
255 |
} |
|
256 |
})) |
|
257 |
} else { |
|
258 | 7x |
text <- unlist(lapply(text, function(f) { |
259 | 156x |
tryCatch( |
260 |
{ |
|
261 | 156x |
con <- file(f, encoding = handle_encoding(f)) |
262 | 156x |
on.exit(close(con)) |
263 | 156x |
d <- readLines(con, warn = FALSE, skipNul = TRUE) |
264 | 156x |
d[d != ""] |
265 |
}, |
|
266 | 156x |
error = function(e) stop("failed to read in file ", f, call. = FALSE) |
267 |
) |
|
268 |
})) |
|
269 |
} |
|
270 | 10x |
id <- names(text) |
271 |
} |
|
272 | 30x |
} else if (length(text) == 1 && dirname(text) != "." && dir.exists(dirname(dirname(text)))) { |
273 | ! |
stop("text appears to be a directory, but it does not exist") |
274 |
} |
|
275 | 42x |
if (text_as_paths && missing(id)) { |
276 | 2x |
id <- text |
277 | ! |
if (anyDuplicated(id)) id <- names(unlist(lapply(split(id, factor(id, unique(id))), seq_along))) |
278 |
} |
|
279 |
} |
|
280 | 59x |
if (is.null(dim(text))) { |
281 | 53x |
if (!read_in) { |
282 | 1x |
if (!text_as_paths && !is.null(text_column)) stop("text_column is specified, but text has no columns", call. = FALSE) |
283 | 1x |
if (!is.null(id_column)) stop("id_column is specified, but text has no columns", call. = FALSE) |
284 |
} |
|
285 |
} else { |
|
286 | 1x |
if (length(id) == 1 && id %in% colnames(text)) id_column <- id |
287 | 6x |
if (!is.null(id_column)) { |
288 | 2x |
if (id_column %in% colnames(text)) { |
289 | 1x |
id <- text[, id_column, drop = TRUE] |
290 |
} else { |
|
291 | 1x |
stop("id_column not found in text", call. = FALSE) |
292 |
} |
|
293 |
} |
|
294 | 5x |
if (!is.null(text_column)) { |
295 | 3x |
if (text_column %in% colnames(text)) { |
296 | 2x |
text <- text[, text_column, drop = TRUE] |
297 |
} else { |
|
298 | 1x |
if (!text_as_paths) stop("text_column not found in text", call. = FALSE) |
299 |
} |
|
300 |
} |
|
301 | 4x |
if (!is.null(dim(text))) { |
302 | 2x |
if (ncol(text) == 1) { |
303 | 1x |
text <- text[, 1, drop = TRUE] |
304 |
} else { |
|
305 | 1x |
stop("text has dimensions, but no text_column column", call. = FALSE) |
306 |
} |
|
307 |
} |
|
308 |
} |
|
309 | 2x |
if (!is.character(text)) text <- as.character(text) |
310 | 1x |
if (!length(text)) stop("no texts were found after resolving the text argument") |
311 | 4x |
if (length(id) && !is.character(id)) id <- as.character(id) |
312 | 53x |
provided_id <- FALSE |
313 | 53x |
if (length(id)) { |
314 | 1x |
if (length(id) != length(text)) stop("id is not the same length as text", call. = FALSE) |
315 | 1x |
if (anyDuplicated(id)) stop("id contains duplicate values", call. = FALSE) |
316 | 14x |
provided_id <- TRUE |
317 |
} else { |
|
318 | 37x |
id <- paste0("t", seq_along(text)) |
319 |
} |
|
320 | 1x |
if (!is.numeric(retry_limit)) retry_limit <- 0 |
321 | 51x |
url_parts <- unlist(strsplit(regmatches( |
322 | 51x |
url, gregexpr("/[Vv]\\d+(?:/[^/]+)?", url) |
323 | 51x |
)[[1]], "/", fixed = TRUE)) |
324 | 51x |
if (version == "") version <- if (length(url_parts) > 1) url_parts[[2]] else "v1" |
325 | 51x |
if (endpoint == "") { |
326 | 51x |
endpoint <- if (length(url_parts) > 2) { |
327 | 1x |
url_parts[[3]] |
328 |
} else { |
|
329 | ! |
if (tolower(version) == "v1") "framework" else "taxonomies" |
330 |
} |
|
331 |
} |
|
332 | 51x |
url <- paste0(sub("/+[Vv]\\d+(/.*)?$|/+$", "", url), "/", version, "/") |
333 | 51x |
full_url <- paste0(url, endpoint, "/bulk") |
334 | ! |
if (!is.list(api_args)) api_args <- as.list(api_args) |
335 | 51x |
args_hash <- digest::digest(jsonlite::toJSON(c( |
336 | 51x |
api_args, |
337 | 51x |
url = full_url, key = key, secret = secret |
338 | 51x |
), auto_unbox = TRUE), serialize = FALSE) |
339 | ||
340 |
# ping API |
|
341 | 51x |
if (make_request) { |
342 | 2x |
if (verbose) message("pinging API (", round(proc.time()[[3]] - st, 4), ")") |
343 | 49x |
ping <- receptiviti_status(url, key, secret, verbose = FALSE) |
344 | 1x |
if (is.null(ping)) stop("URL is unreachable", call. = FALSE) |
345 | 1x |
if (ping$status_code != 200) stop(ping$status_message, call. = FALSE) |
346 |
} |
|
347 | ||
348 |
# prepare text |
|
349 | 2x |
if (verbose) message("preparing text (", round(proc.time()[[3]] - st, 4), ")") |
350 | 47x |
data <- data.frame(text = text, id = id, stringsAsFactors = FALSE) |
351 | 47x |
text <- data[!is.na(data$text) & data$text != "" & !duplicated(data$text), ] |
352 | 1x |
if (!nrow(text)) stop("no valid texts to process", call. = FALSE) |
353 | 1x |
if (!is.numeric(bundle_size)) bundle_size <- 1000 |
354 | 46x |
n_texts <- nrow(text) |
355 | 46x |
n <- ceiling(n_texts / min(1000, max(1, bundle_size))) |
356 | 46x |
bundles <- split(text, sort(rep_len(seq_len(n), nrow(text)))) |
357 | 46x |
size_fun <- if (text_as_paths) function(b) sum(file.size(b$text)) else object.size |
358 | 46x |
for (i in rev(seq_along(bundles))) { |
359 | 157x |
size <- size_fun(bundles[[i]]) |
360 | 157x |
if (size > bundle_byte_limit) { |
361 | 2x |
sizes <- vapply(seq_len(nrow(bundles[[i]])), function(r) as.numeric(size_fun(bundles[[i]][r, ])), 0) |
362 | 2x |
if (any(sizes > bundle_byte_limit)) { |
363 | 1x |
stop( |
364 | 1x |
"one of your texts is over the individual size limit (", bundle_byte_limit / 1e6, " MB)", |
365 | 1x |
call. = FALSE |
366 |
) |
|
367 |
} |
|
368 | 1x |
bins <- rep(1, length(sizes)) |
369 | 1x |
bin_size <- 0 |
370 | 1x |
bi <- 1 |
371 | 1x |
for (ti in seq_along(bins)) { |
372 | 50x |
bin_size <- bin_size + sizes[ti] |
373 | 50x |
if (bin_size > bundle_byte_limit) { |
374 | 2x |
bin_size <- sizes[ti] |
375 | 2x |
bi <- bi + 1 |
376 |
} |
|
377 | 50x |
bins[ti] <- bi |
378 |
} |
|
379 | 1x |
bundles <- c(bundles[-i], unname(split(bundles[[i]], paste0(i, ".", bins)))) |
380 |
} |
|
381 |
} |
|
382 | 45x |
n_bundles <- length(bundles) |
383 | 45x |
bundle_ref <- if (n_bundles == 1) "bundle" else "bundles" |
384 | 2x |
if (verbose) message("prepared text in ", n_bundles, " ", bundle_ref, " (", round(proc.time()[[3]] - st, 4), ")") |
385 | ||
386 |
# prepare cache |
|
387 | 45x |
if (is.character(cache) && cache != "") { |
388 | 30x |
temp <- normalizePath(cache, "/", FALSE) |
389 | 30x |
cache <- TRUE |
390 | 2x |
if (clear_cache) unlink(temp, recursive = TRUE) |
391 | 30x |
dir.create(temp, FALSE) |
392 |
} else { |
|
393 | 15x |
temp <- NULL |
394 | 15x |
cache <- FALSE |
395 |
} |
|
396 | ||
397 | 45x |
check_cache <- cache && !cache_overwrite |
398 | 45x |
auth <- paste0(key, ":", secret) |
399 | 2x |
if (missing(in_memory) && (use_future || cores > 1) && n_bundles > cores) in_memory <- FALSE |
400 | 45x |
request_scratch <- NULL |
401 | 45x |
if (!in_memory) { |
402 | ! |
if (verbose) message("writing ", bundle_ref, " to disc (", round(proc.time()[[3]] - st, 4), ")") |
403 | 4x |
request_scratch <- paste0(tempdir(), "/receptiviti_request_scratch/") |
404 | 4x |
dir.create(request_scratch, FALSE) |
405 | 4x |
if (clear_scratch_cache) on.exit(unlink(request_scratch, recursive = TRUE)) |
406 | 4x |
bundles <- vapply(bundles, function(b) { |
407 | 106x |
scratch_bundle <- paste0(request_scratch, digest::digest(b), ".rds") |
408 | 106x |
if (!file.exists(scratch_bundle)) saveRDS(b, scratch_bundle, compress = FALSE) |
409 | 106x |
scratch_bundle |
410 | 4x |
}, "", USE.NAMES = FALSE) |
411 |
} |
|
412 | ||
413 | 45x |
doprocess <- function(bundles, cores, future) { |
414 | 8x |
env <- parent.frame() |
415 | 8x |
if (future) { |
416 | 1x |
eval(expression(future.apply::future_lapply(bundles, process)), envir = env) |
417 |
} else { |
|
418 | 7x |
cl <- parallel::makeCluster(cores) |
419 | 7x |
parallel::clusterExport(cl, ls(envir = env), env) |
420 | 7x |
on.exit(parallel::stopCluster(cl)) |
421 | 7x |
(if (length(bundles) > cores * 2) parallel::parLapplyLB else parallel::parLapply)(cl, bundles, process) |
422 |
} |
|
423 |
} |
|
424 | ||
425 | 45x |
request <- function(body, bin, ids, attempt = retry_limit) { |
426 | 307x |
unpack <- function(d) { |
427 | 21270x |
if (is.list(d)) as.data.frame(lapply(d, unpack), optional = TRUE) else d |
428 |
} |
|
429 | 307x |
json <- jsonlite::toJSON(unname(body), auto_unbox = TRUE) |
430 | 307x |
temp_file <- paste0(tempdir(), "/", digest::digest(json, serialize = FALSE), ".json") |
431 | 288x |
if (!request_cache) unlink(temp_file) |
432 | 307x |
res <- NULL |
433 | 307x |
if (!file.exists(temp_file)) { |
434 | 302x |
if (make_request) { |
435 | 301x |
handler <- tryCatch( |
436 | 301x |
curl::new_handle(httpauth = 1, userpwd = auth, copypostfields = json), |
437 | 301x |
error = function(e) e$message |
438 |
) |
|
439 | 301x |
if (is.character(handler)) { |
440 | ! |
stop(if (grepl("libcurl", handler, fixed = TRUE)) { |
441 | ! |
"libcurl encountered an error; try setting the bundle_byte_limit argument to a smaller value" |
442 |
} else { |
|
443 | ! |
paste("failed to create handler:", handler) |
444 | ! |
}, call. = FALSE) |
445 |
} |
|
446 | 301x |
res <- curl::curl_fetch_disk(full_url, temp_file, handler) |
447 |
} else { |
|
448 | 1x |
stop("make_request is FALSE, but there are texts with no cached results", call. = FALSE) |
449 |
} |
|
450 |
} |
|
451 | 306x |
result <- if (file.exists(temp_file)) { |
452 | 306x |
if (is.null(res$type) || grepl("application/json", res$type, fixed = TRUE)) { |
453 | 306x |
tryCatch( |
454 | 306x |
jsonlite::read_json(temp_file, simplifyVector = TRUE), |
455 | 306x |
error = function(e) list(message = "invalid response format") |
456 |
) |
|
457 |
} else { |
|
458 | ! |
list(message = "invalid response format") |
459 |
} |
|
460 |
} else { |
|
461 | ! |
list(message = rawToChar(res$content)) |
462 |
} |
|
463 | 306x |
if (!is.null(result$results) || is.null(result$message)) { |
464 | 82x |
if (!is.null(result$results)) result <- result$results |
465 | 83x |
if ("error" %in% names(result)) { |
466 | 1x |
su <- !is.na(result$error$code) |
467 | 1x |
errors <- result[su & !duplicated(result$error$code), "error"] |
468 | 1x |
warning( |
469 | 1x |
if (sum(su) > 1) "some texts were invalid: " else "a text was invalid: ", |
470 | 1x |
paste( |
471 | 1x |
do.call(paste0, data.frame("(", errors$code, ") ", errors$message, stringsAsFactors = FALSE)), |
472 | 1x |
collapse = "; " |
473 |
), |
|
474 | 1x |
call. = FALSE |
475 |
) |
|
476 |
} |
|
477 | 83x |
result <- unpack(result[!names(result) %in% c("response_id", "language", "version", "error")]) |
478 | 83x |
if (!is.null(result) && nrow(result)) { |
479 | 83x |
if (colnames(result)[1] == "request_id") { |
480 | 82x |
colnames(result)[1] <- "text_hash" |
481 |
} else { |
|
482 | 1x |
result <- cbind(text_hash = vapply(body, "[[", "", "request_id"), result) |
483 |
} |
|
484 | 83x |
cbind(id = ids, bin = bin, result) |
485 |
} |
|
486 |
} else { |
|
487 | 223x |
unlink(temp_file) |
488 | 223x |
if (length(result$message) == 1 && substr(result$message, 1, 1) == "{") { |
489 | ! |
result <- jsonlite::fromJSON(result$message) |
490 |
} |
|
491 | 223x |
if (attempt > 0 && (length(result$code) == 1 && result$code == 1420) || ( |
492 | 223x |
length(result$message) == 1 && result$message == "invalid response format" |
493 |
)) { |
|
494 | 198x |
wait_time <- as.numeric(regmatches(result$message, regexec("[0-9]+(?:\\.[0-9]+)?", result$message))) |
495 | 198x |
Sys.sleep(if (is.na(wait_time)) 1 else wait_time / 1e3) |
496 | 198x |
request(body, bin, ids, attempt - 1) |
497 |
} else { |
|
498 | 25x |
stop(paste0(if (length(result$code)) { |
499 | 25x |
paste0( |
500 | 25x |
if (is.null(res$status_code)) 200 else res$status_code, " (", result$code, "): " |
501 |
) |
|
502 | 25x |
}, result$message), call. = FALSE) |
503 |
} |
|
504 |
} |
|
505 |
} |
|
506 | ||
507 | 45x |
process <- function(bundle) { |
508 | 146x |
opts <- getOption("stringsAsFactors") |
509 | 146x |
options("stringsAsFactors" = FALSE) |
510 | 146x |
on.exit(options("stringsAsFactors" = opts)) |
511 | 94x |
if (is.character(bundle)) bundle <- readRDS(bundle) |
512 | 146x |
text <- bundle$text |
513 | 146x |
bin <- NULL |
514 | 146x |
if (text_as_paths) { |
515 | 4x |
if (all(grepl("\\.csv", text, TRUE))) { |
516 | ! |
if (is.null(text_column)) stop("files appear to be csv, but no text_column was specified", call. = FALSE) |
517 | 3x |
text <- vapply(text, function(f) { |
518 | 51x |
tryCatch( |
519 | 51x |
paste(arrow::read_csv_arrow(f, read_options = arrow::CsvReadOptions$create( |
520 | 51x |
encoding = handle_encoding(f) |
521 | 51x |
), col_select = dplyr::all_of(text_column))[[1]], collapse = " "), |
522 | 51x |
error = function(e) stop("failed to read in file ", f, call. = FALSE) |
523 |
) |
|
524 |
}, "") |
|
525 |
} else { |
|
526 | 1x |
text <- vapply(text, function(f) { |
527 | 1x |
tryCatch( |
528 |
{ |
|
529 | 1x |
con <- file(f, encoding = handle_encoding(f)) |
530 | 1x |
on.exit(close(con)) |
531 | 1x |
paste(readLines(con, warn = FALSE, skipNul = TRUE), collapse = " ") |
532 |
}, |
|
533 | 1x |
error = function(e) stop("failed to read in file ", f, call. = FALSE) |
534 |
) |
|
535 |
}, "") |
|
536 |
} |
|
537 |
} |
|
538 | 146x |
bundle$hashes <- paste0(vapply(paste0(args_hash, text), digest::digest, "", serialize = FALSE)) |
539 | 146x |
initial <- paste0("h", substr(bundle$hashes, 1, 1)) |
540 | 146x |
set <- !is.na(text) & text != "" & text != "logical(0)" & !duplicated(bundle$hashes) |
541 | 146x |
res_cached <- res_fresh <- NULL |
542 | 146x |
if (check_cache && dir.exists(paste0(temp, "/bin=h"))) { |
543 | 41x |
db <- arrow::open_dataset(temp, partitioning = arrow::schema(bin = arrow::string()), format = cache_format) |
544 | 41x |
cached <- if (!is.null(db$schema$GetFieldByName("text_hash"))) { |
545 | 41x |
tryCatch( |
546 | 41x |
dplyr::compute(dplyr::filter(db, bin %in% unique(initial), text_hash %in% bundle$hashes)), |
547 | 41x |
error = function(e) matrix(integer(), 0) |
548 |
) |
|
549 |
} else { |
|
550 | ! |
matrix(integer(), 0) |
551 |
} |
|
552 | 41x |
if (nrow(cached)) { |
553 | 39x |
cached <- as.data.frame(cached$to_data_frame()) |
554 | ! |
if (anyDuplicated(cached$text_hash)) cached <- cached[!duplicated(cached$text_hash), ] |
555 | 39x |
rownames(cached) <- cached$text_hash |
556 | 39x |
cached_set <- which(bundle$hashes %in% cached$text_hash) |
557 | 39x |
set[cached_set] <- FALSE |
558 | 39x |
res_cached <- cbind(id = bundle[cached_set, "id"], cached[bundle[cached_set, "hashes"], ]) |
559 |
} |
|
560 |
} |
|
561 | 146x |
valid_options <- names(api_args) |
562 | 146x |
if (any(set)) { |
563 | 109x |
set <- which(set) |
564 | 109x |
res_fresh <- request(lapply( |
565 | 109x |
set, function(i) c(api_args, list(content = text[[i]], request_id = bundle[i, "hashes"])) |
566 | 109x |
), initial[set], bundle[set, "id"]) |
567 | 83x |
valid_options <- valid_options[valid_options %in% colnames(res_fresh)] |
568 | 83x |
if (length(valid_options)) { |
569 | 2x |
res_fresh <- res_fresh[, !colnames(res_fresh) %in% valid_options, drop = FALSE] |
570 |
} |
|
571 | 83x |
if (check_cache && !is.null(res_cached) && !all(colnames(res_cached) %in% colnames(res_fresh))) { |
572 | ! |
res_cached <- NULL |
573 | ! |
res_fresh <- rbind( |
574 | ! |
res_fresh, |
575 | ! |
request(lapply( |
576 | ! |
cached_set, function(i) c(api_args, list(content = text[[i]], request_id = bundle[i, "hashes"])) |
577 | ! |
), initial[cached_set], bundle[cached_set, "id"]) |
578 |
) |
|
579 |
} |
|
580 |
} |
|
581 | 120x |
res <- rbind(res_cached, res_fresh) |
582 | 2x |
if (length(valid_options)) for (n in valid_options) res[[n]] <- api_args[[n]] |
583 | 120x |
missing_ids <- !bundle$id %in% res$id |
584 | 120x |
if (any(missing_ids)) { |
585 | 1x |
varnames <- colnames(res)[colnames(res) != "id"] |
586 | 1x |
res <- rbind(res, cbind( |
587 | 1x |
id = bundle[missing_ids, "id"], |
588 | 1x |
as.data.frame(matrix(NA, sum(missing_ids), length(varnames), dimnames = list(NULL, varnames))) |
589 |
)) |
|
590 | 1x |
res$text_hash <- structure(bundle$hashes, names = bundle$id)[res$id] |
591 |
} |
|
592 | 120x |
prog(amount = nrow(res)) |
593 | 120x |
res |
594 |
} |
|
595 | ||
596 |
# make request(s) |
|
597 | 45x |
cores <- if (is.numeric(cores)) max(1, min(n_bundles, cores)) else 1 |
598 | 45x |
prog <- progressor(n_texts) |
599 | 45x |
results <- if (use_future || cores > 1) { |
600 | 8x |
call_env <- new.env(parent = globalenv()) |
601 | 8x |
environment(doprocess) <- call_env |
602 | 8x |
environment(request) <- call_env |
603 | 8x |
environment(process) <- call_env |
604 | 8x |
for (name in c( |
605 | 8x |
"doprocess", "request", "process", "text_column", "prog", "make_request", "check_cache", "full_url", |
606 | 8x |
"temp", "use_future", "cores", "bundles", "cache_format", "request_cache", "auth", |
607 | 8x |
"text_as_paths", "retry_limit", "api_args", "args_hash", "encoding", "handle_encoding" |
608 |
)) { |
|
609 | 168x |
call_env[[name]] <- get(name) |
610 |
} |
|
611 | 8x |
if (verbose) { |
612 | ! |
message( |
613 | ! |
"processing ", bundle_ref, " using ", if (use_future) "future backend" else paste(cores, "cores"), |
614 | ! |
" (", round(proc.time()[[3]] - st, 4), ")" |
615 |
) |
|
616 |
} |
|
617 | 8x |
eval(expression(doprocess(bundles, cores, use_future)), envir = call_env) |
618 |
} else { |
|
619 | 2x |
if (verbose) message("processing ", bundle_ref, " sequentially (", round(proc.time()[[3]] - st, 4), ")") |
620 | 37x |
lapply(bundles, process) |
621 |
} |
|
622 | 2x |
if (verbose) message("done retrieving; preparing final results (", round(proc.time()[[3]] - st, 4), ")") |
623 | 43x |
final_res <- do.call(rbind, results) |
624 | 43x |
if (length(api_args)) { |
625 | 3x |
su <- !names(api_args) %in% colnames(final_res) |
626 | 1x |
if (any(su)) warning("unrecognized api_args: ", paste(names(api_args)[su], collapse = ", "), call. = FALSE) |
627 |
} |
|
628 | ||
629 |
# update cache |
|
630 | 43x |
if (!is.null(temp)) { |
631 | 2x |
if (verbose) message("checking cache (", round(proc.time()[[3]] - st, 4), ")") |
632 | 30x |
initialized <- dir.exists(paste0(temp, "/bin=h")) |
633 | 30x |
if (initialized) { |
634 | 28x |
db <- arrow::open_dataset(temp, partitioning = arrow::schema(bin = arrow::string()), format = cache_format) |
635 | 28x |
if (db$num_cols != (ncol(final_res) - 1)) { |
636 | ! |
if (verbose) message("clearing existing cache since columns did not align (", round(proc.time()[[3]] - st, 4), ")") |
637 | ! |
dir.create(temp, FALSE) |
638 | ! |
initialized <- FALSE |
639 |
} |
|
640 |
} |
|
641 | 30x |
exclude <- c("id", names(api_args)) |
642 | 30x |
if (!initialized) { |
643 | 2x |
su <- !colnames(final_res) %in% exclude |
644 | 2x |
if (sum(su) > 2) { |
645 | 2x |
initial <- final_res[1, su] |
646 | 2x |
initial$text_hash <- "" |
647 | 2x |
initial$bin <- "h" |
648 | 2x |
initial[, !colnames(initial) %in% c( |
649 | 2x |
"summary.word_count", "summary.sentence_count" |
650 | 2x |
) & !vapply(initial, is.character, TRUE)] <- .1 |
651 | 2x |
initial <- rbind(initial, final_res[, colnames(initial)]) |
652 | 2x |
if (verbose) { |
653 | 1x |
message( |
654 | 1x |
"initializing cache with ", nrow(final_res), " result", |
655 | 1x |
if (nrow(final_res) > 1) "s", " (", round(proc.time()[[3]] - st, 4), ")" |
656 |
) |
|
657 |
} |
|
658 | 2x |
arrow::write_dataset(initial, temp, partitioning = "bin", format = cache_format) |
659 |
} |
|
660 |
} else { |
|
661 | 28x |
fresh <- final_res[ |
662 | 28x |
!duplicated(final_res$text_hash), !colnames(final_res) %in% names(api_args), |
663 | 28x |
drop = FALSE |
664 |
] |
|
665 | 28x |
cached <- dplyr::filter(db, bin %in% unique(fresh$bin), text_hash %in% fresh$text_hash) |
666 | 28x |
if (!any(dim(cached) == 0) || nrow(cached) != nrow(fresh)) { |
667 | 28x |
uncached_hashes <- if (nrow(cached)) { |
668 | 26x |
!fresh$text_hash %in% dplyr::collect(dplyr::select(cached, text_hash))[[1]] |
669 |
} else { |
|
670 | 2x |
rep(TRUE, nrow(fresh)) |
671 |
} |
|
672 | 28x |
if (any(uncached_hashes)) { |
673 | 5x |
if (verbose) { |
674 | 1x |
message( |
675 | 1x |
"updating cache with ", sum(uncached_hashes), " result", |
676 | 1x |
if (sum(uncached_hashes) > 1) "s", " (", round(proc.time()[[3]] - st, 4), ")" |
677 |
) |
|
678 |
} |
|
679 | 5x |
arrow::write_dataset( |
680 | 5x |
fresh[uncached_hashes, !colnames(fresh) %in% exclude], temp, |
681 | 5x |
partitioning = "bin", format = cache_format |
682 |
) |
|
683 |
} |
|
684 |
} |
|
685 |
} |
|
686 |
} |
|
687 | ||
688 |
# prepare final results |
|
689 | 2x |
if (verbose) message("preparing output (", round(proc.time()[[3]] - st, 4), ")") |
690 | 43x |
rownames(final_res) <- final_res$id |
691 | 43x |
rownames(data) <- data$id |
692 | 43x |
data$text_hash <- structure(final_res$text_hash, names = data[final_res$id, "text"])[data$text] |
693 | 43x |
final_res <- cbind( |
694 | 43x |
data[, c(if (return_text) "text", if (provided_id) "id", "text_hash"), drop = FALSE], |
695 | 43x |
final_res[ |
696 | 43x |
structure(final_res$id, names = final_res$text_hash)[data$text_hash], |
697 | 43x |
!colnames(final_res) %in% c("id", "bin", "text_hash", "custom"), |
698 | 43x |
drop = FALSE |
699 |
] |
|
700 |
) |
|
701 | 43x |
row.names(final_res) <- NULL |
702 | 43x |
if (!is.null(output)) { |
703 | ! |
if (!grepl("\\.csv", output, TRUE)) output <- paste0(output, ".csv") |
704 | 1x |
if (compress && !grepl(".xz", output, fixed = TRUE)) output <- paste0(output, ".xz") |
705 | 1x |
if (grepl(".xz", output, fixed = TRUE)) compress <- TRUE |
706 | 1x |
if (verbose) message("writing results to file: ", output, " (", round(proc.time()[[3]] - st, 4), ")") |
707 | 3x |
dir.create(dirname(output), FALSE, TRUE) |
708 | 2x |
if (overwrite) unlink(output) |
709 | 1x |
if (compress) output <- xzfile(output) |
710 | 3x |
arrow::write_csv_arrow(final_res, file = output) |
711 |
} |
|
712 | ||
713 | 43x |
if (is.character(frameworks) && frameworks[1] != "all") { |
714 | ! |
if (verbose) message("selecting frameworks (", round(proc.time()[[3]] - st, 4), ")") |
715 | 7x |
vars <- colnames(final_res) |
716 | 7x |
sel <- grepl(paste0("^(?:", paste(tolower(frameworks), collapse = "|"), ")"), vars) |
717 | 7x |
if (any(sel)) { |
718 | 4x |
if (missing(framework_prefix) && (length(frameworks) == 1 && frameworks != "all")) framework_prefix <- FALSE |
719 | 6x |
sel <- unique(c("text", "id", "text_hash", names(api_args), vars[sel])) |
720 | 6x |
sel <- sel[sel %in% vars] |
721 | 6x |
final_res <- final_res[, sel] |
722 |
} else { |
|
723 | 1x |
warning("frameworks did not match any columns -- returning all", call. = FALSE) |
724 |
} |
|
725 |
} |
|
726 | 43x |
if (as_list) { |
727 | 1x |
if (missing(framework_prefix)) framework_prefix <- FALSE |
728 | 1x |
inall <- c("text", "id", "text_hash", names(api_args)) |
729 | 1x |
cols <- colnames(final_res) |
730 | 1x |
inall <- inall[inall %in% cols] |
731 | 1x |
pre <- sub("\\..*$", "", cols) |
732 | 1x |
pre <- unique(pre[!pre %in% inall]) |
733 | 1x |
final_res <- lapply(structure(pre, names = pre), function(f) { |
734 | 9x |
res <- final_res[, c(inall, grep(paste0("^", f), cols, value = TRUE))] |
735 | 9x |
if (!framework_prefix) colnames(res) <- sub("^.+\\.", "", colnames(res)) |
736 | 9x |
res |
737 |
}) |
|
738 | 5x |
} else if (!framework_prefix) colnames(final_res) <- sub("^.+\\.", "", colnames(final_res)) |
739 | 2x |
if (verbose) message("done (", round(proc.time()[[3]] - st, 4), ")") |
740 | 43x |
invisible(final_res) |
741 |
} |
1 |
#' @rdname receptiviti |
|
2 |
#' @export |
|
3 | ||
4 |
receptiviti_status <- function(url = Sys.getenv("RECEPTIVITI_URL"), key = Sys.getenv("RECEPTIVITI_KEY"), |
|
5 |
secret = Sys.getenv("RECEPTIVITI_SECRET"), verbose = TRUE, include_headers = FALSE) { |
|
6 | 2x |
if (key == "") stop("specify your key, or set it to the RECEPTIVITI_KEY environment variable", call. = FALSE) |
7 | 2x |
if (secret == "") stop("specify your secret, or set it to the RECEPTIVITI_SECRET environment variable", call. = FALSE) |
8 | 53x |
handler <- new_handle(httpauth = 1, userpwd = paste0(key, ":", secret)) |
9 | 53x |
url <- paste0( |
10 | 53x |
if (!grepl("http", tolower(url), fixed = TRUE)) "https://", |
11 | 53x |
sub("/+[Vv]\\d+(?:/.*)?$|/+$", "", url), "/v1/ping" |
12 |
) |
|
13 | 1x |
if (!grepl("^https?://[^.]+[.:][^.]", url, TRUE)) stop("url does not appear to be valid: ", url) |
14 | 52x |
ping <- tryCatch(curl_fetch_memory(url, handler), error = function(e) NULL) |
15 | 52x |
if (is.null(ping)) { |
16 | 1x |
if (verbose) message("Status: ERROR\nMessage: URL is unreachable") |
17 | 2x |
invisible(return()) |
18 |
} |
|
19 | 50x |
ping$content <- list(message = rawToChar(ping$content)) |
20 | 49x |
if (substr(ping$content, 1, 1) == "{") ping$content <- fromJSON(ping$content$message) |
21 | 50x |
ok <- ping$status_code == 200 && !length(ping$content$code) |
22 | 50x |
ping$status_message <- if (ok) { |
23 | 46x |
ping$content$pong |
24 |
} else { |
|
25 | 4x |
paste0( |
26 | 4x |
if (length(ping$content$code)) paste0(ping$status_code, " (", ping$content$code, "): "), |
27 | 4x |
if (nchar(ping$content$message) > 500 || grepl("<", ping$content$message, fixed = TRUE)) { |
28 | 1x |
ping$status_code |
29 |
} else { |
|
30 | 3x |
ping$content$message |
31 |
} |
|
32 |
) |
|
33 |
} |
|
34 | 50x |
if (verbose) { |
35 | 4x |
message("Status: ", if (ok) "OK" else "ERROR", "\nMessage: ", ping$status_message) |
36 | 4x |
if (include_headers) { |
37 | 1x |
ping$headers <- strsplit(rawToChar(ping$headers), "[\r\n]+", perl = TRUE)[[1]] |
38 | 1x |
json <- regexec("\\{.+\\}", ping$headers) |
39 | 1x |
for (i in seq_along(json)) { |
40 | 10x |
if (json[[i]] != -1) { |
41 | 2x |
regmatches(ping$headers[[i]], json[[i]]) <- paste(" ", strsplit(toJSON( |
42 | 2x |
fromJSON(regmatches(ping$headers[[i]], json[[i]])), |
43 | 2x |
auto_unbox = TRUE, pretty = TRUE |
44 | 2x |
), "\n")[[1]], collapse = "\n") |
45 |
} |
|
46 |
} |
|
47 | 1x |
message(paste0("\n", paste(" ", ping$headers, collapse = "\n"))) |
48 |
} |
|
49 |
} |
|
50 | 50x |
invisible(ping) |
51 |
} |