使用 R 在 http 上进行 grep

发布于04月27日

我无法下载文件，我如何更新下面的函数，如果我打印一些值FILE_NAMES，它们是空的.会出什么问题呢？

# Set the download URL
url <- "https://ftp.ncbi.nlm.nih.gov/pub/pmc/manuscript/xml/"

# Create a vector of file numbers to download
file_nums <- sprintf("%02d", 1:3) 

# Create a pattern to match the beginning of the file names
pattern <- paste0("^author_manuscript_xml.PMC0", file_nums)

# Get a vector of all file names from the URL and filter based on the pattern
file_names <- grep(pattern, list.files(url), value = TRUE)

# Loop through the file names and download each file
for (file_name in file_names) {
  file_url <- paste0(url, file_name)
  download.file(file_url, file_name)
}

推荐答案

下面的代码解决了这个问题.诀窍是遍历pattern，而不是文件名.

suppressPackageStartupMessages({
  library(rvest)
  library(dplyr)
})

# create a temp dir where to download the files
tmpdir <- tempdir()
tmpdir <- chartr("\\", "/", tmpdir)

# Create a vector of file numbers to download
file_nums <- sprintf("%02d", 1:3) 

# Create a pattern to match the beginning of the file names
pattern <- paste0("^author_manuscript_xml.PMC0", file_nums)

# this is the page link
link <- "https://ftp.ncbi.nlm.nih.gov/pub/pmc/manuscript/xml/"

# read the page only once
page <- read_html(link)

# extract all links
file_names <- page %>%
  html_elements("a") %>%
  html_attr("href")

# this is probably necessary, some files are 1G+
old_opts <- options(timeout = max(600, getOption("timeout")))

# download the files
res <- lapply(pattern, \(p) {
  i <- grep(p, file_names)
  files <- file_names[i]
  urls <- paste0(link, files)
  destfls <- paste(tmpdir, files, sep = "/")
  mapply(\(u, d) {
    tryCatch(download.file(u, d), error = function(e) e)
  }, urls, destfls)
})

# restore the default timeout
options(old_opts)