使用rvest将数据从web刮到dataframe

Question

library(rvest)
library(tidyverse)

# Start with permit number list 

permitlist <- list("DC0000175", "VA0076384", "VA0021318")

# For each permit number, create a URL list that leads to the EPA site associated with that permit

urls <- list()
for(i in 1:length(permitlist)) {
  url <- paste0("https://iaspub.epa.gov/enviro/fii_query_dtl.disp_program_facility?pgm_sys_id_in=", permitlist[i], "&pgm_sys_acrnm_in=NPDES")
urls[i] <- url
} 

# Open each URL and scrape SIC code data

complete<-data.frame()
tbl <- list()

for (j in seq_along(urls)) {
  data <- tbl[[j]] <- urls[[j]] %>%  
    read_html() %>% 
    html_nodes("tr") %>%
    html_text()
  data <- unlist(strsplit(trimws(grep("^NPDES\n", tbl[[j]], value=TRUE)[1]), "\n", perl=TRUE))
  complete <-rbind(complete,data, stringsAsFactors = FALSE)
  }

1    NPDES    3273                READY-MIXED CONCRETE
2    NPDES    3714                MOTOR VEHICLE PARTS AND ACCESSORIES
3    NPDES    4952                SEWERAGE SYSTEMS

# (Row number, type of permit (all will be NPDES), NPDES SIC code, and description.)

理想情况下，我的最终数据帧将如下所示：

1    NPDES    3273    DC0000175  READY-MIXED CONCRETE
2    NPDES    1611    DC0000175  HIGHWAY AND STREET CONSTRUCTION, EXCEPT ELEVATED HIGHWAYS
3    NPDES    3714    VA0076384  MOTOR VEHICLE PARTS AND ACCESSORIES
4    NPDES    4952    VA0021318  SEWERAGE SYSTEMS

我已经挣扎了一段时间，现在让这个工作，并将感谢任何帮助。

Answer 1

进行两项更改：

• 使用c()将permitlist从列表更改为向量-这也使它运行得更快！
• 向数据df添加新列，并分配相应的许可值

library(rvest)
library(tidyverse)

# Start with permit number list 

# vector is more efficient and more easily iterable
permitlist <- c("DC0000175", "VA0076384", "VA0021318")

# For each permit number, create a URL list that leads to the EPA site associated with that permit

urls <- list()
for(i in 1:length(permitlist)) {
  url <- paste0("https://iaspub.epa.gov/enviro/fii_query_dtl.disp_program_facility?pgm_sys_id_in=", permitlist[i], "&pgm_sys_acrnm_in=NPDES")
  urls[i] <- url
} 

# Open each URL and scrape SIC code data

complete<-data.frame()
tbl <- list()

for (j in seq_along(urls)) {
  tbl[[j]] <- urls[[j]] %>%  
    read_html() %>% 
    html_nodes("tr") %>%
    html_text()

  data <- data.frame(A = NA, B = NA, c = NA, permit = NA)
  a <- grep("^NPDES\n", tbl[[j]], value=TRUE)

  for(k in seq_along(a)){
  dt <- unlist(strsplit(trimws(a[k]), "\n", perl=TRUE))
  dt <- c(dt, permitlist[j])
  data[k,] <- dt
  }
  complete <- bind_rows(complete, data)

}

complete

通过在填充表之前指定列名（a,B,C,permit）,编辑使输出数据帧更整齐。