Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions .Rbuildignore
Original file line number Diff line number Diff line change
Expand Up @@ -13,3 +13,7 @@
^\.vscode$
^\.idea$
^\.Rhistory$
^.*\.gitkeep$
README.html
sticker_queimadasR.png

6 changes: 2 additions & 4 deletions DESCRIPTION
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
Package: queimadasR
Type: Package
Title: Download and Analysis of Wildfire Data from INPE
Version: 0.2.0
Version: 0.2.1
Authors@R: c(
person("Wagner", "Tassinari",
email = "wagner.tassinari@ufrrj.br",
Expand Down Expand Up @@ -38,7 +38,6 @@ License: MIT + file LICENSE
Encoding: UTF-8
LazyData: false
Roxygen: list(markdown = TRUE)
RoxygenNote: 7.3.1
Depends:
R (>= 3.5.0)
Imports:
Expand All @@ -48,7 +47,6 @@ Suggests:
testthat (>= 3.0.0),
knitr,
rmarkdown
VignetteBuilder: knitr
URL: https://github.com/wtassinari/queimadasR
BugReports: https://github.com/wtassinari/queimadasR/issues

Config/roxygen2/version: 8.0.0
2 changes: 2 additions & 0 deletions LICENSE
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
YEAR: 2026
COPYRIGHT HOLDER: queimadasR authors
21 changes: 21 additions & 0 deletions LICENSE.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
# MIT License

Copyright (c) 2026 queimadasR authors

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
223 changes: 120 additions & 103 deletions R/download_fire_spots.R
Original file line number Diff line number Diff line change
Expand Up @@ -48,8 +48,8 @@
#' north_states <- c("ACRE", "AMAPA", "AMAZONAS", "PARA", "RONDONIA", "RORAIMA", "TOCANTINS")
#' north_data <- download_fire_spots(
#' start_date_str = "01/08/2023",
#' end_date_str = "30/09/2023",
#' target_states = north_states,
#' end_date_str = "30/09/2023",
#' target_states = north_states,
#' deduplicate_final = TRUE
#' )
#'
Expand All @@ -66,14 +66,14 @@
#' @export
download_fire_spots <- function(
start_date_str, end_date_str,
region = "Brasil",
target_states = NULL,
region = "Brasil",
target_states = NULL,
target_satellites = NULL,
timeout = 300,
sleep_sec = 1,
timeout = 300,
sleep_sec = 1,
show_satellites_when_empty = TRUE,
deduplicate_final = TRUE,
dedup_keys = c("latitude", "longitude", "data_pas", "municipio")
dedup_keys = c("latitude", "longitude", "data_pas", "municipio")
) {
options(timeout = timeout)

Expand Down Expand Up @@ -101,7 +101,8 @@ download_fire_spots <- function(
# Parse datetime strings to POSIXct in GMT, trying multiple formats
parse_datetime_gmt <- function(x) {
as.POSIXct(
x, tz = "GMT",
x,
tz = "GMT",
tryFormats = c(
"%Y-%m-%d %H:%M:%S",
"%Y-%m-%d %H:%M",
Expand All @@ -124,16 +125,18 @@ download_fire_spots <- function(
# Also try any column whose name starts with "data"
candidates <- unique(c(candidates, grep("^data", names(df), ignore.case = TRUE, value = TRUE)))
candidates <- intersect(candidates, names(df))
if (length(candidates) == 0) return(NULL)
if (length(candidates) == 0) {
return(NULL)
}

best <- NULL
best_ok <- -1L
best <- NULL
best_ok <- -1L
for (nm in candidates) {
x <- parse_datetime_gmt(df[[nm]])
x <- parse_datetime_gmt(df[[nm]])
ok <- sum(!is.na(x))
if (ok > best_ok) {
best_ok <- ok
best <- nm
best <- nm
}
}
best
Expand All @@ -147,12 +150,14 @@ download_fire_spots <- function(
"GOES_13" = c("GOES-13", "GOES_13", "GOES-13D", "GOES_13D"),
"NPP_375" = c("NPP-375", "NPP_375", "NPP-375D", "NPP_375D", "SUOMI_NPP_375", "SUOMI_NPP_375D")
)
sat_alias <- lapply(sat_alias, norm_sat)
sat_alias <- lapply(sat_alias, norm_sat)
names(sat_alias) <- norm_sat(names(sat_alias))

# Expand satellite names to include all known aliases
expand_sats <- function(sats) {
if (is.null(sats)) return(NULL)
if (is.null(sats)) {
return(NULL)
}
s_norm <- norm_sat(sats)
unique(unlist(lapply(s_norm, function(s) {
if (s %in% names(sat_alias)) sat_alias[[s]] else s
Expand Down Expand Up @@ -181,16 +186,16 @@ download_fire_spots <- function(
# Parse input dates
# ---------------------------
start_date <- as.Date(start_date_str, format = "%d/%m/%Y")
end_date <- as.Date(end_date_str, format = "%d/%m/%Y")
end_date <- as.Date(end_date_str, format = "%d/%m/%Y")
if (is.na(start_date) || is.na(end_date)) stop("Invalid date format. Use 'DD/MM/YYYY'.")
if (end_date < start_date) stop("end_date must be >= start_date.")
if (end_date < start_date) stop("end_date must be >= start_date.")

start_year <- as.integer(format(start_date, "%Y"))
end_year <- as.integer(format(end_date, "%Y"))
years <- seq(start_year, end_year)
end_year <- as.integer(format(end_date, "%Y"))
years <- seq(start_year, end_year)

dt_start <- as.POSIXct(paste0(format(start_date, "%Y-%m-%d"), " 00:00:00"), tz = "GMT")
dt_end <- as.POSIXct(paste0(format(end_date, "%Y-%m-%d"), " 23:59:59"), tz = "GMT")
dt_end <- as.POSIXct(paste0(format(end_date, "%Y-%m-%d"), " 23:59:59"), tz = "GMT")

use_all_satellites <- is.null(target_satellites) ||
(length(target_satellites) == 1 && toupper(target_satellites) %in% c("TODOS", "ALL"))
Expand All @@ -199,7 +204,7 @@ download_fire_spots <- function(
cat("Filter Period:", start_date_str, "to", end_date_str, "\n")
cat("Years to Download:", paste(years, collapse = ", "), "\n")
cat("Region:", region, "\n")
if (!is.null(target_states)) cat("Filtering by States:", paste(target_states, collapse = ", "), "\n")
if (!is.null(target_states)) cat("Filtering by States:", paste(target_states, collapse = ", "), "\n")
if (use_all_satellites) {
cat("Filtering by Satellites: ALL\n")
} else {
Expand All @@ -210,112 +215,120 @@ download_fire_spots <- function(
# ---------------------------
# Loop over years
# ---------------------------
data_list <- vector("list", length(years))
success_count <- 0L
error_count <- 0L
data_list <- vector("list", length(years))
success_count <- 0L
error_count <- 0L

for (i in seq_along(years)) {
current_year <- years[i]
file_name <- paste0("focos_br_todos-sats_", current_year, ".zip")
file_name <- paste0("focos_br_todos-sats_", current_year, ".zip")
url <- paste0(
"https://dataserver-coids.inpe.br/queimadas/queimadas/focos/csv/anual/Brasil_todos_sats/",
file_name
)

cat(sprintf("[%d/%d] %s ... ", i, length(years), current_year))

temp_dir <- tempdir()
temp_dir <- tempdir()
local_file <- file.path(temp_dir, file_name)

res <- tryCatch({
# Download file
status <- download.file(url, local_file, mode = "wb", quiet = TRUE, method = "auto")
if (status != 0) stop("Download failed (download.file returned != 0).")

# Find CSV inside the ZIP
zip_list <- unzip(local_file, list = TRUE)
csvs <- zip_list$Name[grepl("\\.csv$", zip_list$Name, ignore.case = TRUE)]
if (length(csvs) == 0) stop("ZIP does not contain a CSV file.")
csv_in_zip <- csvs[1]

# Read CSV directly from the ZIP
year_data <- read.csv(unz(local_file, csv_in_zip), stringsAsFactors = FALSE)

# Detect datetime column and standardize as data_pas
dt_col <- find_datetime_column(year_data)
if (is.null(dt_col)) {
stop(paste0("Could not find a datetime column. Columns found: ", paste(names(year_data), collapse = ", ")))
}
res <- tryCatch(
{
# Download file
status <- utils::download.file(url, local_file,
mode = "wb",
quiet = TRUE, method = "auto"
)
if (status != 0) stop("Download failed (download.file returned != 0).")

# Find CSV inside the ZIP
zip_list <- utils::unzip(local_file, list = TRUE)
csvs <-
zip_list$Name[grepl("\\.csv$", zip_list$Name, ignore.case = TRUE)]
if (length(csvs) == 0) stop("ZIP does not contain a CSV file.")
csv_in_zip <- csvs[1]

# Read CSV directly from the ZIP
year_data <- utils::read.csv(unz(local_file, csv_in_zip),
stringsAsFactors = FALSE
)

# Detect datetime column and standardize as data_pas
dt_col <- find_datetime_column(year_data)
if (is.null(dt_col)) {
stop(paste0("Could not find a datetime column. Columns found: ", paste(names(year_data), collapse = ", ")))
}

dt_parsed <- parse_datetime_gmt(year_data[[dt_col]])
if (all(is.na(dt_parsed))) stop(paste0("Failed to convert datetime column: ", dt_col))

# Ensure data_pas column is always present as POSIXct
year_data$data_pas <- dt_parsed

# Filter by period
filtered_data <- year_data[year_data$data_pas >= dt_start & year_data$data_pas <= dt_end, , drop = FALSE]

# Filter by state
if (!is.null(target_states) && region == "Brasil") {
uf_candidates <- c("estado", "uf", "sigla_uf", "estado_sigla")
uf_col <- intersect(uf_candidates, names(filtered_data))
if (length(uf_col) > 0) {
uf_col <- uf_col[1]
target_norm <- norm_txt(target_states)
filtered_data <- filtered_data[norm_txt(filtered_data[[uf_col]]) %in% target_norm, , drop = FALSE]
} else {
warning("State filter was requested, but no 'estado/uf' column was found.")
dt_parsed <- parse_datetime_gmt(year_data[[dt_col]])
if (all(is.na(dt_parsed))) stop(paste0("Failed to convert datetime column: ", dt_col))

# Ensure data_pas column is always present as POSIXct
year_data$data_pas <- dt_parsed

# Filter by period
filtered_data <- year_data[year_data$data_pas >= dt_start & year_data$data_pas <= dt_end, , drop = FALSE]

# Filter by state
if (!is.null(target_states) && region == "Brasil") {
uf_candidates <- c("estado", "uf", "sigla_uf", "estado_sigla")
uf_col <- intersect(uf_candidates, names(filtered_data))
if (length(uf_col) > 0) {
uf_col <- uf_col[1]
target_norm <- norm_txt(target_states)
filtered_data <- filtered_data[norm_txt(filtered_data[[uf_col]]) %in% target_norm, , drop = FALSE]
} else {
warning("State filter was requested, but no 'estado/uf' column was found.")
}
}
}

# Filter by satellite (with aliases), only if NOT using all satellites
if (!use_all_satellites && region == "Brasil") {
sat_candidates <- c("satelite", "satellite", "sensor")
sat_col <- intersect(sat_candidates, names(filtered_data))
if (length(sat_col) > 0) {
sat_col <- sat_col[1]
# Filter by satellite (with aliases), only if NOT using all satellites
if (!use_all_satellites && region == "Brasil") {
sat_candidates <- c("satelite", "satellite", "sensor")
sat_col <- intersect(sat_candidates, names(filtered_data))
if (length(sat_col) > 0) {
sat_col <- sat_col[1]

expanded_targets <- expand_sats(target_satellites) # normalized
sat_vals_norm <- norm_sat(filtered_data[[sat_col]])
expanded_targets <- expand_sats(target_satellites) # normalized
sat_vals_norm <- norm_sat(filtered_data[[sat_col]])

filtered_data2 <- filtered_data[sat_vals_norm %in% expanded_targets, , drop = FALSE]
filtered_data2 <- filtered_data[sat_vals_norm %in% expanded_targets, , drop = FALSE]

if (nrow(filtered_data2) == 0L && show_satellites_when_empty) {
top_sats <- sort(table(norm_sat(filtered_data[[sat_col]])), decreasing = TRUE)
top_sats <- head(top_sats, 10)
cat("\n Warning: satellite filter returned zero records for this year. Available satellites (top 10):\n")
print(top_sats)
}
if (nrow(filtered_data2) == 0L && show_satellites_when_empty) {
top_sats <- sort(table(norm_sat(filtered_data[[sat_col]])), decreasing = TRUE)
top_sats <- utils::head(top_sats, 10)
cat("\n Warning: satellite filter returned zero records for this year. Available satellites (top 10):\n")
print(top_sats)
}

filtered_data <- filtered_data2
} else {
warning("Satellite filter was requested, but no 'satelite/satellite/sensor' column was found.")
filtered_data <- filtered_data2
} else {
warning("Satellite filter was requested, but no 'satelite/satellite/sensor' column was found.")
}
}
}

# Tag reference year
filtered_data$ref_year <- rep.int(current_year, nrow(filtered_data))
# Tag reference year
filtered_data$ref_year <- rep.int(current_year, nrow(filtered_data))

# Clean up ZIP file
if (file.exists(local_file)) file.remove(local_file)
# Clean up ZIP file
if (file.exists(local_file)) file.remove(local_file)

list(ok = TRUE, data = filtered_data)

}, error = function(e) {
if (file.exists(local_file)) file.remove(local_file)
list(ok = FALSE, err = conditionMessage(e))
})
list(ok = TRUE, data = filtered_data)
},
error = function(e) {
if (file.exists(local_file)) file.remove(local_file)
list(ok = FALSE, err = conditionMessage(e))
}
)

if (isTRUE(res$ok)) {
data_list[[i]] <- res$data
success_count <- success_count + 1L
cat(" (", format(nrow(res$data), big.mark = ","), "fire spots)\n")
data_list[[i]] <- res$data
success_count <- success_count + 1L
cat("\\u2713 (", format(nrow(res$data), big.mark = ","), "fire spots)\n")
Sys.sleep(sleep_sec)
} else {
error_count <- error_count + 1L
data_list[[i]] <- NULL
cat(" Error\n")
error_count <- error_count + 1L
data_list[[i]] <- NULL
cat("\\u2717 Error\n")
cat(" Details:", res$err, "\n")
}
}
Expand All @@ -332,13 +345,17 @@ download_fire_spots <- function(
cat("\nCombining data...\n")
complete_data <- do.call(rbind.data.frame, data_list[!vapply(data_list, is.null, logical(1))])
cat("Total (before deduplication):", format(nrow(complete_data), big.mark = ","), "\n")
cat("Memory size:", format(object.size(complete_data), units = "MB"), "\n")
cat(
"Memory size:",
format(utils::object.size(complete_data), units = "MB"),
"\n"
)

if (isTRUE(deduplicate_final)) {
cat("\nDeduplicating by:", paste(dedup_keys, collapse = ", "), "...\n")
before <- nrow(complete_data)
before <- nrow(complete_data)
complete_data <- dedup_fire_spots(complete_data, dedup_keys)
after <- nrow(complete_data)
after <- nrow(complete_data)
cat("Removed:", format(before - after, big.mark = ","), "duplicates.\n")
cat("Total (after deduplication):", format(after, big.mark = ","), "\n")
}
Expand Down
Loading