Chapter 15 On your own
Congratulations! You made it to the end of the planned content in one peice.
You now have the opportunity to apply some of the skills you have learnt to a completely new database. Cut and paste your code from previous chapters to check the data, write the analysis date frames and explore the dataset.
I will help you read in the data - and correct a mistake I made. Then you are on your own!
15.1 Working with a new dataset
15.1.1 Read in the data and packages
# Load your data
pro <- read.csv("data/raw_data/your_data/projects.csv", header=T)
# I missed of the project ID from all the dataframes so we need to update those
pro$project_id <- pro$project_name
img <- read.csv("data/raw_data/your_data/images.csv", header=T)
img$project_id <- pro$project_name
dep <- read.csv("data/raw_data/your_data/deployments.csv", header=T)
dep$project_id <- pro$project_name
cam <- read.csv("data/raw_data/your_data/cameras.csv", header=T)
cam$project_id <- pro$project_name
# A list of the required packages
list.of.packages <- c("activity", "corrplot", "cowplot", "dplyr", "elevatr", "gfcanalysis", "ggplot2", "gridExtra", "iNEXT", "kableExtra", "Hmsc", "leaflet", "lme4", "lubridate", "magrittr", "MCMCvis", "MODISTools", "osmdata", "pals", "plotly", "remotes", "rmarkdown", "sf", "spOccupancy", "stars", "stringr", "terra", "tibble", "tidyr", "unmarked", "viridis", "jtools", "vegan", "MuMIn")
# A check to see which ones you have and which are missing
new.packages <- list.of.packages[!(list.of.packages %in% installed.packages()[,"Package"])]
# Code which tells R to install the missing packages
if(length(new.packages)) install.packages(new.packages)
lapply(list.of.packages, require, character.only = TRUE)
# Add the 'placename' column to the image data.
# The following code matches every deployment ID in images, with deployment_id's and placenames in the deployment data.
img <- left_join(img, dep[,c("deployment_id", "placename")])15.1.3 Plot and correct spatial co-ordinates
# Pick the category you want to color
category <- "feature_type"
# We first convert this category to a factor with discrete levels
dep[,category] <- factor(dep[,category])
# then use the turbo() function to assign each level a color
col.cat <- turbo(length(levels(dep[,category])))
# then we apply it to the dataframe
dep$colours <- col.cat[dep[,category]]
m <- leaflet() %>%
addProviderTiles(providers$Esri.WorldImagery, group="Satellite") %>%
addTiles(group="Base") %>% # Include a basemap option too
addCircleMarkers(lng=dep$longitude, lat=dep$latitude,
# Co lour the markers depending on the 'feature type'
color=dep$colours,
# Add a popup of the placename and feature_type together
popup=paste(dep$placename, dep[,category])) %>%
# Add a legend explaining what is going on
addLegend("bottomleft", colors = col.cat, labels = levels(dep[,category]),
title = category,
labFormat = labelFormat(prefix = "$"),
opacity = 1) %>%
# add a layer control box to toggle between the layers
addLayersControl(
baseGroups = c("Satellite", "Base"))
mQUESTION What is the survey design? Hint: zoom in.
15.1.4 Check camera spacing
# Distance between traps
camera_locs <- dep %>%
dplyr::select(placename, latitude, longitude) %>%
unique() %>% # remove duplicated rows (rows where the placename and coordinates match)
st_as_sf(coords = c("longitude", "latitude"), crs = "+proj=longlat") # Convert to `sf` format
# distance matrix for all cameras
camera_dist <- st_distance(camera_locs) %>%
as.dist() %>%
usedist::dist_setNames(as.character(camera_locs$placename)) %>%
as.matrix()
camera_dist_list <- list()
i <- 1
for(i in 1:nrow(camera_dist))
{
camera_dist_list[[i]]<- data.frame(placename=row.names(camera_dist)[i], dist= min(camera_dist[i,][camera_dist[i,]!=0]))
}
camera_dist_list <- bind_rows(camera_dist_list)
summary(camera_dist_list$dist)Do you have anything to be worried about here?
15.1.7 5c. Check deployments occur where you expect
# Call the plot
p <- plot_ly()
# We want a separate row for each 'placename' - so lets turn it into a factor
dep$placename <- as.factor(dep$placename)
# loop through each place name
for(i in seq_along(levels(dep$placename)))
{
#Subset the data to just that placename
tmp <- dep[dep$placename==levels(dep$placename)[i],]
# Order by date
tmp <- tmp[order(tmp$start_date),]
# Loop through each deployment at that placename
for(j in 1:nrow(tmp))
{
# Add a line to 'p'
p <- add_trace(p,
#Use the start and end date as x coordinates
x = c(tmp$start_date[j], tmp$end_date[j]),
#Use the counter for the y coordinates
y = c(i,i),
# State the type of chart
type="scatter",
# make a line that also has points
mode = "lines+markers",
# Add the deployment ID as hover text
hovertext=tmp$deployment_id[j],
# Color it all black
color=I("black"),
# Suppress the legend
showlegend = FALSE)
}
}
# Add a categorical y axis
p <- p %>% layout(yaxis = list(
ticktext = as.list(levels(dep$placename)),
tickvals = as.list(1:length(levels(dep$placename))),
tickmode = "array"))
pQUESTION What do you think about the camera deployments?
15.1.8 Check images occur within deployments
#Camera activity plot
# Make a separate plot for each 30 stations
# Order by placename to make the graphs easier to follow
dep <- dep[order(dep$placename),]
# To do this make a plot dataframe
tmp <- data.frame("deployment_id"=unique(dep$deployment_id), "plot_group"=ceiling(1:length(unique(dep$deployment_id))/30))
dep_tmp <- left_join(dep,tmp, by="deployment_id")
# Make a colour palette for blanks vs. real detections
cols <- c("red", "blue")
for(i in 1:max(dep_tmp$plot_group))
{
# Call the plot
#Subset the data to just that placename
tmp <- dep_tmp[dep_tmp$plot_group==i,]
# Order by placename
tmp <- tmp[order(tmp$placename),]
p <- plot_ly()
# Loop through each deployment at that placename
for(j in 1:nrow(tmp))
{
#Subset the image data
tmp_img <- img[img$deployment_id==tmp$deployment_id[j],]
if(nrow(tmp_img)>0)
{
p <- add_trace(p,
#Use the start and end date as x coordinates
x = c(ymd_hms(tmp_img$timestamp)),
#Use the counter for the y coordinates
y = rep(j, nrow(tmp_img)),
# State the type of chart
type="scatter",
# make a line that also has points
mode = "markers",
# Add the deployment ID as hover text
hovertext=paste(tmp_img$genus,tmp_img$species),
# Color it all black
marker = list(color = cols[tmp_img$is_blank+1]),
# Suppress the legend
showlegend = FALSE)
}
# Add a line to 'p'
p <- add_trace(p,
#Use the start and end date as x coordinates
x = c(tmp$start_date[j], tmp$end_date[j]+days(1)),
#Use the counter for the y coordinates
y = c(j,j),
# State the type of chart
type="scatter",
# make a line that also has points
mode = "lines",
# Add the deployment ID as hover text
hovertext=tmp$deployment_id[j],
# Color it all black
color=I("black"),
# Suppress the legend
showlegend = FALSE)
}
# Add custom y axis labels
p <- p %>% layout(yaxis = list(
ticktext = as.list(tmp$deployment_id),
tickvals = as.list(1:nrow(tmp)),
tickmode = "array"))
print(p)
} Note - there are two panels to check!
QUESTION Do you need to make any edits? Note - there are three plot windows to check.
The camera failed for LAC061, but the deployment dates already reflect that. LAC064 also looks somewhat strange, that would be worth checking what happened. Lets leave it in for now.
We could maybe remove OFF067? That would be a good idea.
15.1.9 Check species taxonomy
# Create a species list
# add an sp column to the img dataframe - remember the genus and species columns are not pasted together yet
img$sp <- paste(img$genus, img$species, sep=".")
### Species lists
# First define vector of the headings you want to see (we will use this trick a lot later on)
taxonomy_headings <- c("class", "order", "family", "genus", "species", "common_name")
# Subset the image data to just those columns
tmp<- img[,colnames(img)%in% taxonomy_headings]
# Remove duplicates
tmp <- tmp[duplicated(tmp)==F,]
# Create an ordered species list
sp_list <- tmp[order(tmp$class, tmp$order, tmp$family, tmp$genus, tmp$species),]
sp_list$sp <- paste(sp_list$genus, sp_list$species, sep=".")
write.csv(sp_list, paste0("data/raw_data/",pro$project_id[1],"_raw_species_list.csv"))QUESTION Are there any classifications you will remove when we create our analysis dataframes?
15.1.10 Check species activity
# Diel activity check
# First lets convert our timestamp to decimal hours
img$hours <- hour(img$timestamp) + minute(img$timestamp)/60 + second(img$timestamp)/(60*60)
# Count all of the captures
tmp <- img %>% group_by(common_name) %>% summarize(count=n())
yform <- list(categoryorder = "array",
categoryarray = tmp$common_name)
fig <- plot_ly(x = img$hours, y = img$common_name,type="scatter",
height=1000, text=img$deployment_id, hoverinfo='text',
mode = 'markers',
marker = list(size = 5,
color = 'rgba(50, 100, 255, .2)',
line = list(color = 'rgba(0, 0, 0, 0)',
width = 0))) %>%
layout(yaxis = yform)
fig
# Remove the column
img$hours <- NULLQUESTION Are there any images you would like to check?
Clue - Paca’s are nocturnal but they have some diurnal detections here. Check which station they come from.
15.1.11 Filter to target species
# Create your processed dataframe folder
dir.create("data/processed_data")
# Filter your species
# Remove observations without animals detected, where we don't know the species, and non-mammals
img_sub <- img %>% filter(is_blank==0, # Remove the blanks
is.na(img$species)==FALSE, # Remove classifications which don't have species
class=="Mammalia", # Subset to mammals
species!="sapiens",
species!="") # remove instances without species labels
table(img_sub$common_name)QUESTION Are there any other classifications you would like to remove?
15.1.12 Make effort look_up
##########################################
# Create your daily lookup
# Remove any deployments without end dates
tmp <- dep[is.na(dep$end_date)==F,]
# Create an empty list to store our days
daily_lookup <- list()
# Loop through the deployment dataframe and create a row for every day the camera is active
for(i in 1:nrow(tmp))
{
if(ymd(tmp$start_date[i])!=ymd(tmp$end_date[i]))
{
daily_lookup[[i]] <- data.frame("date"=seq(ymd(tmp$start_date[i]), ymd(tmp$end_date[i]), by="days"), "placename"=tmp$placename[i])
}
}
# Merge the lists into a dataframe
row_lookup <- bind_rows(daily_lookup)
# Remove duplicates - when start and end days are the same for successive deployments
row_lookup <- row_lookup[duplicated(row_lookup)==F,]15.1.13 Create independent data
###################################
# Create your independent detections
independent <- 30
# Check for a `group_size` variable?
table(img_sub$group_size)
# Check for a 'number_of_objects' variable
table(img_sub$number_of_objects)QUESTION WHich variable will you use for your animal_count?
# Create your independent data
img_tmp <- img_sub %>%
arrange(deployment_id) %>% # Order by deployment_id
group_by(deployment_id, sp) %>% # Group species together
mutate(duration = int_length(timestamp %--% lag(timestamp))) # Calculate the gap bet
library(stringr)
# An image starts a new event if it is the first for that species at that deployment
# (duration is NA), or if the gap since the previous image is longer than the threshold
img_tmp$new_event <- is.na(img_tmp$duration) | img_tmp$duration > (independent * 60)
# Count up the new events to give every image its event number, then format it as a code
img_tmp$event_id <- paste0("E", str_pad(cumsum(img_tmp$new_event), nchar(nrow(img_tmp)) + 1, pad = "0"))
# Remove the working columns
img_tmp$duration <- NULL
img_tmp$new_event <- NULL
# find out the last and the first of the time in the group
top <- img_tmp %>% group_by(event_id) %>% top_n(1,timestamp) %>% dplyr::select(event_id, timestamp)
bot <- img_tmp %>% group_by(event_id) %>% top_n(-1,timestamp) %>% dplyr::select(event_id, timestamp)
names(bot)[2] <- c("timestamp_end")
img_num <- img_tmp %>% group_by(event_id) %>% summarise(event_observations=n()) # number of images in the event
event_grp <- img_tmp %>% group_by(event_id) %>% summarise(event_groupsize=max(animal_count))
# calculate the duration and add the other elements
diff <- top %>% left_join(bot, by="event_id") %>%
mutate(event_duration=abs(int_length(timestamp %--% timestamp_end))) %>%
left_join(event_grp, by="event_id")%>%
left_join(img_num, by="event_id")
# Remove columns you don't need
diff$timestamp <-NULL
diff$timestamp_end <-NULL
diff <- diff[duplicated(diff)==F,]
# Merge the img_tmp with the event data
img_tmp <-
left_join(img_tmp,diff,by="event_id")
# Remove duplicates
ind_dat <- img_tmp[duplicated(img_tmp$event_id)==F,]
# Make a unique code for ever day and deployment where cameras were functioning
tmp <- paste(row_lookup$date, row_lookup$placename)
#Subset ind_dat to data that matches the unique codes
ind_dat <- ind_dat[paste(substr(ind_dat$timestamp,1,10), ind_dat$placename) %in% tmp, ]
# Convert your species names to factors
ind_dat$sp <- as.factor(ind_dat$sp)
# Export your data frames
write.csv(ind_dat, paste0("data/processed_data/",ind_dat$project_id[1], "_",independent ,"min_independent_detections.csv"), row.names = F)
# also write the cleaned all detections file (some activity analyses require it)
write.csv(img_tmp, paste0("data/processed_data/",ind_dat$project_id[1], "_raw_detections.csv"), row.names = F)
write.csv(row_lookup, paste0("data/processed_data/",ind_dat$project_id[1], "_daily_lookup.csv"), row.names = F)
#Subset the columns
tmp <- dep[, c("project_id", "placename", "longitude", "latitude", "feature_type")]
# Remove duplicated rows
tmp<- tmp[duplicated(tmp)==F,]
# write the file
write.csv(tmp, paste0("data/processed_data/",ind_dat$project_id[1], "_camera_locations.csv"), row.names = F)
tmp <- sp_list[sp_list$sp %in% ind_dat$sp,]
write.csv(tmp, paste0("data/processed_data/",ind_dat$project_id[1], "_species_list.csv"), row.names = F)15.1.14 Create analysis dataframes
In the next step, we won’t create the daily dataframe as it takes too long!
# Total counts
# Station / Month / deport / Species
tmp <- row_lookup
# Calculate the number of days at each site
total_obs <- tmp %>%
group_by(placename) %>%
summarise(days = n())
# Convert to a data frame
total_obs <- as.data.frame(total_obs)
# Add columns for each species
total_obs[, levels(ind_dat$sp)] <- NA
# Duplicate for counts
total_count <- total_obs
# Test counter
i <-1
# For each station, count the number of individuals/observations
for(i in 1:nrow(total_obs))
{
tmp <- ind_dat[ind_dat$placename==total_obs$placename[i],]
tmp_stats <- tmp %>% group_by(sp, .drop=F) %>% summarise(obs=n(), count=sum(animal_count))
total_obs[i,as.character(tmp_stats$sp)] <- tmp_stats$obs
total_count[i,as.character(tmp_stats$sp)] <- tmp_stats$count
}
# Save them
write.csv(total_obs, paste0("data/processed_data/",ind_dat$project_id[1], "_",independent ,"min_independent_total_observations.csv"), row.names = F)
write.csv(total_count, paste0("data/processed_data/",ind_dat$project_id[1], "_",independent ,"min_independent_total_counts.csv"), row.names = F)
# Monthly counts
# Station / Month / days / Covariates / Species
tmp <- row_lookup
# Simplify the date to monthly
tmp$date <- substr(tmp$date,1,7)
# Calculate the number of days in each month
mon_obs <- tmp %>%
group_by(placename,date ) %>%
summarise(days = n())
# Convert to a data frame
mon_obs <- as.data.frame(mon_obs)
mon_obs[, levels(ind_dat$sp)] <- NA
mon_count <- mon_obs
# For each month, count the number of individuals/observations
for(i in 1:nrow(mon_obs))
{
tmp <- ind_dat[ind_dat$placename==mon_obs$placename[i] & substr(ind_dat$timestamp,1,7)== mon_obs$date[i],]
tmp_stats <- tmp %>% group_by(sp, .drop=F) %>% summarise(obs=n(), count=sum(animal_count))
mon_obs[i,as.character(tmp_stats$sp)] <- tmp_stats$obs
mon_count[i,as.character(tmp_stats$sp)] <- tmp_stats$count
}
write.csv(mon_obs, paste0("data/processed_data/",ind_dat$project_id[1], "_",independent ,"min_independent_monthly_observations.csv"), row.names = F)
write.csv(mon_count, paste0("data/processed_data/",ind_dat$project_id[1], "_",independent ,"min_independent_monthly_counts.csv"), row.names = F)
### Weekly
# Weekly format
# Station / Month / days / Covariates / Species
tmp <- row_lookup
# Simplify the date to year-week
tmp$date <- strftime(tmp$date, format = "%Y-W%U", tz = "UTC")
# The way this is coded is the counter W01 starts at the first Sunday of the year, everything before that is W00. Weeks do not roll across years.
# Calculate the number of days in each week
week_obs <- tmp %>%
group_by(placename,date ) %>%
summarise(days = n())
# Convert to a data frame
week_obs <- as.data.frame(week_obs)
# Add species columns
week_obs[, levels(ind_dat$sp)] <- NA
# Duplicate for counts
week_count <- week_obs
# For each week, count the number of individuals/observations
for(i in 1:nrow(week_obs))
{
tmp <- ind_dat[ind_dat$placename==week_obs$placename[i] & strftime(ind_dat$timestamp, format = "%Y-W%U")== week_obs$date[i],]
tmp_stats <- tmp %>% group_by(sp, .drop=F) %>% summarise(obs=n(), count=sum(animal_count))
week_obs[i,as.character(tmp_stats$sp)] <- tmp_stats$obs
week_count[i,as.character(tmp_stats$sp)] <- tmp_stats$count
}
write.csv(week_obs, paste0("data/processed_data/",ind_dat$project_id[1], "_",independent ,"min_independent_weekly_observations.csv"), row.names = F)
write.csv(week_count, paste0("data/processed_data/",ind_dat$project_id[1], "_",independent ,"min_independent_weekly_counts.csv"), row.names = F) 15.1.15 Check analysis dataframe counts
# Do an final check of the counts
tmp <- cbind(data.frame("Time"=c("Total", "Monthly", "Weekly")),
rbind(colSums(total_obs[,2:ncol(total_obs)]),
colSums(mon_obs[,3:ncol(mon_obs)]),
colSums(week_obs[,3:ncol(week_obs)])))
tmp %>%
kbl() %>%
kable_styling(full_width = T) %>%
column_spec(1, bold = T, border_right = T)%>%
kableExtra::scroll_box(width = "100%")15.2 Add your covariates
# LOad the packages
library(kableExtra);library(dplyr); library(sf); library(MODISTools); library(lubridate); library(corrplot); library(traitdata); library(terra); library(osmdata); library(elevatr)15.2.1 Species Traits
# Start by reading in your species list
sp_summary <- read.csv(paste0("data/processed_data/",pro$project_id[1],"_species_list.csv"), header=T)
locs <- read.csv(paste0("data/processed_data/",pro$project_id[1],"_camera_locations.csv"), header=T)
# Species traits
library(traitdata)
data("elton_mammals")
elton_mammals$sp <- paste0(elton_mammals$Genus,"." ,elton_mammals$Species)
tmp <- elton_mammals[, c("sp","BodyMass.Value", "Activity.Nocturnal", "Activity.Crepuscular", "Activity.Diurnal")]
# Lets rename the columns to make them more usable
tmp <- tmp %>% rename(
mass_g = BodyMass.Value,
act_noct = Activity.Nocturnal,
act_crep = Activity.Crepuscular,
act_diur = Activity.Diurnal)
sp_summary <- left_join(sp_summary, tmp)QUESTION Are there any missing values you need to fill in?
Don’t worry about adding them right now - carry on!
15.3 Spatial data
15.3.2 Get elevation data
###################
# Elevation
library(elevatr)
locs_sf <- get_elev_point(locs_sf,
src="aws", #Amazon Web Service Terrain Tiles - available globally
z = 12) # z specifies the zoom level, the lower the value the faster the code runs, but the coarser the elevation values are
boxplot(locs_sf$elevation)15.3.3 Open street map data
#######
# Distance to water (there are no roads here)
library(osmdata)
aoi <- st_bbox(st_buffer(locs_sf, 10000)) # Units are in meters
water <- opq(aoi) %>%
add_osm_feature(key = 'natural', value = 'water') %>%
osmdata_sf()
water
# THERE IS NO OPEN STREET MAP DATA FOR THIS AREA - ARGH15.3.3.0.0.0.1