Looking at Specialty Training Competition Ratios

#Load the required packages
require(rvest) #This package allows us to scrape data from html into R
require(dplyr) #To wrangle data
require(ggplot2) #To plot graphs

#The data for each year is laid out in its own separate page
x2015 <- read_html("http://specialtytraining.hee.nhs.uk/specialty-recruitment/competition-ratios/2015-competition-ratios/") %>%
  html_nodes("table") %>% 
  .[[1]] %>% 
  html_table()
x2014 <- read_html("http://specialtytraining.hee.nhs.uk/specialty-recruitment/competition-ratios/2014-competition-ratios/") %>%
  html_nodes("table") %>% 
  .[[1]] %>% 
  html_table() %>%
  select(-X3, -X5) #To remove the 2 columns with empty data
x2013 <- read_html("http://specialtytraining.hee.nhs.uk/specialty-recruitment/competition-ratios/2013-competition-ratios/") %>%
  html_nodes("table") %>% 
  .[[1]] %>% 
  html_table() %>%
  select(-X3, -X6, -X7)

x2015[2,1] <- "ACCS EM"
x2014[2,1] <- "ACCS EM"
x2013[2,1] <- "ACCS EM"

x2015[3,1] <- "Anaesthetics"
x2014[3,1] <- "Anaesthetics"
x2013[3,1] <- "Anaesthetics"

x2015[4,1] <- "Broad-Based Training"
x2014[4,1] <- "Broad-Based Training"
x2013[4,1] <- "Broad-Based Training"

x2015[8,1] <- "Core Medical Training"
x2014[8,1] <- "Core Medical Training"
x2013[7,1] <- "Core Medical Training"

x2013[8,1] <- "Core Psychiatry Training"

x2014[18,1] <- "Paediatrics"
x2013[16,1] <- "Paediatrics"

#Join up the data into one dataframe
x <- inner_join(x2013, x2014, by = "X1") %>% inner_join(x2015, by = "X1")

#Rename the columns correctly
colnames(x)[1] <- "Specialty"
colnames(x)[2] <- "Apps2013"
colnames(x)[3] <- "Posts2013"
colnames(x)[4] <- "Ratio2013"
colnames(x)[5] <- "Apps2014"
colnames(x)[6] <- "Posts2014"
colnames(x)[7] <- "Ratio2014"
colnames(x)[8] <- "Apps2015"
colnames(x)[9] <- "Posts2015"
colnames(x)[10] <- "Ratio2015"
#Remove the 1st row
x <- x[-1, ]
row.names(x) <- 1:nrow(x)

require(knitr)
kable(x)

#First let's add a column for the year
x2015 <- mutate(x2015, Year = 2015)
x2014 <- mutate(x2014, Year = 2014)
x2013 <- mutate(x2013, Year = 2013)

#Rename the columns (Could definitely be improved into an apply function)
colnames(x2015)[1] <- "Specialty"
colnames(x2015)[2] <- "Applicants"
colnames(x2015)[3] <- "Posts"
colnames(x2015)[4] <- "Ratio"

colnames(x2014)[1] <- "Specialty"
colnames(x2014)[2] <- "Applicants"
colnames(x2014)[3] <- "Posts"
colnames(x2014)[4] <- "Ratio"

colnames(x2013)[1] <- "Specialty"
colnames(x2013)[2] <- "Applicants"
colnames(x2013)[3] <- "Posts"
colnames(x2013)[4] <- "Ratio"

#Let's drop the first and last rows
x2015 <- x2015[-1, ]
x2015 <- x2015[-18, ]
row.names(x2015) <- 1:nrow(x2015)

#Repeat for the 2014 and 2013 dataframes
x2014 <- x2014[-1, ]
x2014 <- x2014[-19, ]
row.names(x2014) <- 1:nrow(x2014)

x2013 <- x2013[-1, ]
x2013 <- x2013[-17, ]
row.names(x2013) <- 1:nrow(x2013)

#Now we can bind the rows together
CompetitionRatios <- bind_rows(x2013, x2014) %>% bind_rows(x2015)

require(knitr)
kable(CompetitionRatios)

#Need to first convert the Specialty column from a vector of characters to factors
CompetitionRatios$Specialty <- as.factor(CompetitionRatios$Specialty)
CompetitionRatios$Ratio <- as.numeric(CompetitionRatios$Ratio)
CompetitionRatios$Year <- as.factor(CompetitionRatios$Year)
CompetitionRatios$Applicants <- as.numeric(CompetitionRatios$Applicants)
CompetitionRatios$Posts <- as.numeric(CompetitionRatios$Posts)

ggplot(data=CompetitionRatios, aes(x=Year, y=Ratio, group = Specialty, colour = Specialty)) +
  geom_line()

#Need to first convert the Specialty column from a vector of characters to factors
CompetitionRatios2 <- CompetitionRatios %>% filter(Specialty == "Anaesthetics" | Specialty == "Core Medical Training" | Specialty == "Core Surgical Training" | Specialty == "General Practice" | Specialty == "Paediatrics")

ggplot(data=CompetitionRatios2, aes(x=Year, y=Ratio, group = Specialty, colour = Specialty)) +
  geom_line() +
  geom_point(aes(shape = factor(Specialty)))

#Need to first convert the Specialty column from a vector of characters to factors
CompetitionRatios2 <- CompetitionRatios %>% filter(Specialty == "Anaesthetics" | Specialty == "Core Medical Training" | Specialty == "Core Surgical Training" | Specialty == "General Practice" | Specialty == "Paediatrics")

ggplot(data=CompetitionRatios2, aes(x=Year, y=Applicants, group = Specialty, colour = Specialty)) +
  geom_line() +
  geom_point(aes(shape = factor(Specialty)))

ggplot(data=CompetitionRatios2, aes(x=Year, y=Posts, group = Specialty, colour = Specialty)) +
  geom_line() +
  geom_point(aes(shape = factor(Specialty)))