Calculate summary statistics and then merge all results into single data.frame

Question

I am trying to learn to simplify my code and merge multiple data.frames (>2) simultaneously into a single dataset. First, I would like to calculate the "site" mean, sd, and n (number of "individuals" at each site) for each of the four PCA columns (Morph_PC1, Morph_PC2, ...). Second, merge the results into a single data.frame. Below is sample data and code that I attempted this task.

I realize that there maybe a method that generates a single dataset that does not require merging and this would be great, but I would also like to know how to make the merge_all command from the package reshape to work.

Sample data:

WW_Data <- structure(list(Individual_ID = c("WW_00A_05", "WW_00A_03", "WW_00A_02", 
"WW_00A_01", "WW_00A_04", "WW_00A_06", "WW_00A_08", "WW_00A_09", 
"WW_00A_07", "WW_00A_10", "WW_09AB_14", "WW_09AB_09", "WW_09AB_13", 
"WW_10AD_01", "WW_10AD_09", "WW_10AD_04", "WW_10AD_02", "WW_10AD_03", 
"WW_10AD_07", "WW_10AD_08"), Site_Name = c("Alnön", "Alnön", 
"Alnön", "Alnön", "Alnön", "Alnön", "Alnön", "Alnön", "Alnön", 
"Alnön", "Anjan", "Anjan", "Anjan", "Anjan", "Anjan", "Anjan", 
"Anjan", "Anjan", "Anjan", "Anjan"), Morph_PC1 = c(-2.08424433316496, 
-1.85413711191957, -1.67227075271696, -1.0486265729884, -0.809415702756541, 
-2.81781338129716, -2.08471369525797, -0.183840575363918, -0.753930407169699, 
0.0719252507535882, 1.02353521593315, 1.34441686821234, 0.755249445355964, 
-0.564426004755035, 0.720689649641627, -0.243471506156601, -0.245437522679261, 
-0.69936850894502, 0.9160796809062, 2.2881261039382), Morph_PC2 = c(1.28499189140338, 
-0.349487815669147, 0.0148183164519594, -1.55929148726881, -0.681590397005219, 
1.21595114750227, 0.116028310510466, 0.187613229042593, -0.923592436104444, 
-1.50956083294446, 1.44864057855388, 1.46254159976068, 1.20375736157205, 
0.174071006609975, -0.722049893415186, 1.03516327411773, 0.808851776990861, 
-0.928263134752596, -0.175511637463994, -0.389421342417043), 
    Morph_PC3 = c(-0.445087364125436, -0.704903876393893, 0.161983939922481, 
    1.14604411022773, 0.701508422965674, -0.78133408496171, -0.306619974141955, 
    1.05643337302175, 0.163868647932456, -0.673344807228353, 
    -0.337986608605208, -1.01911125040091, 0.258004835638601, 
    -0.648040419259003, -0.196770002944659, 0.614010430132367, 
    0.755886614924319, -0.0631407344114064, -1.28178468134549, 
    0.226362214551239), Morph_PC4 = c(0.0476276463048772, 0.342957387676778, 
    -0.117383887482525, 0.289881853573214, 0.649579005842321, 
    0.600433718752986, 0.295294947111845, -0.293754065807853, 
    -0.43805381119461, 0.520363554131325, -0.393329204345947, 
    -1.05629143416274, -0.370922397397109, 0.115121369773473, 
    0.91445926597504, 0.280048079793911, -0.802245210297552, 
    0.00368405602889952, -0.251898295768711, -0.607995193037228
    )), .Names = c("Individual_ID", "Site_Name", "Morph_PC1", 
"Morph_PC2", "Morph_PC3", "Morph_PC4"), row.names = c(36L, 37L, 
38L, 39L, 40L, 41L, 42L, 43L, 44L, 45L, 137L, 138L, 139L, 140L, 
141L, 142L, 143L, 144L, 145L, 146L), class = "data.frame")

The code:

## Calculate statistics for each site ##
WW_PC1_Mean <- subset(melt(tapply(WW_Data$Morph_PC1,list(WW_Data$Site_Name),mean)), value != FALSE)
WW_PC1_SD <- subset(melt(tapply(WW_Data$Morph_PC1,list(WW_Data$Site_Name),sd)), value != FALSE)
WW_PC2_Mean <- subset(melt(tapply(WW_Data$Morph_PC2,list(WW_Data$Site_Name),mean)), value != FALSE)
WW_Site_SD <- subset(melt(tapply(WW_Data$Morph_PC2,list(WW_Data$Site_Name),sd)), value != FALSE)

## merge the all the datasets with one command - THIS FAILS!
WW_Stats <- merge_all(WW_Site_PC1_Mean, WW_Site_PC1_SD, WW_Site_PC2_Mean, by = c("indices"))

Edits: Now I have a good result for quickly getting the summary stats into three files, but I still have a problem trying to merge_all (although I am unsure whether I should be using merge_recurse - regardless I get the same error) the results. Here is my attempt:

## Calculate statistics for each site ##
WW_Site_PC_Mean <- ddply(WW_Data, .(Site_Name), numcolwise(mean))
colnames(WW_Site_PC_Mean) <- c("Site_Name", "PC1_Mean", "PC2_Mean", "PC3_Mean", "PC4_Mean")
WW_Site_PC_SD <- ddply(WW_Data, .(Site_Name), numcolwise(sd))
colnames(WW_Site_PC_Mean) <- c("Site_Name", "PC1_SD", "PC2_SD", "PC3_SD", "PC4_SD")
WW_Site_PC_N <- count(WW_Data$Site_Name)
colnames(WW_Site_PC_N) <- c("Site_Name", "PCA_N")


## merge the all the datasets with one command - THIS FAILS!
WW_Stats <- merge_recurse(WW_Site_PC_Mean, WW_Site_PC_SD, WW_Site_PC_N, by = "Site_Name")

Error output:

Error in fix.by(by.x, x) : 
  'by' must specify column(s) as numbers, names or logical

Answer 1

Staying in base R, you can use aggregate:

WW_Data_mean = aggregate(list(mean = WW_Data[, -c(1, 2)]), 
                         list(Site_Name = WW_Data$Site_Name), mean)
WW_Data_sd = aggregate(list(mean = WW_Data[, -c(1, 2)]), 
                       list(Site_Name = WW_Data$Site_Name), sd)

Update (the second part of your question)

Your code has several mistakes, and perhaps you need to "play" with merge a little bit more.

First, the mistakes. The line that fails in your example fails because:

1. It isn't structured correctly; the data.frames to be merged should be in a list.
2. It references objects that don't exist from your example! You're trying to merge an object named WW_Site_Name_PC1_Mean but the name of the object is WW_PC1_Mean.

Second, here are some other things to try. Fix your column names:

# Fix your column names
# There's probably an easier way to do this, but...
names(WW_PC1_Mean)[2] = "WW_PC1_Mean"
names(WW_PC1_SD)[2] = "WW_PC1_SD"
names(WW_PC2_Mean)[2] = "WW_PC2_Mean"
names(WW_Site_SD)[2] = "WW_Site_SD"

Now, try merge_all. Note that you need to provide a list of data.frames. It seems that merge_all always just gives two columns---but maybe I'm doing something wrong.

# Not what you want
merge_all(list(WW_PC1_Mean, WW_PC1_SD, 
               WW_PC2_Mean, WW_Site_SD), by="indices")
  indices WW_PC1_Mean
1   Alnön  -1.3237067
2   Anjan   0.5295393

Move on to merge_recurse. This works:

# This is what you want
merge_recurse(list(WW_PC1_Mean, WW_PC1_SD, 
                   WW_PC2_Mean, WW_Site_SD), by="indices")
  indices WW_PC1_Mean WW_PC1_SD WW_PC2_Mean WW_Site_SD
1   Alnön  -1.3237067 0.9252417   -0.220412  0.9912227
2   Anjan   0.5295393 0.9511800    0.391778  0.9112450

You can also use Reduce in base R.

# Base R also has a solution
Reduce(function(x, y) merge(x, y, all=TRUE), 
       list(WW_PC1_Mean, WW_PC1_SD, WW_PC2_Mean, WW_Site_SD))