Subsetting with R and complex data frames in nested lists

Question 1

This is a classic case where Split-Apply-Combine works.

require(plyr)
llply(spp_output$npl, subset, y > 10)

Question 2

It seems like the problem is that you're dealing with a nested list. Here's a minimal example similar to yours:

# Some sample data. This is a nested list
set.seed(123)
myList <- list(
  npl = list(
    chr1 = data.frame(x = rnorm(10),
                      y = sample(30, 10, replace = TRUE)),
    chr2 = data.frame(x = rnorm(10),
                      y = sample(30, 10, replace = TRUE))
  )
)

# Compare the structure
str(myList)
# List of 1
#  $ npl:List of 2
#   ..$ chr1:'data.frame':  10 obs. of  2 variables:
#   .. ..$ x: num [1:10] -0.5605 -0.2302 1.5587 0.0705 0.1293 ...
#   .. ..$ y: int [1:10] 27 21 20 30 20 22 17 18 9 5
#   ..$ chr2:'data.frame':  10 obs. of  2 variables:
#   .. ..$ x: num [1:10] 1.787 0.498 -1.967 0.701 -0.473 ...
#   .. ..$ y: int [1:10] 2 14 24 4 17 7 4 23 27 12

So, how do we extract certain values from the "y" column from all of these sub-lists? Use lapply with unlist with recursive = FALSE as follows:

lapply(unlist(myList, recursive=FALSE), function(x) x[x$y > 5, ])
# $npl.chr1
#             x  y
# 1 -0.56047565 27
# 2 -0.23017749 21
# 3  1.55870831 20
# 4  0.07050839 30
# 5  0.12928774 20
# 6  1.71506499 22
# 7  0.46091621 17
# 8 -1.26506123 18
# 9 -0.68685285  9
# 
# $npl.chr2
#             x  y
# 2   0.4978505 14
# 3  -1.9666172 24
# 5  -0.4727914 17
# 6  -1.0678237  7
# 8  -1.0260044 23
# 9  -0.7288912 27
# 10 -0.6250393 12

To see what's going on, try str(unlist(myList, recursive = FALSE)). You'll see that the list has now been "flattened" to a certain extent. This allows you to use lapply to get to the individual data.frames directly.

Question 3

It appears that you want to select from bp$npl only those chrs where the y value is greater than 40. If so, the following should give you the desired results:

# find which chrs have a y value greater than 40
Indxs <- lapply(bp$npl, function(n) n$y > 40)

# subset just those
croppedList <- mapply(function(m, i) m[i, ], bp$npl, Indxs, SIMPLIFY=FALSE)

  ## BEFORE:

  > bp
  $npl
  $npl$chr1
         x  y evalue   fdr
  1  -0.63 77  -0.04 -0.06
  2   0.18 40  -0.02 -0.16
  3  -0.84 63   0.94 -1.47
  4   1.60 36   0.82 -0.48
  5   0.33 43   0.59  0.42
  6  -0.82 49   0.92  1.36
  7   0.49 30   0.78 -0.10
  8   0.74 49   0.07  0.39
  9   0.58 74  -1.99 -0.05
  10 -0.31 47   0.62 -1.38

  $npl$chr2
         x  y evalue   fdr
  1  -0.41 42   0.40  2.40
  2  -0.39 33  -0.61 -0.04
  3  -0.06 62   0.34  0.69
  4   1.10 74  -1.13  0.03
  5   0.76 69   1.43 -0.74
  6  -0.16 70   1.98  0.19
  7  -0.25 53  -0.37 -1.80
  8   0.70 50  -1.04  1.47
  9   0.56 71   0.57  0.15
  10 -0.69 60  -0.14  2.17

  ## AFTER: 

  > croppedList
  $chr1
         x  y evalue   fdr
  1  -0.63 77  -0.04 -0.06
  3  -0.84 63   0.94 -1.47
  5   0.33 43   0.59  0.42
  6  -0.82 49   0.92  1.36
  8   0.74 49   0.07  0.39
  9   0.58 74  -1.99 -0.05
  10 -0.31 47   0.62 -1.38

  $chr2
         x  y evalue   fdr
  1  -0.41 42   0.40  2.40
  3  -0.06 62   0.34  0.69
  4   1.10 74  -1.13  0.03
  5   0.76 69   1.43 -0.74
  6  -0.16 70   1.98  0.19
  7  -0.25 53  -0.37 -1.80
  8   0.70 50  -1.04  1.47
  9   0.56 71   0.57  0.15
  10 -0.69 60  -0.14  2.17