How can I calculate the slope of multiple subsets of a dataframe more efficiently?
I have a dataset that contains optical absorption ( a
) in the wavelength range ( wl
) for 16 different samples ( bod
) on 5 different days ( day
). The output dput
for samples ( bod
) 1 - 3 is below.
I need the slope of the natural log absorbance versus wavelength for each sample on each day.
My current approach was to do it manually:
# calculate and extract the slope for each sample and date
s275.1.0 <- lm(log(a) ~ wl, data = spec275, subset = bod == 1 & day == "2014-06-10")
s275.1.0.slope <- coef(s275.1.0)["wl"]
s275.2.0 <- lm(log(a) ~ wl, data = spec275, subset = bod == 2 & day == "2014-06-10")
s275.2.0.slope <- coef(s275.1.0)["wl"]
# etc...
# combine slopes into a vector
s275.slopes <- c(s275.1.0, s275.2.0) # etc...
Obviously, this is pretty tedious. Is there an easy way to streamline this code to get R to repeat these calculations over all patterns and days?
Output signal
structure(list(bod = c(1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L,
1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L,
1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 2L, 2L, 2L, 2L, 2L, 2L, 2L,
2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L,
2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 3L, 3L, 3L, 3L,
3L, 3L, 3L, 3L, 3L, 3L, 3L, 3L, 3L, 3L, 3L, 3L, 3L, 3L, 3L, 3L,
3L, 3L, 3L, 3L, 3L, 3L, 3L, 3L, 3L, 3L, 3L, 3L, 3L, 3L, 3L),
CPOM = structure(c(2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L,
2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L,
2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 1L, 1L, 1L, 1L, 1L,
1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L,
1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L,
1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L,
1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L,
1L, 1L, 1L, 1L, 1L), .Label = c("no", "yes"), class = "factor"),
Nutrient = structure(c(1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L,
1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L,
1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 2L, 2L, 2L, 2L,
2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L,
2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L,
2L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L,
1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L,
1L, 1L, 1L, 1L, 1L, 1L), .Label = c("no", "yes"), class = "factor"),
day = structure(c(5L, 5L, 5L, 5L, 5L, 5L, 2L, 2L, 2L, 5L,
2L, 2L, 2L, 1L, 2L, 2L, 1L, 1L, 1L, 1L, 1L, 4L, 4L, 4L, 4L,
4L, 4L, 4L, 3L, 3L, 3L, 3L, 3L, 3L, 3L, 2L, 2L, 2L, 2L, 2L,
2L, 2L, 1L, 1L, 1L, 1L, 3L, 3L, 3L, 3L, 3L, 3L, 3L, 1L, 4L,
4L, 4L, 4L, 4L, 4L, 4L, 1L, 1L, 5L, 5L, 5L, 5L, 5L, 5L, 5L,
1L, 1L, 1L, 1L, 1L, 1L, 1L, 2L, 2L, 2L, 2L, 2L, 3L, 5L, 5L,
5L, 2L, 3L, 5L, 2L, 5L, 5L, 5L, 3L, 3L, 3L, 3L, 3L, 4L, 4L,
4L, 4L, 4L, 4L, 4L), .Label = c("2014-06-10", "2014-06-12",
"2014-06-17", "2014-06-24", "2014-07-01"), class = "factor"),
wl = c(295L, 279L, 283L, 286L, 289L, 292L, 284L, 279L, 282L,
276L, 288L, 291L, 294L, 276L, 275L, 278L, 281L, 284L, 287L,
290L, 293L, 275L, 278L, 281L, 284L, 287L, 290L, 293L, 288L,
281L, 284L, 287L, 277L, 290L, 294L, 282L, 283L, 287L, 289L,
285L, 277L, 275L, 281L, 284L, 287L, 290L, 276L, 279L, 282L,
285L, 288L, 291L, 294L, 293L, 275L, 278L, 281L, 284L, 287L,
290L, 293L, 275L, 278L, 292L, 295L, 288L, 277L, 279L, 282L,
285L, 277L, 280L, 283L, 286L, 289L, 292L, 295L, 284L, 287L,
290L, 293L, 279L, 293L, 275L, 278L, 281L, 277L, 291L, 293L,
276L, 286L, 289L, 295L, 279L, 287L, 275L, 281L, 284L, 292L,
278L, 287L, 280L, 284L, 288L, 276L), abs = c(0.083, 0.105,
0.098, 0.093, 0.089, 0.086, 0.206, 0.221, 0.212, 0.109, 0.196,
0.19, 0.181, 0.436, 0.231, 0.224, 0.415, 0.397, 0.379, 0.361,
0.341, 0.124, 0.12, 0.114, 0.109, 0.105, 0.1, 0.096, 0.111,
0.124, 0.116, 0.113, 0.131, 0.108, 0.1, 0.123, 0.122, 0.115,
0.112, 0.118, 0.131, 0.136, 0.106, 0.1, 0.096, 0.093, 0.131,
0.128, 0.122, 0.117, 0.114, 0.11, 0.106, 0.086, 0.122, 0.118,
0.113, 0.109, 0.105, 0.103, 0.099, 0.114, 0.111, 0.099, 0.095,
0.103, 0.119, 0.116, 0.111, 0.107, 0.056, 0.055, 0.051, 0.048,
0.046, 0.043, 0.043, 0.144, 0.139, 0.135, 0.13, 0.153, 0.099,
0.097, 0.093, 0.088, 0.157, 0.102, 0.076, 0.159, 0.082, 0.078,
0.074, 0.119, 0.108, 0.122, 0.116, 0.11, 0.08, 0.097, 0.086,
0.094, 0.089, 0.084, 0.1), a = c(19.1149, 24.1815, 22.5694,
21.4179, 20.4967, 19.8058, 47.4418, 50.8963, 48.8236, 25.1027,
45.1388, 43.757, 41.6843, 100.4108, 53.1993, 51.5872, 95.5745,
91.4291, 87.2837, 83.1383, 78.5323, 28.5572, 27.636, 26.2542,
25.1027, 24.1815, 23.03, 22.1088, 25.5633, 28.5572, 26.7148,
26.0239, 30.1693, 24.8724, 23.03, 28.3269, 28.0966, 26.4845,
25.7936, 27.1754, 30.1693, 31.3208, 24.4118, 23.03, 22.1088,
21.4179, 30.1693, 29.4784, 28.0966, 26.9451, 26.2542, 25.333,
24.4118, 19.8058, 28.0966, 27.1754, 26.0239, 25.1027, 24.1815,
23.7209, 22.7997, 26.2542, 25.5633, 22.7997, 21.8785, 23.7209,
27.4057, 26.7148, 25.5633, 24.6421, 12.8968, 12.6665, 11.7453,
11.0544, 10.5938, 9.9029, 9.9029, 33.1632, 32.0117, 31.0905,
29.939, 35.2359, 22.7997, 22.3391, 21.4179, 20.2664, 36.1571,
23.4906, 17.5028, 36.6177, 18.8846, 17.9634, 17.0422, 27.4057,
24.8724, 28.0966, 26.7148, 25.333, 18.424, 22.3391, 19.8058,
21.6482, 20.4967, 19.3452, 23.03)), .Names = c("bod", "CPOM",
"Nutrient", "day", "wl", "abs", "a"), row.names = c(151L, 378L,
382L, 385L, 388L, 391L, 406L, 522L, 525L, 618L, 774L, 777L, 780L,
1105L, 1126L, 1129L, 1474L, 1477L, 1480L, 1483L, 1486L, 1656L,
1659L, 1662L, 1665L, 1668L, 1671L, 1674L, 2717L, 2832L, 2835L,
2838L, 2949L, 2963L, 3052L, 3143L, 3150L, 3169L, 3184L, 3194L,
3213L, 3293L, 3724L, 3727L, 3730L, 3733L, 4582L, 4585L, 4588L,
4591L, 4594L, 4597L, 4600L, 4829L, 5003L, 5006L, 5009L, 5012L,
5015L, 5018L, 5021L, 5541L, 5544L, 5807L, 5810L, 6127L, 6185L,
6200L, 6203L, 6206L, 6637L, 7006L, 7009L, 7012L, 7015L, 7018L,
7021L, 7157L, 7524L, 7527L, 7530L, 7881L, 7884L, 7975L, 7978L,
7981L, 8366L, 8369L, 8480L, 8486L, 8594L, 8597L, 8603L, 8844L,
8852L, 8961L, 8967L, 8970L, 9158L, 9268L, 9304L, 9310L, 9314L,
9345L, 9389L), class = "data.frame")
+3
source to share
4 answers
This is a typical split - apply - combine operation :
- split the data into appropriate chunks, in this case by
day
, thenbod
- apply a function to each piece of data; in this case linear model and extract the slope factor,
- combine the slope parameters plus the aggregation data into a final output object, usually a data frame.
There are many ways to do this, especially say using plyr , and more recently the dplyr packages , but I still feel that the basic R version is enlightening.
First, split the data up. Here I am using split()
and spl
ends up being a list of data frames, one per combination day
and bod
:
spl <- with(spec275, split(spec275, list(day = day, bod = bod)))
Now we want the wrapper function to fit the model lm()
and return the slope factor. This shell does it:
coefLM <- function(x) {
coef(lm(log(a) ~ wl, data = x))[2]
}
We apply in this case through sapply()
, but it vapply()
will be more secure, a wrapper function for each component spl
.
coefs <- sapply(spl, coefLM)
Return the slope vector:
head(coefs)
> head(coefs)
2014-06-10.1.wl 2014-06-12.1.wl 2014-06-17.1.wl 2014-06-24.1.wl 2014-07-01.1.wl
-0.01456420 -0.01285560 -0.01554979 -0.01446049 -0.01478953
2014-06-10.2.wl
-0.01545840
Then it is required to return this vector into an object with unique combinations of day
and bod
.
If you want it all together, follow these steps:
spl <- with(spec275, split(spec275, list(day = day, bod = bod)))
out <- unique(spec275[, c("day", "bod")])
out <- transform(out, slope = sapply(spl, coefLM))
> out
day bod slope
151 2014-07-01 1 -0.01456420
406 2014-06-12 1 -0.01285560
1105 2014-06-10 1 -0.01554979
1656 2014-06-24 1 -0.01446049
2717 2014-06-17 1 -0.01478953
3143 2014-06-12 2 -0.01545840
3724 2014-06-10 2 -0.01359000
4582 2014-06-17 2 -0.01197854
5003 2014-06-24 2 -0.01157193
5807 2014-07-01 2 -0.01237455
6637 2014-06-10 3 -0.01652268
7157 2014-06-12 3 -0.01177575
7884 2014-06-17 3 -0.01194247
7975 2014-07-01 3 -0.01388849
9158 2014-06-24 3 -0.01365501
+1
source to share
Let's assume your dataset is called "data":
library(dplyr)
library(broom) # to change model results into data frame
results = data %>%
group_by(bod, day) %>%
do(tidy(lm(log(a) ~ wl, data = .))) %>%
filter(term == "wl")
The results will be as follows:
> head(results)
Source: local data frame [6 x 7]
Groups: bod, day
bod day term estimate std.error statistic p.value
1 1 2014-06-10 wl -0.01456420 0.0008484573 -17.16550 6.757132e-05
2 1 2014-06-12 wl -0.01285560 0.0002345532 -54.80887 2.476989e-09
3 1 2014-06-17 wl -0.01554979 0.0006418376 -24.22699 2.233117e-06
4 1 2014-06-24 wl -0.01446049 0.0002585413 -55.93107 3.455827e-08
5 1 2014-07-01 wl -0.01478953 0.0004929797 -30.00027 7.718303e-07
6 2 2014-06-10 wl -0.01545840 0.0007932582 -19.48723 6.567167e-06
+7
source to share
library(data.table)
setDT(spec275)[,.(slope=coef(lm(log(a) ~ wl))["wl"]),by=c("bod","day")]
bod day slope
1: 1 2014-07-01 -0.01478953
2: 1 2014-06-12 -0.01285560
3: 1 2014-06-10 -0.01456420
4: 1 2014-06-24 -0.01446049
5: 1 2014-06-17 -0.01554979
6: 2 2014-06-12 -0.01359000
7: 2 2014-06-10 -0.01545840
8: 2 2014-06-17 -0.01197854
9: 2 2014-06-24 -0.01157193
10: 2 2014-07-01 -0.01237455
11: 3 2014-06-10 -0.01652268
12: 3 2014-06-12 -0.01177575
13: 3 2014-06-17 -0.01194247
14: 3 2014-07-01 -0.01365501
15: 3 2014-06-24 -0.01388849
BaseR solution:
spec275$group<-with(spec275,interaction(bod,day))
do.call(rbind,lapply(unique(spec275$group),function(i){
data.frame(cbind(model=paste0("model",i),slope=coef(lm(log(a) ~ wl, data = spec275[spec275$group==i,]))["wl"]))}))
model slope
wl model1.2014-07-01 -0.0147895254594997
wl1 model1.2014-06-12 -0.0128555970222881
wl2 model1.2014-06-10 -0.0145641963972831
wl3 model1.2014-06-24 -0.0144604920742437
wl4 model1.2014-06-17 -0.015549793252286
wl5 model2.2014-06-12 -0.0135899954428342
wl6 model2.2014-06-10 -0.0154584020879208
wl7 model2.2014-06-17 -0.0119785366650048
wl8 model2.2014-06-24 -0.0115719324383674
wl9 model2.2014-07-01 -0.0123745534390794
wl10 model3.2014-06-10 -0.0165226797671165
wl11 model3.2014-06-12 -0.0117757454366332
wl12 model3.2014-06-17 -0.0119424723122258
wl13 model3.2014-07-01 -0.013655010998267
wl14 model3.2014-06-24 -0.0138884860775378
+2
source to share
Usage dplyr
:
library(dplyr)
spec275 %>% group_by(bod, day) %>%
do(model = lm(log(a) ~ wl, data = .)) %>%
mutate(coef=coef(model)["wl"])
gives:
head(results)
Source: local data frame [6 x 4]
bod day model coef
1 1 2014-06-10 <S3:lm> -0.01456420
2 1 2014-06-12 <S3:lm> -0.01285560
3 1 2014-06-17 <S3:lm> -0.01554979
4 1 2014-06-24 <S3:lm> -0.01446049
5 1 2014-07-01 <S3:lm> -0.01478953
6 2 2014-06-10 <S3:lm> -0.01545840
The model column contains the raw lm if you want other options.
+1
source to share