You can try aggregate
using formula as below.
df
## cat subcat y1 y2
## 1 cat1 a 10 10
## 2 cat1 b 10 10
## 3 cat1 c 10 10
## 4 cat2 a 1 1
## 5 cat2 b 1 2
## 6 cat2 c 1 3
## 7 cat3 a 1 4
## 8 cat3 b 1 5
## 9 cat3 c 1 6
res <- aggregate(formula = cbind(y1, y2) * ifelse(cat == "cat1", 1, -1) ~ subcat, data = df,
FUN = sum)
cbind(cat = "new.cat1", res)
## cat subcat y1 y2
## 1 new.cat1 a 8 5
## 2 new.cat1 b 8 3
## 3 new.cat1 c 8 1