apply function to return a data.table, or convert the list directly to a data.table

Question

I would like to apply a function that returns a matrix to each row of a large data.table object (original file is around 30 GB, I have 80 GB ram), and get back a data.table object. I'd like to do it efficiently. My current approach is the following:

my.function <- function(x){
    alnRanges<-cigarToIRanges(x[6]);
    alnStarts<-start(alnRanges)+as.numeric(x[4])-1;
    alnEnds<-end(alnRanges)+as.numeric(x[4])-1;
    y<-x[-4];
    ys<-matrix(rep(y,length(alnRanges)),nrow=length(alnRanges),ncol=length(y),byrow=TRUE);
    ys<-cbind(ys,alnStarts,alnEnds);
    return(ys);     # ys is a matrix
}

    my.dt<-fread(my.file.name);
    my.list.of.matrices<-apply(my.dt,1,my.function);
    new.df<-do.call(rbind.data.frame,my.list.of.matrices);
    colnames(new.df)[1:14]<-colnames(my.dt)[-4];
    new.dt<-as.data.table(new.df);

Note1: I specify the my.function just to show that it returns a matrix, and that my apply line is therefore a list of matrices.

Note2: I am not sure how slow are the operations I am doing but seems that I could reduce the number of lines. For example, is it slow to convert a data frame to a data table for large objects?

Reproducible example:

Note that Arun and Roland made me think harder about the problem so I am still working on it... may be that I do not need these lines...

I want to take a sam file, and then create a new coordinates file where each read is split according to its CIGAR field.

My sam file:
qname   rname   pos cigar
2218    chr1    24613476    42M2S
2067    chr1    87221030    44M
2129    chr1    79702717    44M
2165    chr1    43113438    44M
2086    chr1    52155089    4M921N40M

code:

library("data.table");
library("GenomicRanges");

sam2bed <- function(x){
    alnRanges<-cigarToIRanges(x[4]);
    alnStarts<-start(alnRanges)+as.numeric(x[3])-1;
    alnEnds<-end(alnRanges)+as.numeric(x[3])-1;
    #y<-as.data.frame(x[,pos:=NULL]);
    #ys<-y[rep(seq_len(nrow(y)),length(alnRanges)),];
    y<-x[-3];
    ys<-matrix(rep(y,length(alnRanges)),nrow=length(alnRanges),ncol=length(y),byrow=TRUE);
    ys<-cbind(ys,alnStarts,alnEnds);
    return(ys);
}


sam.chr.dt<-fread(sam.parent.chr.file);
setnames(sam.chr.dt,old=c("V1","V2","V3","V4"),new=c("qname","rname","pos","cigar"));
bed.chr.lom<-apply(sam.chr.dt,1,sam2bed);
> bed.chr.lom
[[1]]
                           alnStarts  alnEnds   
[1,] "2218" "chr1" "42M2S" "24613476" "24613517"

[[2]]
                         alnStarts  alnEnds   
[1,] "2067" "chr1" "44M" "87221030" "87221073"

[[3]]
                         alnStarts  alnEnds   
[1,] "2129" "chr1" "44M" "79702717" "79702760"

[[4]]
                         alnStarts  alnEnds   
[1,] "2165" "chr1" "44M" "43113438" "43113481"

[[5]]
                               alnStarts  alnEnds   
[1,] "2086" "chr1" "4M921N40M" "52155089" "52155092"
[2,] "2086" "chr1" "4M921N40M" "52156014" "52156053"

bed.chr.df<-do.call(rbind.data.frame,bed.chr.lom);

> bed.chr.df
    V1   V2        V3 alnStarts  alnEnds
1 2218 chr1     42M2S  24613476 24613517
2 2067 chr1       44M  87221030 87221073
3 2129 chr1       44M  79702717 79702760
4 2165 chr1       44M  43113438 43113481
5 2086 chr1 4M921N40M  52155089 52155092
6 2086 chr1 4M921N40M  52156014 52156053

bed.chr.dt<-as.data.table(bed.chr.df);

> bed.chr.dt
     V1   V2        V3 alnStarts  alnEnds
1: 2218 chr1     42M2S  24613476 24613517
2: 2067 chr1       44M  87221030 87221073
3: 2129 chr1       44M  79702717 79702760
4: 2165 chr1       44M  43113438 43113481
5: 2086 chr1 4M921N40M  52155089 52155092
6: 2086 chr1 4M921N40M  52156014 52156053

Answer 1

Assuming ff is your data.table, how about this?

splits <- cigarToIRangesListByAlignment(ff$cigar, ff$pos, reduce.ranges = TRUE)
widths <- width(attr(splits, 'partitioning'))
cbind(data.table(qname=rep.int(ff$qname, widths), 
            rname=rep.int(ff$rname, widths)), as.data.frame(splits))

   qname rname space    start      end width
1:  2218  chr1     1 24613476 24613517    42
2:  2067  chr1     2 87221030 87221073    44
3:  2129  chr1     3 79702717 79702760    44
4:  2165  chr1     4 43113438 43113481    44
5:  2086  chr1     5 52155089 52155092     4
6:  2086  chr1     5 52156014 52156053    40