Problem is here. Consider i=popsize-1 then you will write to pop[perm][popsize] which is an invalid memory position. Then when allocations below are requested, sometimes requested memory position is intersected with pop[perm][popsize] so you are getting the runtime error.
for (i = 0; i < popSize; i++) {
for (j = 0; j < n; j++) {
perm = rand()%(n-j)+j;
t = pop[perm][i];
pop[perm][i] = pop[j][i];
if(i!=popsize-1)
pop[perm][i+1] = pop[j][i];
pop[j][i] = t;
if(i!=popsize-1)
pop[j][i+1] = t;
}
}
On the other hand there many wrong memory allocations:
pop = (int**)malloc(n * sizeof(int*));
for (i = 0; i < popSize; i++) {
pop[i] = (int*)malloc(popSize * sizeof(int));
}
should be
pop = (int**)malloc(n * sizeof(int*));
for (i = 0; i < n; i++) {
pop[i] = (int*)malloc(popSize * sizeof(int));
}
and
rtes = (int**)malloc(n * sizeof(int*));
for (i = 0; i < 4; i++) {
rtes[i] = (int*)malloc(4 * sizeof(int));
}
should be
rtes = (int**)malloc(n * sizeof(int*));
for (i = 0; i < n; i++) {
rtes[i] = (int*)malloc(4 * sizeof(int));
}
and
tmpPop = (int**)malloc(n * sizeof(int*));
for (i = 0; i < 4; i++) {
tmpPop[i] = (int*)malloc(4 * sizeof(int));
}
should be
tmpPop = (int**)malloc(n * sizeof(int*));
for (i = 0; i < n; i++) {
tmpPop[i] = (int*)malloc(4 * sizeof(int));
}
There is one more ambiguity in the problem.
numIter = 1*10^4;
This makes numIter 14 since ^ is logical XOR operator. Are you sure to want to do so?