Return to Answer

without any severe performance degression. The only thing that you will observe is that call cApplyNested[d] for the first time will require some extra time for compilation process. But since we use memoization here, the time of the next call cApplyNested[d] the compile function will be already known.

without any severe performance degression. The only thing that you will observe is that call cApplyNested[d] for the first time will require some extra time for compilation process. But since we use memoization here, the time of the next call cApplyNested[d] the CompiledFunction will be already known.

cApplyNested = Compile[{{X, _Real, 3}, {w, _Real, 1}},
   
   Block[{m, n, w1, w2, result, X11, X12, X21, X22},
    m = Dimensions[X][[2]];
    result = Table[0., {m + 1}, {2}];
    result[[1, 1]] = w1 = Compile`GetElement[w, 1];
    result[[1, 2]] = w2 = Compile`GetElement[w, 2];
    Do[
     X11 = Compile`GetElement[X, j, 1, 1];
     X12 = Compile`GetElement[X, j, 1, 2];
     X21 = Compile`GetElement[X, j, 2, 1];
     X22 = Compile`GetElement[X, j, 2, 2];
     result[[j + 1, 1]] = w1 = w1 - 0.1 (w1 (X11 X11 + X21 X21) + w2 (X11 X12 + X21 X22));
     result[[j + 1, 2]] = w2 = w2 - 0.1 (w1 (X11 X12 + X21 X22) + w2 (X12 X12 + X22 X22));
     , {j, 1, m}];
    result
    ],
   
   CompilationTarget -> "C",
   RuntimeAttributes -> {Listable},
   Parallelization -> True,
   RuntimeOptions -> "Speed"
   ];

Xlist = RandomVariate[NormalDistribution[], {numSamples, numSteps, b, d}];
wb0 = RandomVariate[NormalDistribution[], {numSamples, d}];
result = cf[Xlist, wb0]; 

ClearAll[cApplyNested];
cApplyNested[d_] := cApplyNested[d] = Block[{XX, X, j, w, ww, code},
    XX = Table[Compile`GetElement[X, j, k, l], {k, 1, d}, {l, 1, d}];
    ww = Table[Compile`GetElement[w, k], {k, 1, d}];
With[{code = ww - 0.1 XX\[Transpose] . XX . ww, dim = d},
 Compile[{{X, _Real, 3}, {w0, _Real, 1}},
  
  Block[{m, w, result},
   m = Dimensions[X][[2]];
   result = Table[0., {m + 1}, {dim}];
   w = Table[0., {dim}];
   
   result[[1]] = w = w0;
   
   Do[result[[j + 1]] = w = code;, {j, 1, m}];

   result
   ],
  
  CompilationTarget -> "C",
  RuntimeAttributes -> {Listable},
  Parallelization -> True,
  RuntimeOptions -> "Speed"
  ]
 ]
];

cApplyNested = Compile[{{X, _Real, 3}, {w, _Real, 1}},
   
   Block[{m, n, w1, w2, result, X11, X12, X21, X22},
    m = Dimensions[X][[1]];
    result = Table[0., {m + 1}, {2}];
    result[[1, 1]] = w1 = Compile`GetElement[w, 1];
    result[[1, 2]] = w2 = Compile`GetElement[w, 2];
    Do[
     X11 = Compile`GetElement[X, j, 1, 1];
     X12 = Compile`GetElement[X, j, 1, 2];
     X21 = Compile`GetElement[X, j, 2, 1];
     X22 = Compile`GetElement[X, j, 2, 2];
     result[[j + 1, 1]] = w1 = w1 - 0.1 (w1 (X11 X11 + X21 X21) + w2 (X11 X12 + X21 X22));
     result[[j + 1, 2]] = w2 = w2 - 0.1 (w1 (X11 X12 + X21 X22) + w2 (X12 X12 + X22 X22));
     , {j, 1, m}];
    result
    ],
   
   CompilationTarget -> "C",
   RuntimeAttributes -> {Listable},
   Parallelization -> True,
   RuntimeOptions -> "Speed"
   ];

Xlist = RandomVariate[NormalDistribution[], {numSamples, numSteps, b, d}];
wb0 = RandomVariate[NormalDistribution[], {numSamples, d}];
result = cApplyNested[Xlist, wb0]; 

ClearAll[cApplyNested];
cApplyNested[d_] := cApplyNested[d] = Block[{XX, X, j, w, ww, code},
    XX = Table[Compile`GetElement[X, j, k, l], {k, 1, d}, {l, 1, d}];
    ww = Table[Compile`GetElement[w, k], {k, 1, d}];
With[{code = ww - 0.1 XX\[Transpose] . XX . ww, dim = d},
 Compile[{{X, _Real, 3}, {w0, _Real, 1}},
  
  Block[{m, w, result},
   m = Dimensions[X][[1]];
   result = Table[0., {m + 1}, {dim}];
   w = Table[0., {dim}];
   
   result[[1]] = w = w0;
   
   Do[result[[j + 1]] = w = code;, {j, 1, m}];

   result
   ],
  
  CompilationTarget -> "C",
  RuntimeAttributes -> {Listable},
  Parallelization -> True,
  RuntimeOptions -> "Speed"
  ]
 ]
];

Edit

If you want to be a bit more flexible, you can employ Mathematica to symbolically generate the code. Then you can employ Compile as a JIT-compiler to create a function for each d you need (and only if and when you need it). This could be done as follows:

ClearAll[cApplyNested];
cApplyNested[d_] := cApplyNested[d] = Block[{XX, X, j, w, ww, code},
    XX = Table[Compile`GetElement[X, j, k, l], {k, 1, d}, {l, 1, d}];
    ww = Table[Compile`GetElement[w, k], {k, 1, d}];
With[{code = ww - 0.1 XX\[Transpose] . XX . ww, dim = d},
 Compile[{{X, _Real, 3}, {w0, _Real, 1}},
  
  Block[{m, w, result},
   m = Dimensions[X][[2]];
   result = Table[0., {m + 1}, {dim}];
   w = Table[0., {dim}];
   
   result[[1]] = w = w0;
   
   Do[result[[j + 1]] = w = code;, {j, 1, m}];

   result
   ],
  
  CompilationTarget -> "C",
  RuntimeAttributes -> {Listable},
  Parallelization -> True,
  RuntimeOptions -> "Speed"
  ]
 ]
];

Now you can do

numSteps = 20;
numSamples = 10000;
d = 2;

Xlist = RandomVariate[NormalDistribution[], {numSamples, numSteps, d, d}];
wb0 = RandomVariate[NormalDistribution[], {numSamples, d}];
result = cApplyNested[d][Xlist, wb0];

without any severe performance degression. The only thing that you will observe is that call cApplyNested[d] for the first time will require some extra time for compilation process. But since we use memoization here, the time of the next call cApplyNested[d] the compile function will be already known.