Color each data point using Style
and remove the non-numeric data points:
data = {{1, 2, 3}, {None, 2, None}, {1, None, 5}};
colors = ColorData[59, "ColorList"][[;; Length@data[[1]]]];
styleddata = Style @@@ (Thread[{#, colors}]) & /@ data /. Style[ None, __] :> (## &[]);
BarChart[styleddata, ChartLegends -> SwatchLegend[59, {"A", "B", "C"}],
ChartLabels -> {{"X", "Y", "Z"}, None},
LabelingFunction -> (Placed[Row[{"", #}], Bottom] &), BarSpacing -> None]

SeedRandom[1]
data2 = RandomChoice[{1, 1, 1, 1, 1, 4} -> {1, 2, 3, 4, 5, None}, {5, 6}];
colors = ColorData[54, "ColorList"][[;; Length@data2[[1]]]];
styleddata = Style @@@ (Thread[{#, colors}]) & /@ data2 /. Style[ None, __] :> (## &[]);
BarChart[styleddata,
ChartLegends -> SwatchLegend[54, CharacterRange["A", "Z"][[;; Length @ data2[[1]]]]],
ChartLabels -> {{"X", "Y", "Z", "W", "V"}, None},
LabelingFunction -> (Placed[Row[{"", #}], Bottom] &), BarSpacing -> None]
