lists = {{1, 2, 3, 4, 5},
{"a", "b", "c", "d", "e", "f", "g", "h", "i", "j"},
{"A", "B", "C", "D", "E", "F", "G"}};
lst = Flatten[lists];
indices = TakeList[Range[22], {5, 10, 7}];
FWIW, an attempt to make a custom TogglerBar
with ragged layout and Select/Deselect All
buttons attached to rows:
Manipulate[a,
{{a, {1}, Rotate[Style["SELECTION", "Panel", Gray, 14], 90 Degree]},
lists,
Panel @ raggedTogglerBarWAddRemove[#, lists] &}]

Manipulate[a,
{{a, {1}, Rotate[Style["SELECTION", "Panel", Gray, 14], 90 Degree]},
lists,
Panel @ raggedTogglerBarWAddRemove[#, lists, Automatic, Left] &}]

Manipulate[Grid[{Prepend["part "] @ part,
Prepend[StringForm["lst[[``]] ", part]] @ lst[[part]]},
Dividers -> All],
{part, {1}, Rotate[Style["SELECTION", "Panel", Gray, 14], 90 Degree]},
indices, Panel @ raggedTogglerBarWAddRemove[#, indices, lists] &}]

Code
ClearAll[toggleAddRemove, raggedTogglerBarWAddRemove, toRagged,
raggedTogglerBar, addRemoveTogglerBar]
toRagged[dims_, side_ : Automatic, spacings_ : Automatic] :=
Module[{s = side /. {Automatic | Right -> PadLeft, _ -> PadRight},
rs = spacings /. {Automatic -> 0, _ -> First[spacings, spacings]},
cs = spacings /. {Automatic -> 0, _ -> Last[spacings, spacings]}},
RawBoxes[ToBoxes[#] /. GridBox[x_, o___] :>
RuleCondition[GridBox[side[#, Automatic, ""] &@TakeList[First@x, dims],
"RowSpacings" -> rs, "ColumnSpacings" -> cs,
BaselinePosition -> Center, o]]]] &
raggedTogglerBar[Dynamic[xx_], values_, labels_ : Automatic, side_ : Automatic,
size_ : Automatic, spacings_ : Automatic, opts : OptionsPattern[]] :=
DynamicModule[{v = Flatten @ values,
lbls = Flatten[{labels /. Automatic -> values}],
lengths = Length /@ values},
toRagged[lengths,
side /. {Automatic | Right -> PadLeft, Left -> PadRight},
spacings]@
TogglerBar[Dynamic[xx], Thread[v -> lbls],
ImageSize -> size /. Automatic -> {25, 25},
Alignment -> Center, Appearance -> "Horizontal", opts]]
Examples:
Row[{raggedTogglerBar[Dynamic[q], lists], Dynamic[q]}, "\[VerticalSeparator]"]

raggedTogglerBar[Dynamic[r],
TakeList[CharacterRange["A", "Z"], {7, 3, 5, All}]]

raggedTogglerBar[Dynamic[r],
TakeList[CharacterRange["A", "Z"], {7, 3, 5, All}],
Automatic, Left]

Add/remove button icons:
addAllButton[size_ : 25] := Show[ColorReplace[
ToExpression @ FrontEndResource["FEBitmaps", "SquarePlusIconMedium"],
{GrayLevel[0.7019607777695525] -> Darker@Green}],
ImageSize -> size]
deleteAllButton[size_ : 25] := Show[ColorReplace[
ToExpression @ FrontEndResource["FEBitmaps", "SquareMinusIconMedium"],
{GrayLevel[0.7019607777695525] -> Red}], ImageSize -> size]
Row[{addAllButton[90], deleteAllButton[90]}, Spacer[5]]

SetAttributes[toggleAddRemove, HoldAllComplete]
toggleAddRemove[var_, v_] := If[IntersectingQ[var, v],
var = DeleteCases[var, Alternatives @@ v],
var = DeleteDuplicates[Join[var, v]]]
addRemoveTogglerBar[Dynamic[var_], vl_, size_ : Automatic,
spacings_ : Automatic, opts_ : OptionsPattern[]] :=
DynamicModule[{sz = size /. Automatic -> {25, 25},
sp = spacings /. Automatic -> {0, 0}},
If[! ValueQ[var], var = {}];
If[! ListQ[var], var = {}];
Grid[Table[With[{v = v},
{Setter[Dynamic[IntersectingQ[var, v],
toggleAddRemove[var, v] &], {True},
Dynamic[IntersectingQ[var, v] /.
{True -> deleteAllButton[sz], _ | False -> addAllButton[sz]}],
Appearance -> None, Alignment -> Center, ImageSize -> sz]}],
{v, vl}],
Spacings -> sp, BaselinePosition -> Center]]
raggedTogglerBarWAddRemove[Dynamic[xx_], values_, labels_ : Automatic,
side_ : Automatic, size_ : Automatic, spacings_ : Automatic,
opts : OptionsPattern[]] :=
Row[(side /. {Automatic | PadLeft | Right -> Identity, _ -> Reverse})@
{raggedTogglerBar[Dynamic[xx], values, labels, side, size, spacings],
addRemoveTogglerBar[Dynamic[xx], values, size, spacings, opts]},
Spacer[1]]
Examples:
Row[{Panel[raggedTogglerBarWAddRemove[Dynamic[w], lists],
Background -> LightBlue], Dynamic[w]}, "\[VerticalSeparator]"]

Row[{Panel[
raggedTogglerBarWAddRemove[Dynamic[z], lists, Automatic, Left],
Background -> LightBlue], Dynamic[z]}, "\[VerticalSeparator]"]
