non ho capito la tua osservazione, non vedo nessun button group, cmq ho capito il problema, non si può inscatolare un CompoundBorder all'interno di un CompoundBorder quindi una possibilie soluzione è inscatolare un jpanel con bordo in un altro jpanel con bordo, ecco il codice funzionante:

codice:
package it.panel;

import java.awt.BorderLayout;
import java.awt.Container;
import java.awt.GridLayout;

import javax.swing.BorderFactory;
import javax.swing.JFrame;
import javax.swing.JLabel;
import javax.swing.JPanel;
import javax.swing.border.Border;
import javax.swing.border.CompoundBorder;
import javax.swing.border.EmptyBorder;
import javax.swing.border.TitledBorder;

public class ProvaBordo {
    static JPanel  outsidePanel;
    /**
     * 
     */
    public ProvaBordo() {
        JPanel insidePanel = new JPanel();                 
        TitledBorder title = BorderFactory.createTitledBorder("bla bla bla bla bla");
        Border marginInside = new EmptyBorder(10,10,10,10);        
        insidePanel.setBorder(new CompoundBorder(title, marginInside));
        insidePanel.setLayout(new GridLayout(0,1));
        insidePanel.add(new JLabel("uno"));
        insidePanel.add(new JLabel("uno"));
        insidePanel.add(new JLabel("uno"));
        insidePanel.add(new JLabel("uno"));
        
        outsidePanel = new JPanel();
        outsidePanel.setLayout(new BorderLayout());
        outsidePanel.setBorder(new EmptyBorder(10,10,10,10)); 
        outsidePanel.add(insidePanel);
       
    }

    public static void main(String[] args) {
        new ProvaBordo();
        JFrame frame = new JFrame("ciao");
        Container c = frame.getContentPane();        
        c.setLayout(new BorderLayout());
        c.add("North", outsidePanel);
        frame.setSize(500, 600);
        frame.setVisible(true);       
    }
}