Direi che potresti fare una cosa tipo questa:
codice:
    Private Sub ControllaStato()
        Dim c As Control
        Dim ck As CheckBox
        Dim n As Integer

        For Each c In GroupBox1.Controls
            If TypeOf c Is CheckBox Then
                ck = c
                If ck.Checked Then n += 1
            End If
        Next

        If n = 2 Then GroupBox1.Enabled = False
    End Sub

    Private Sub CheckBox1_CheckedChanged(ByVal sender As Object, ByVal e As System.EventArgs) Handles CheckBox1.CheckedChanged
        ControllaStato()
    End Sub

    Private Sub CheckBox2_CheckedChanged(ByVal sender As Object, ByVal e As System.EventArgs) Handles CheckBox2.CheckedChanged
        ControllaStato()
    End Sub

    Private Sub CheckBox3_CheckedChanged(ByVal sender As Object, ByVal e As System.EventArgs) Handles CheckBox3.CheckedChanged
        ControllaStato()
    End Sub
Ciao