Ho risolto...posto il codice in caso servisse a qualcuno:


html5 (con l'input type="number")
codice:
<script src="http://code.jquery.com/jquery-latest.min.js" type="text/javascript"></script>
<script type="text/javascript">
    $(document).ready(function() {
        var limite = 5;
        var somma;
        var val;
        $(".number").change(function(){
            val = parseInt($(this).val());
            somma = 0;
            $(".number").each(function(){
                somma += parseInt($(this).val());
            });
            var resto = limite - somma > 0 ? limite - somma : 0;
            $("#limite").html(resto);
            if(somma > limite){
                var diff = somma - limite;
                $(this).val(val - diff);
            }
        })
    });
</script>
Limite : <span id="limite">5</span><br/>
a <input type="number" size="2" name="a" min="0" max="5" value="0" class="number"/><br/>
b <input type="number" size="2" name="b" min="0" max="5" value="0" class="number"/><br/>
c <input type="number" size="2" name="c" min="0" max="5" value="0" class="number"/><br/>
d <input type="number" size="2" name="d" min="0" max="5" value="0" class="number"/><br/>
e <input type="number" size="2" name="e" min="0" max="5" value="0" class="number"/><br/>

html vecchio (con l'input type="text")
codice:
        <script src="http://code.jquery.com/jquery-latest.min.js" type="text/javascript"></script>
        <script type="text/javascript">
            function char_consentiti(campo) {
                var re = new RegExp("^[0-9]+$", "");
                var valore = campo.value;  
                for ( var i = 0; i < valore.length; i++ ) {
                    if (!valore.charAt(i).match(re)) {
                        campo.value = valore.substring(0, i);
                        return;
                    }   
                }
            }  
    $(document).ready(function() {
        var limite = 5;
        var somma;
        var val;
        $(".number").keyup(function(){
            val = parseInt($(this).val());
            somma = 0;
            $(".number").each(function(){
                somma += parseInt($(this).val());
            });      
            if(somma > limite){
                var diff = somma - limite;
                $(this).val(val - diff);
            }
        })
    });
</script>
a <input type="text" size="2" name="a" value="0" class="number" onkeyup="char_consentiti(this)" /><br/>
b <input type="text" size="2" name="b" value="0" class="number" onkeyup="char_consentiti(this)" /><br/>
c <input type="text" size="2" name="c" value="0" class="number" onkeyup="char_consentiti(this)" /><br/>
d <input type="text" size="2" name="d" value="0" class="number" onkeyup="char_consentiti(this)" /><br/>
e <input type="text" size="2" name="e" value="0" class="number" onkeyup="char_consentiti(this)" /><br/>