ok, ho risolto così:
codice:
function raber() {
    var coll = $('#rab_coll').val();
    if (!coll) {
        alert('Choose a collection!');
    } else {
        $.ajax({
            type: "POST",
            url: '../' + urlService + "anagrafiche_raber.php",
            headers: {
                Accept: 'application/octet-stream',
            },
            data: {coll: coll.toUpperCase()},
            beforeSend: function (xhr) {
                $('#modal_wait').modal('show');
            }
        }).done(function (res) {
            const a = document.createElement('a');
            a.style = 'display: none';
            document.body.appendChild(a);
            const blob = new Blob([res], {type: 'octet/stream'});
            const url = URL.createObjectURL(blob);
            a.href = url;
            a.download = coll + '.dat';
            a.click();
            URL.revokeObjectURL(url);
            $('#modal_wait').modal('hide');
        }).fail(function (err) {
            $('#modal_wait').modal('hide');
            alert("ERRORE raber: " + err);
            console.log(err)
        });
    }
}
e lato server:
Codice PHP:
    header("Cache-Control: public");
    header("Content-Description: File Transfer");
    header("Content-Disposition: attachment; filename=$coll.dat");
    header("Content-Type: application/octet-stream");
    header("Content-Transfer-Encoding: binary");
    header('Cache-Control: must-revalidate');
    header('Content-Length: ' . filesize($outFileExt));
    readfile($outFileExt);