Forse hai qualche errore nelle formule...
codice:
/*
 * To change this template, choose Tools | Templates
 * and open the template in the editor.
 */
package html.exercise;

/**
 *
 * @author Andrea
 */

class CustomGeometricException extends Exception {
    
    public CustomGeometricException() {
        super();
    }
    
    public CustomGeometricException (String err) {
        super(err);
    }
}
class Point {
    private double x, y;
    
    public Point(double x, double y) {
        this.x = x;
        this.y = y;
    }
    
    public Point() {
        this(0,0);
    }
    
    public double getX() {
        return this.x;
    }
    
    public double getY() {
        return this.y;
    }
    
    public void setX(double x) {
        this.x = x;
    }
    
    public void setY(double y) {
        this.y = y;
    }
    
    public String toString() {
        return "( "+x+" ; "+y+" )";
    }
}

class Retta {
    double a,b,c;
    double m,q;
        
    public Retta(double a, double b, double c) {
        this.a = a;
        this.b = b;
        this.c = c;
        
        m = (b == 0 ? Double.POSITIVE_INFINITY : -a/b);
        q = (b == 0 ? Double.NaN : -c/b);
    }
    
    public double getA() {
        return this.a;
    }
    
    public double getB() {
        return this.b;
    }
    
    public double getC() {
        return this.c;
    }
    
    public Point intersezione(Retta r) throws CustomGeometricException {
        double det = a*r.getB()-b*r.getA();
        if (det == 0) {
            throw new CustomGeometricException("Rette parallele");
        }
        double int_x = -(c*r.getB()-b*r.getC())/det;
        double int_y = (c*r.getA()-a*r.getC())/det;
        return new Point(int_x, int_y);
    }
    
}

public class Esercizio01 {
    
    public static void main (String[] args) {
        Retta r1 = new Retta(1,1,-1);
        Retta r2 = new Retta(-1,1,0);
        //Retta r2 = new Retta(1,1,0); - è parallala a r1
        
        try {
            System.out.println("Intersezione r1 / r2: "+r1.intersezione(r2));
        }
        catch (CustomGeometricException e) {
            e.printStackTrace();
        }
                
    }
    
}
Con le limitazioni del caso (calcolo in virgola mobile e perdita di precisione)