ha funzionato mettendo un empty border della stessa dimensione del bordo che viene creato quando si apssa sopra con il mouse:

codice:
import java.awt.Dimension;
import java.awt.Insets;
import java.awt.event.MouseEvent;
import java.awt.event.MouseListener;
import java.io.File;
import java.util.Hashtable;

import javax.swing.*;
import javax.swing.border.*;

public class MenuBar extends JMenuBar implements MouseListener {

	public static JMenu menuFile;
	public static JMenu menuPlugin;
	public static JMenuItem miExit;

	public static Hashtable menuItems = new Hashtable();

	private static Hashtable commands;
	private static JMenuBar menubar;
	
	int space = 2;
	public MenuBar() {
		super();
		
		JMenu menu;	

		//make menu file
		menuFile = new JMenu("file");
		menuFile.addMouseListener(this);
		menuFile.setBorder(new EmptyBorder(space,space,space,space));
		add(menuFile);
		
		//make menu edit
		menu = new JMenu("edit");
		menu.addMouseListener(this);	
		menu.setBorder(new EmptyBorder(space,space,space,space));	
		add(menu);

		//make menu search
		menu = new JMenu("search");
		menu.addMouseListener(this);
		menu.setBorder(new EmptyBorder(space,space,space,space));		
		add(menu);
	} //end constructor	

	public void mouseClicked(MouseEvent arg0) {
	}

	public void mouseEntered(MouseEvent arg0) {		
		Object obj = arg0.getSource();
		if (obj instanceof JMenuItem) {			
			JMenuItem button = (JMenuItem) obj;
			button.setBorder(BorderFactory.createRaisedBevelBorder());			
		}
	}

	public void mouseExited(MouseEvent arg0) {	
		Object obj = arg0.getSource();
		if (obj instanceof JMenuItem) {
			JMenuItem button = (JMenuItem) obj;
			button.setBorder(new EmptyBorder(space,space,space,space));		
		}

	}

	public void mousePressed(MouseEvent arg0) {		
	}

	public void mouseReleased(MouseEvent arg0) {
	}

		public static void main (String[] args) {
			
		JFrame f = new JFrame();
		f.setJMenuBar(new MenuBar());		
		f.setVisible(true);
		f.setSize(400, 200);
}
} // end class