Find below a simple example with a TAnnotationTool and a TRectangleTool.
If you try to scroll the chart with the right mouse button, it works fine over the TAnnotationTool but not over the TRectangleTool.

uses Series, TeeTools;

procedure TForm1.FormCreate(Sender: TObject);
begin
  Chart1.Legend.Hide;
  Chart1.View3D:=False;

  Chart1.AddSeries(TLineSeries).FillSampleValues;

  with Chart1.Tools.Add(TRectangleTool) as TRectangleTool do
  begin
    Shape.AutoSize:=False;
    Shape.Color:=clRed;
    Shape.Text:='Rectangle';
    Shape.Transparency:=30;
    Shape.Left:=400;
    Shape.Top:=100;
    Height:=100;
    Width:=100;
  end;

  with Chart1.Tools.Add(TAnnotationTool) as TAnnotationTool do
  begin
    Shape.CustomPosition:=True;
    Shape.AutoSize:=False;
    Shape.Text:='Annotation';
    Shape.Top:=100;
    Shape.Left:=200;
    Shape.Height:=100;
    Shape.Width:=100;
  end;
end;