open(ROBOT,"/usr1/www/pages/mhs/psychotherapy/robot.txt") || die "Cannot open robot.txt\n";
$www_offset = "/usr1/www/pages";
while (<ROBOT>) {
   if (/Disallow: (.*)/) {
      $norobhref{$1} = 1;
      $file = $www_offset.$1;
      $norobfile{$file} = 1;
   }
}

foreach $href (keys %norobfile) {
   print "file = $href\n";
}
close(ROBOT);