040e01004d8a558fc6d713141368a6393d022198
hiram
  Wed Apr 10 14:46:54 2019 -0700
now returning errors properly refs #18869

diff --git src/hg/hubApi/tests/jsonConsumer.pl src/hg/hubApi/tests/jsonConsumer.pl
index d77ec7c..0f177eb 100755
--- src/hg/hubApi/tests/jsonConsumer.pl
+++ src/hg/hubApi/tests/jsonConsumer.pl
@@ -42,30 +42,54 @@
 -start=<coordinate> - restrict the operation to a range, use both start and end
 -end=<coordinate> - restrict the operation to a range, use both start and end
 -maxItemsOutput=<N> - limit output to this number of items.  Default 1,000
                       maximum allowed 1,000,000
 -endpoint=<function> - where <function> is one of the following:
    /list/publicHubs - provide a listing of all available public hubs
    /list/ucscGenomes - provide a listing of all available UCSC genomes
    /list/hubGenomes - list genomes from a specified hub (with hubUrl=...)
    /list/tracks - list data tracks available in specified hub or database genome
    /list/chromosomes - list chromosomes from specified data track
    /getData/sequence - return sequence from specified hub or database genome
    /getData/track - return data from specified track in hub or database genome
 ";
 }
 
+#########################################################################
+# generic output of a hash pointer
+sub hashOutput($) {
+  my ($hashRef) = @_;
+  foreach my $key (sort keys %$hashRef) {
+    my $value = $hashRef->{$key};
+    $value = "<array>" if (ref($value) eq "ARRAY");
+    $value = "<hash>" if (ref($value) eq "HASH");
+     printf STDERR "%s - %s\n", $key, $hashRef->{$key};
+  }
+}
+
+sub arrayOutput($) {
+  my ($ary) = @_;
+  my $i = 0;
+  foreach my $element (@$ary) {
+     printf STDERR "# %d\t%s\n", $i++, ref($element);
+     if (ref($element) eq "HASH") {
+       hashOutput($element);
+     }
+  }
+}
+#########################################################################
+
 ##############################################################################
 ###
 ### these functions were copied from Ensembl HTTP::Tiny example code:
 ###  https://github.com/Ensembl/ensembl-rest/wiki/Example-Perl-Client
 ###
 ##############################################################################
 
 ##############################################################################
 sub performJsonAction {
   my ($endpoint, $parameters) = @_;
   my $headers = $global_headers;
   my $content = performRestAction($endpoint, $parameters, $headers);
   return {} unless $content;
   my $json = decode_json($content);
   return $json;
@@ -96,69 +120,54 @@
     my @params;
     foreach my $key (keys %{$parameters}) {
       my $value = $parameters->{$key};
       push(@params, "$key=$value");
     }
     my $param_string = join(';', @params);
     $url.= '?'.$param_string;
   }
   if ($debug) { $url .= ";debug=1"; }
   printf STDERR "### '%s'\n", $url;
   my $response = $http->get($url, {headers => $headers});
   my $status = $response->{status};
   if(!$response->{success}) {
     # Quickly check for rate limit exceeded & Retry-After (lowercase due to our client)
     if($status == 429 && exists $response->{headers}->{'retry-after'}) {
+      my ($status, $reason) = ($response->{status}, $response->{reason});
       my $retry = $response->{headers}->{'retry-after'};
+      printf STDERR "Failed for $endpoint! Status code: ${status}. Reason: ${reason}, retry-after: $retry seconds\n";
+#      hashOutput($response->{headers});
       Time::HiRes::sleep($retry);
       # After sleeping see that we re-request
       return performRestAction($endpoint, $parameters, $headers);
     }
     else {
       my ($status, $reason) = ($response->{status}, $response->{reason});
-      die "Failed for $endpoint! Status code: ${status}. Reason: ${reason}\n";
+#      die "Failed for $endpoint! Status code: ${status}. Reason: ${reason}\n";
+      printf STDERR "Failed for $endpoint! Status code: ${status}. Reason: ${reason}\n";
+# hashOutput($response->{headers});
+# printf STDERR "'%s'\n", $response->{headers};
+      return return $response->{content};
     }
   }
   $request_count++;
   if(length $response->{content}) {
     return $response->{content};
   }
   return;
 }
 
-# generic output of a hash pointer
-sub hashOutput($) {
-  my ($hashRef) = @_;
-  foreach my $key (sort keys %$hashRef) {
-    my $value = $hashRef->{$key};
-    $value = "<array>" if (ref($value) eq "ARRAY");
-    $value = "<hash>" if (ref($value) eq "HASH");
-     printf STDERR "%s - %s\n", $key, $hashRef->{$key};
-  }
-}
-
-sub arrayOutput($) {
-  my ($ary) = @_;
-  my $i = 0;
-  foreach my $element (@$ary) {
-     printf STDERR "# %d\t%s\n", $i++, ref($element);
-     if (ref($element) eq "HASH") {
-       hashOutput($element);
-     }
-  }
-}
-
 #############################################################################
 sub columnNames($) {
   my ($nameArray) = @_;
   if (ref($nameArray) ne "ARRAY") {
     printf "ERROR: do not have an array reference in columnNames\n";
   } else {
     printf "### Column names in table return:\n";
     my $i = 0;
     foreach my $name (@$nameArray) {
       printf "%d\t\"%s\"\n", ++$i, $name;
     }
   }
 }
 
 sub topLevelKeys($) {