Merge pull request #62 from j0k3r/rate-limit-reached

Stop processing messages when rate limit is reached
j0k3r · Mar 16, 2017 · 4088bac · 4088bac
2 parents 740f34f + c199503
commit 4088bac
Show file tree

Hide file tree

Showing 4 changed files with 96 additions and 21 deletions.
diff --git a/src/AppBundle/Consumer/SyncStarredRepos.php b/src/AppBundle/Consumer/SyncStarredRepos.php
@@ -52,7 +52,7 @@ public function process(Message $message, array $options)
         if (false === $this->client) {
             $this->logger->error('No client provided');
 
-            return;
+            return false;
         }
 
         $data = json_decode($message->getBody(), true);
@@ -67,13 +67,8 @@ public function process(Message $message, array $options)
 
         $this->logger->notice('Consume banditore.sync_starred_repos message', ['user' => $user->getUsername()]);
 
-        try {
-            $nbRepos = $this->doSyncRepo($user);
-        } catch (\Exception $e) {
-            $this->logger->error('Error while syncing starred repos', ['exception' => $e->getMessage(), 'user' => $user->getUsername()]);
-
-            return;
-        }
+        // this shouldn't be catched so the worker will die when an exception is thrown
+        $nbRepos = $this->doSyncRepo($user);
 
         $this->logger->notice('Synced repos: ' . $nbRepos, ['user' => $user->getUsername()]);
     }

diff --git a/src/AppBundle/Consumer/SyncVersions.php b/src/AppBundle/Consumer/SyncVersions.php
@@ -47,7 +47,7 @@ public function process(Message $message, array $options)
         if (false === $this->client) {
             $this->logger->error('No client provided');
 
-            return;
+            return false;
         }
 
         $data = json_decode($message->getBody(), true);
@@ -62,7 +62,15 @@ public function process(Message $message, array $options)
 
         $this->logger->notice('Consume banditore.sync_versions message', ['repo' => $repo->getFullName()]);
 
-        $this->logger->notice('[' . $this->getRateLimits($this->client, $this->logger) . '] Check <info>' . $repo->getFullName() . '</info> … ');
+        $rateLimit = $this->getRateLimits($this->client, $this->logger);
+
+        $this->logger->notice('[' . $rateLimit . '] Check <info>' . $repo->getFullName() . '</info> … ');
+
+        if (0 === $rateLimit || false === $rateLimit) {
+            $this->logger->warning('RateLimit reached, stopping.');
+
+            return false;
+        }
 
         // this shouldn't be catched so the worker will die when an exception is thrown
         $nbVersions = $this->doSyncVersions($repo);

diff --git a/tests/AppBundle/Consumer/SyncStarredReposTest.php b/tests/AppBundle/Consumer/SyncStarredReposTest.php
@@ -162,6 +162,10 @@ public function testProcessSuccessfulMessage()
         $this->assertSame('Synced repos: 1', $records[3]['message']);
     }
 
+    /**
+     * @expectedException \Exception
+     * @expectedExceptionMessage booboo
+     */
     public function testProcessUnexpectedError()
     {
         $em = $this->getMockBuilder('Doctrine\ORM\EntityManager')
@@ -238,26 +242,16 @@ public function testProcessUnexpectedError()
 
         $githubClient = $this->getMockClient($responses);
 
-        $logger = new Logger('foo');
-        $logHandler = new TestHandler();
-        $logger->pushHandler($logHandler);
-
         $processor = new SyncStarredRepos(
             $doctrine,
             $userRepository,
             $starRepository,
             $repoRepository,
             $githubClient,
-            $logger
+            new NullLogger()
         );
 
         $processor->process(new Message(json_encode(['user_id' => 123])), []);
-
-        $records = $logHandler->getRecords();
-
-        $this->assertSame('Consume banditore.sync_starred_repos message', $records[0]['message']);
-        $this->assertSame('    sync 1 starred repos', $records[1]['message']);
-        $this->assertSame('Error while syncing starred repos', $records[2]['message']);
     }
 
     /**

diff --git a/tests/AppBundle/Consumer/SyncVersionsTest.php b/tests/AppBundle/Consumer/SyncVersionsTest.php
@@ -370,6 +370,84 @@ public function testProcessRepoTagFailed()
         $this->assertContains('(repo/tags) <error>', $records[2]['message']);
     }
 
+    /**
+     * Not enough calls remaining.
+     */
+    public function testProcessCallsRemaingLow()
+    {
+        $em = $this->getMockBuilder('Doctrine\ORM\EntityManager')
+            ->disableOriginalConstructor()
+            ->getMock();
+        $em->expects($this->never())
+            ->method('isOpen');
+
+        $doctrine = $this->getMockBuilder('Doctrine\Bundle\DoctrineBundle\Registry')
+            ->disableOriginalConstructor()
+            ->getMock();
+        $doctrine->expects($this->never())
+            ->method('getManager');
+
+        $repo = new Repo();
+        $repo->setId(123);
+        $repo->setFullName('bob/wow');
+        $repo->setName('wow');
+
+        $repoRepository = $this->getMockBuilder('AppBundle\Repository\RepoRepository')
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $repoRepository->expects($this->once())
+            ->method('find')
+            ->with(123)
+            ->will($this->returnValue($repo));
+
+        $versionRepository = $this->getMockBuilder('AppBundle\Repository\VersionRepository')
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $pubsubhubbub = $this->getMockBuilder('AppBundle\PubSubHubbub\Publisher')
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $pubsubhubbub->expects($this->never())
+            ->method('pingHub');
+
+        $responses = new MockHandler([
+            // rate_limit
+            new Response(200, ['Content-Type' => 'application/json'], json_encode(['resources' => ['core' => ['remaining' => 0]]])),
+        ]);
+
+        $clientHandler = HandlerStack::create($responses);
+        $guzzleClient = new Client([
+            'handler' => $clientHandler,
+        ]);
+
+        $httpClient = new Guzzle6Client($guzzleClient);
+        $httpBuilder = new Builder($httpClient);
+        $githubClient = new GithubClient($httpBuilder);
+
+        $logger = new Logger('foo');
+        $logHandler = new TestHandler();
+        $logger->pushHandler($logHandler);
+
+        $processor = new SyncVersions(
+            $doctrine,
+            $repoRepository,
+            $versionRepository,
+            $pubsubhubbub,
+            $githubClient,
+            $logger
+        );
+
+        $processor->process(new Message(json_encode(['repo_id' => 123])), []);
+
+        $records = $logHandler->getRecords();
+
+        $this->assertSame('Consume banditore.sync_versions message', $records[0]['message']);
+        $this->assertSame('[0] Check <info>bob/wow</info> … ', $records[1]['message']);
+        $this->assertContains('RateLimit reached, stopping.', $records[2]['message']);
+    }
+
     /**
      * The call to markdown will return a bad response.
      */