From cf9e5fee640f13678041025701cfe907e9b08123 Mon Sep 17 00:00:00 2001 From: Rello Date: Wed, 20 Aug 2025 17:47:52 +0700 Subject: [PATCH 1/8] Add GitHub Community SLA datasource --- CHANGELOG.md | 1 + lib/Datasource/GithubCommunitySla.php | 193 ++++++++++++++++++++ tests/Datasource/GithubCommunitySlaTest.php | 41 +++++ 3 files changed, 235 insertions(+) create mode 100644 lib/Datasource/GithubCommunitySla.php create mode 100644 tests/Datasource/GithubCommunitySlaTest.php diff --git a/CHANGELOG.md b/CHANGELOG.md index 6367538e..aaecb0cf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ### Added - Cache report data in the browser (using ETag & If-None-Match) #535 - Warn about unsaved changes before leaving a report +- GitHub Community SLA data source ### Fixed - Fix PHP 8.4 deprecation warnings #534 @[robertoschwald](https://github.com/robertoschwald) diff --git a/lib/Datasource/GithubCommunitySla.php b/lib/Datasource/GithubCommunitySla.php new file mode 100644 index 00000000..1a5eff13 --- /dev/null +++ b/lib/Datasource/GithubCommunitySla.php @@ -0,0 +1,193 @@ + + */ + protected array $excludedAuthors = [ + 'alice', + 'bob', + ]; + + /** + * Repositories to analyse in owner/repo format + * @var array + */ + protected array $repositories = [ + 'nextcloud/server', + 'nextcloud/analytics', + ]; + + public function __construct( + IL10N $l10n, + LoggerInterface $logger + ) { + $this->l10n = $l10n; + $this->logger = $logger; + } + + /** + * @return string Display Name of the data source + */ + public function getName(): string { + return 'GitHub Community SLA'; + } + + /** + * @return int digit unique data source id + */ + public function getId(): int { + return 8; + } + + /** + * @return array available options of the data source + */ + public function getTemplate(): array { + $template = []; + $template[] = [ + 'id' => 'token', + 'name' => $this->l10n->t('Personal access token'), + 'placeholder' => $this->l10n->t('optional') + ]; + return $template; + } + + /** + * Read the Data + * @param $option + * @return array available options of the data source + */ + public function readData($option): array { + $header = [ + $this->l10n->t('Repository'), + $this->l10n->t('Type'), + $this->l10n->t('Number'), + $this->l10n->t('Created'), + $this->l10n->t('Completed'), + $this->l10n->t('Days'), + $this->l10n->t('SLA met') + ]; + $data = []; + + foreach ($this->repositories as $repo) { + // Issues + $issuesUrl = 'https://api.github.com/repos/' . $repo . '/issues?state=all&per_page=100'; + $curlResult = $this->getCurlData($issuesUrl, $option); + if ($curlResult['http_code'] < 200 || $curlResult['http_code'] >= 300) { + return [ + 'header' => [], + 'dimensions' => [], + 'data' => $curlResult['http_code'] === 403 ? 'Rate limit exceeded' : [], + 'rawdata' => $curlResult, + 'error' => 'HTTP response code: ' . $curlResult['http_code'], + ]; + } + + foreach ($curlResult['data'] as $issue) { + if (isset($issue['pull_request'])) { + // skip pull requests in issue endpoint + continue; + } + if (in_array($issue['user']['login'], $this->excludedAuthors, true)) { + continue; + } + $triagedAt = ''; + $eventsUrl = 'https://api.github.com/repos/' . $repo . '/issues/' . $issue['number'] . '/events'; + $eventsCurl = $this->getCurlData($eventsUrl, $option); + if ($eventsCurl['http_code'] >= 200 && $eventsCurl['http_code'] < 300) { + foreach ($eventsCurl['data'] as $event) { + if ($event['event'] === 'labeled' && isset($event['label']['name']) && $event['label']['name'] === '1. to develop') { + $triagedAt = $event['created_at']; + break; + } + } + } + $days = $this->daysBetween($issue['created_at'], $triagedAt ?: date(DATE_ATOM)); + $slaMet = $triagedAt !== '' && $days <= 14; + $data[] = [$repo, 'issue', (int)$issue['number'], $issue['created_at'], $triagedAt, $days, $slaMet ? 1 : 0]; + } + + // Pull requests + $pullsUrl = 'https://api.github.com/repos/' . $repo . '/pulls?state=all&per_page=100'; + $pullsCurl = $this->getCurlData($pullsUrl, $option); + if ($pullsCurl['http_code'] < 200 || $pullsCurl['http_code'] >= 300) { + return [ + 'header' => [], + 'dimensions' => [], + 'data' => $pullsCurl['http_code'] === 403 ? 'Rate limit exceeded' : [], + 'rawdata' => $pullsCurl, + 'error' => 'HTTP response code: ' . $pullsCurl['http_code'], + ]; + } + + foreach ($pullsCurl['data'] as $pr) { + if (in_array($pr['user']['login'], $this->excludedAuthors, true)) { + continue; + } + $mergedAt = $pr['merged_at'] ?? ''; + $days = $this->daysBetween($pr['created_at'], $mergedAt ?: date(DATE_ATOM)); + $slaMet = $mergedAt !== '' && $days <= 14; + $data[] = [$repo, 'pr', (int)$pr['number'], $pr['created_at'], $mergedAt, $days, $slaMet ? 1 : 0]; + } + } + + return [ + 'header' => $header, + 'dimensions' => array_slice($header, 0, count($header) - 1), + 'data' => $data, + 'rawdata' => [], + 'error' => 0, + ]; + } + + protected function getCurlData($url, $option): array { + $ch = curl_init(); + if ($ch !== false) { + curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false); + curl_setopt($ch, CURLOPT_HEADER, false); + curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true); + curl_setopt($ch, CURLOPT_URL, $url); + curl_setopt($ch, CURLOPT_REFERER, $url); + curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); + curl_setopt($ch, CURLOPT_USERAGENT, 'Mozilla/5.0'); + if (isset($option['token']) && $option['token'] !== '') { + $headers = [ + 'Authorization: token ' . $option['token'], + 'User-Agent: AnalyticsApp', + 'Accept: application/vnd.github.v3+json' + ]; + curl_setopt($ch, CURLOPT_HTTPHEADER, $headers); + } + $curlResult = curl_exec($ch); + $http_code = curl_getinfo($ch, CURLINFO_HTTP_CODE); + curl_close($ch); + } else { + $curlResult = ''; + $http_code = 500; + } + $curlResult = json_decode($curlResult, true); + return ['data' => $curlResult, 'http_code' => $http_code]; + } + + private function daysBetween(string $from, string $to): int { + $start = new \DateTime($from); + $end = new \DateTime($to); + return $start->diff($end)->days; + } +} diff --git a/tests/Datasource/GithubCommunitySlaTest.php b/tests/Datasource/GithubCommunitySlaTest.php new file mode 100644 index 00000000..9bfbf4d6 --- /dev/null +++ b/tests/Datasource/GithubCommunitySlaTest.php @@ -0,0 +1,41 @@ + [ + ['event' => 'labeled', 'label' => ['name' => '1. to develop'], 'created_at' => '2025-01-05T00:00:00Z'] + ], 'http_code' => 200]; + } + if (str_contains($url, '/issues?')) { + return ['data' => [ + ['number' => 1, 'created_at' => '2025-01-01T00:00:00Z', 'user' => ['login' => 'communityUser']], + ['number' => 2, 'created_at' => '2025-01-05T00:00:00Z', 'user' => ['login' => 'employee1']] + ], 'http_code' => 200]; + } + if (str_contains($url, '/pulls?')) { + return ['data' => [ + ['number' => 10, 'created_at' => '2025-01-01T00:00:00Z', 'merged_at' => '2025-01-20T00:00:00Z', 'user' => ['login' => 'communityUser']], + ['number' => 11, 'created_at' => '2025-01-02T00:00:00Z', 'merged_at' => null, 'user' => ['login' => 'employee1']] + ], 'http_code' => 200]; + } + return ['data' => [], 'http_code' => 200]; + } + }; + + $result = $datasource->readData([]); + $this->assertSame(['owner/repo1','issue',1,'2025-01-01T00:00:00Z','2025-01-05T00:00:00Z',4,1], $result['data'][0]); + $this->assertSame(['owner/repo1','pr',10,'2025-01-01T00:00:00Z','2025-01-20T00:00:00Z',19,0], $result['data'][1]); + } +} From d5eb751872faa28a333d0211382e159f09a136ee Mon Sep 17 00:00:00 2001 From: Rello Date: Wed, 20 Aug 2025 18:24:51 +0700 Subject: [PATCH 2/8] Refine GitHub SLA datasource triage detection --- CHANGELOG.md | 3 +++ lib/Datasource/GithubCommunitySla.php | 23 ++++++++++++++++++--- tests/Datasource/GithubCommunitySlaTest.php | 14 +++++++------ 3 files changed, 31 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aaecb0cf..9bb1f702 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,9 @@ - Warn about unsaved changes before leaving a report - GitHub Community SLA data source +### Changed +- Detect triage when removing "0. Needs triage" label and support updated-since filtering in GitHub Community SLA data source + ### Fixed - Fix PHP 8.4 deprecation warnings #534 @[robertoschwald](https://github.com/robertoschwald) diff --git a/lib/Datasource/GithubCommunitySla.php b/lib/Datasource/GithubCommunitySla.php index 1a5eff13..bbd118e2 100644 --- a/lib/Datasource/GithubCommunitySla.php +++ b/lib/Datasource/GithubCommunitySla.php @@ -65,6 +65,11 @@ public function getTemplate(): array { 'name' => $this->l10n->t('Personal access token'), 'placeholder' => $this->l10n->t('optional') ]; + $template[] = [ + 'id' => 'days', + 'name' => $this->l10n->t('Updated since (days)'), + 'placeholder' => '30' + ]; return $template; } @@ -85,9 +90,12 @@ public function readData($option): array { ]; $data = []; + $daysFilter = isset($option['days']) && (int)$option['days'] > 0 ? (int)$option['days'] : 30; + $sinceDate = date(DATE_ATOM, time() - ($daysFilter * 86400)); + foreach ($this->repositories as $repo) { // Issues - $issuesUrl = 'https://api.github.com/repos/' . $repo . '/issues?state=all&per_page=100'; + $issuesUrl = 'https://api.github.com/repos/' . $repo . '/issues?state=all&per_page=100&since=' . $sinceDate; $curlResult = $this->getCurlData($issuesUrl, $option); if ($curlResult['http_code'] < 200 || $curlResult['http_code'] >= 300) { return [ @@ -100,6 +108,9 @@ public function readData($option): array { } foreach ($curlResult['data'] as $issue) { + if (isset($issue['updated_at']) && $issue['updated_at'] < $sinceDate) { + continue; + } if (isset($issue['pull_request'])) { // skip pull requests in issue endpoint continue; @@ -112,7 +123,10 @@ public function readData($option): array { $eventsCurl = $this->getCurlData($eventsUrl, $option); if ($eventsCurl['http_code'] >= 200 && $eventsCurl['http_code'] < 300) { foreach ($eventsCurl['data'] as $event) { - if ($event['event'] === 'labeled' && isset($event['label']['name']) && $event['label']['name'] === '1. to develop') { + if ( + ($event['event'] === 'unlabeled' && isset($event['label']['name']) && $event['label']['name'] === '0. Needs triage') || + ($event['event'] === 'labeled' && isset($event['label']['name']) && $event['label']['name'] === '1. to develop') + ) { $triagedAt = $event['created_at']; break; } @@ -124,7 +138,7 @@ public function readData($option): array { } // Pull requests - $pullsUrl = 'https://api.github.com/repos/' . $repo . '/pulls?state=all&per_page=100'; + $pullsUrl = 'https://api.github.com/repos/' . $repo . '/pulls?state=all&per_page=100&sort=updated&direction=desc'; $pullsCurl = $this->getCurlData($pullsUrl, $option); if ($pullsCurl['http_code'] < 200 || $pullsCurl['http_code'] >= 300) { return [ @@ -137,6 +151,9 @@ public function readData($option): array { } foreach ($pullsCurl['data'] as $pr) { + if (isset($pr['updated_at']) && $pr['updated_at'] < $sinceDate) { + continue; + } if (in_array($pr['user']['login'], $this->excludedAuthors, true)) { continue; } diff --git a/tests/Datasource/GithubCommunitySlaTest.php b/tests/Datasource/GithubCommunitySlaTest.php index 9bfbf4d6..298497b4 100644 --- a/tests/Datasource/GithubCommunitySlaTest.php +++ b/tests/Datasource/GithubCommunitySlaTest.php @@ -15,26 +15,28 @@ public function testReadDataCalculatesSla(): void { protected function getCurlData($url, $option): array { if (str_contains($url, '/issues/1/events')) { return ['data' => [ - ['event' => 'labeled', 'label' => ['name' => '1. to develop'], 'created_at' => '2025-01-05T00:00:00Z'] + ['event' => 'unlabeled', 'label' => ['name' => '0. Needs triage'], 'created_at' => '2025-01-05T00:00:00Z'] ], 'http_code' => 200]; } if (str_contains($url, '/issues?')) { return ['data' => [ - ['number' => 1, 'created_at' => '2025-01-01T00:00:00Z', 'user' => ['login' => 'communityUser']], - ['number' => 2, 'created_at' => '2025-01-05T00:00:00Z', 'user' => ['login' => 'employee1']] + ['number' => 1, 'created_at' => '2025-01-01T00:00:00Z', 'updated_at' => '2025-01-05T00:00:00Z', 'user' => ['login' => 'communityUser']], + ['number' => 2, 'created_at' => '2025-01-05T00:00:00Z', 'updated_at' => '2025-01-06T00:00:00Z', 'user' => ['login' => 'employee1']] ], 'http_code' => 200]; } if (str_contains($url, '/pulls?')) { return ['data' => [ - ['number' => 10, 'created_at' => '2025-01-01T00:00:00Z', 'merged_at' => '2025-01-20T00:00:00Z', 'user' => ['login' => 'communityUser']], - ['number' => 11, 'created_at' => '2025-01-02T00:00:00Z', 'merged_at' => null, 'user' => ['login' => 'employee1']] + ['number' => 10, 'created_at' => '2025-01-01T00:00:00Z', 'merged_at' => '2025-01-20T00:00:00Z', 'updated_at' => '2025-01-20T00:00:00Z', 'user' => ['login' => 'communityUser']], + ['number' => 11, 'created_at' => '2025-01-02T00:00:00Z', 'merged_at' => null, 'updated_at' => '2025-01-02T00:00:00Z', 'user' => ['login' => 'employee1']], + ['number' => 12, 'created_at' => '2000-01-01T00:00:00Z', 'merged_at' => '2000-01-02T00:00:00Z', 'updated_at' => '2000-01-02T00:00:00Z', 'user' => ['login' => 'communityUser']] ], 'http_code' => 200]; } return ['data' => [], 'http_code' => 200]; } }; - $result = $datasource->readData([]); + $result = $datasource->readData(['days' => 30]); + $this->assertCount(2, $result['data']); $this->assertSame(['owner/repo1','issue',1,'2025-01-01T00:00:00Z','2025-01-05T00:00:00Z',4,1], $result['data'][0]); $this->assertSame(['owner/repo1','pr',10,'2025-01-01T00:00:00Z','2025-01-20T00:00:00Z',19,0], $result['data'][1]); } From c4f7e63dd3ac00745ec5c1376f23632fee418317 Mon Sep 17 00:00:00 2001 From: Rello Date: Sat, 23 Aug 2025 15:51:34 +0700 Subject: [PATCH 3/8] Use GraphQL for GitHub SLA queries --- CHANGELOG.md | 1 + lib/Datasource/GithubCommunitySla.php | 117 +++++++++++--------- tests/Datasource/GithubCommunitySlaTest.php | 78 +++++++++---- 3 files changed, 123 insertions(+), 73 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9bb1f702..87230032 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ ### Changed - Detect triage when removing "0. Needs triage" label and support updated-since filtering in GitHub Community SLA data source +- Use GitHub GraphQL API for more efficient Community SLA queries ### Fixed - Fix PHP 8.4 deprecation warnings #534 @[robertoschwald](https://github.com/robertoschwald) diff --git a/lib/Datasource/GithubCommunitySla.php b/lib/Datasource/GithubCommunitySla.php index bbd118e2..20932e70 100644 --- a/lib/Datasource/GithubCommunitySla.php +++ b/lib/Datasource/GithubCommunitySla.php @@ -94,10 +94,40 @@ public function readData($option): array { $sinceDate = date(DATE_ATOM, time() - ($daysFilter * 86400)); foreach ($this->repositories as $repo) { - // Issues - $issuesUrl = 'https://api.github.com/repos/' . $repo . '/issues?state=all&per_page=100&since=' . $sinceDate; - $curlResult = $this->getCurlData($issuesUrl, $option); - if ($curlResult['http_code'] < 200 || $curlResult['http_code'] >= 300) { + [$owner, $name] = explode('/', $repo, 2); + $query = <<<'GRAPHQL' +query($owner: String!, $name: String!) { + repository(owner: $owner, name: $name) { + issues(first: 100, orderBy: {field: UPDATED_AT, direction: DESC}, states: [OPEN, CLOSED]) { + nodes { + number + createdAt + updatedAt + author { login } + timelineItems(first: 100, itemTypes: [LABELED_EVENT, UNLABELED_EVENT]) { + nodes { + __typename + ... on LabeledEvent { createdAt label { name } } + ... on UnlabeledEvent { createdAt label { name } } + } + } + } + } + pullRequests(first: 100, orderBy: {field: UPDATED_AT, direction: DESC}, states: [OPEN, MERGED, CLOSED]) { + nodes { + number + createdAt + mergedAt + updatedAt + author { login } + } + } + } +} +GRAPHQL; + + $curlResult = $this->getGraphqlData($query, ['owner' => $owner, 'name' => $name], $option); + if ($curlResult['http_code'] < 200 || $curlResult['http_code'] >= 300 || isset($curlResult['data']['errors'])) { return [ 'header' => [], 'dimensions' => [], @@ -107,60 +137,41 @@ public function readData($option): array { ]; } - foreach ($curlResult['data'] as $issue) { - if (isset($issue['updated_at']) && $issue['updated_at'] < $sinceDate) { - continue; - } - if (isset($issue['pull_request'])) { - // skip pull requests in issue endpoint + $repoData = $curlResult['data']['data']['repository']; + + foreach ($repoData['issues']['nodes'] as $issue) { + if ($issue['updatedAt'] < $sinceDate) { continue; } - if (in_array($issue['user']['login'], $this->excludedAuthors, true)) { + if (in_array($issue['author']['login'], $this->excludedAuthors, true)) { continue; } $triagedAt = ''; - $eventsUrl = 'https://api.github.com/repos/' . $repo . '/issues/' . $issue['number'] . '/events'; - $eventsCurl = $this->getCurlData($eventsUrl, $option); - if ($eventsCurl['http_code'] >= 200 && $eventsCurl['http_code'] < 300) { - foreach ($eventsCurl['data'] as $event) { - if ( - ($event['event'] === 'unlabeled' && isset($event['label']['name']) && $event['label']['name'] === '0. Needs triage') || - ($event['event'] === 'labeled' && isset($event['label']['name']) && $event['label']['name'] === '1. to develop') - ) { - $triagedAt = $event['created_at']; - break; - } + foreach ($issue['timelineItems']['nodes'] as $event) { + if ( + ($event['__typename'] === 'UnlabeledEvent' && isset($event['label']['name']) && $event['label']['name'] === '0. Needs triage') || + ($event['__typename'] === 'LabeledEvent' && isset($event['label']['name']) && $event['label']['name'] === '1. to develop') + ) { + $triagedAt = $event['createdAt']; + break; } } - $days = $this->daysBetween($issue['created_at'], $triagedAt ?: date(DATE_ATOM)); + $days = $this->daysBetween($issue['createdAt'], $triagedAt ?: date(DATE_ATOM)); $slaMet = $triagedAt !== '' && $days <= 14; - $data[] = [$repo, 'issue', (int)$issue['number'], $issue['created_at'], $triagedAt, $days, $slaMet ? 1 : 0]; + $data[] = [$repo, 'issue', (int)$issue['number'], $issue['createdAt'], $triagedAt, $days, $slaMet ? 1 : 0]; } - // Pull requests - $pullsUrl = 'https://api.github.com/repos/' . $repo . '/pulls?state=all&per_page=100&sort=updated&direction=desc'; - $pullsCurl = $this->getCurlData($pullsUrl, $option); - if ($pullsCurl['http_code'] < 200 || $pullsCurl['http_code'] >= 300) { - return [ - 'header' => [], - 'dimensions' => [], - 'data' => $pullsCurl['http_code'] === 403 ? 'Rate limit exceeded' : [], - 'rawdata' => $pullsCurl, - 'error' => 'HTTP response code: ' . $pullsCurl['http_code'], - ]; - } - - foreach ($pullsCurl['data'] as $pr) { - if (isset($pr['updated_at']) && $pr['updated_at'] < $sinceDate) { + foreach ($repoData['pullRequests']['nodes'] as $pr) { + if ($pr['updatedAt'] < $sinceDate) { continue; } - if (in_array($pr['user']['login'], $this->excludedAuthors, true)) { + if (in_array($pr['author']['login'], $this->excludedAuthors, true)) { continue; } - $mergedAt = $pr['merged_at'] ?? ''; - $days = $this->daysBetween($pr['created_at'], $mergedAt ?: date(DATE_ATOM)); + $mergedAt = $pr['mergedAt'] ?? ''; + $days = $this->daysBetween($pr['createdAt'], $mergedAt ?: date(DATE_ATOM)); $slaMet = $mergedAt !== '' && $days <= 14; - $data[] = [$repo, 'pr', (int)$pr['number'], $pr['created_at'], $mergedAt, $days, $slaMet ? 1 : 0]; + $data[] = [$repo, 'pr', (int)$pr['number'], $pr['createdAt'], $mergedAt, $days, $slaMet ? 1 : 0]; } } @@ -173,24 +184,24 @@ public function readData($option): array { ]; } - protected function getCurlData($url, $option): array { - $ch = curl_init(); + protected function getGraphqlData(string $query, array $variables, array $option): array { + $ch = curl_init('https://api.github.com/graphql'); if ($ch !== false) { curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false); curl_setopt($ch, CURLOPT_HEADER, false); curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true); - curl_setopt($ch, CURLOPT_URL, $url); - curl_setopt($ch, CURLOPT_REFERER, $url); curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); curl_setopt($ch, CURLOPT_USERAGENT, 'Mozilla/5.0'); + curl_setopt($ch, CURLOPT_POST, true); + curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode(['query' => $query, 'variables' => $variables])); + $headers = [ + 'Content-Type: application/json', + 'User-Agent: AnalyticsApp' + ]; if (isset($option['token']) && $option['token'] !== '') { - $headers = [ - 'Authorization: token ' . $option['token'], - 'User-Agent: AnalyticsApp', - 'Accept: application/vnd.github.v3+json' - ]; - curl_setopt($ch, CURLOPT_HTTPHEADER, $headers); + $headers[] = 'Authorization: bearer ' . $option['token']; } + curl_setopt($ch, CURLOPT_HTTPHEADER, $headers); $curlResult = curl_exec($ch); $http_code = curl_getinfo($ch, CURLINFO_HTTP_CODE); curl_close($ch); diff --git a/tests/Datasource/GithubCommunitySlaTest.php b/tests/Datasource/GithubCommunitySlaTest.php index 298497b4..289e5929 100644 --- a/tests/Datasource/GithubCommunitySlaTest.php +++ b/tests/Datasource/GithubCommunitySlaTest.php @@ -12,26 +12,64 @@ public function testReadDataCalculatesSla(): void { protected array $repositories = ['owner/repo1']; protected array $excludedAuthors = ['employee1']; - protected function getCurlData($url, $option): array { - if (str_contains($url, '/issues/1/events')) { - return ['data' => [ - ['event' => 'unlabeled', 'label' => ['name' => '0. Needs triage'], 'created_at' => '2025-01-05T00:00:00Z'] - ], 'http_code' => 200]; - } - if (str_contains($url, '/issues?')) { - return ['data' => [ - ['number' => 1, 'created_at' => '2025-01-01T00:00:00Z', 'updated_at' => '2025-01-05T00:00:00Z', 'user' => ['login' => 'communityUser']], - ['number' => 2, 'created_at' => '2025-01-05T00:00:00Z', 'updated_at' => '2025-01-06T00:00:00Z', 'user' => ['login' => 'employee1']] - ], 'http_code' => 200]; - } - if (str_contains($url, '/pulls?')) { - return ['data' => [ - ['number' => 10, 'created_at' => '2025-01-01T00:00:00Z', 'merged_at' => '2025-01-20T00:00:00Z', 'updated_at' => '2025-01-20T00:00:00Z', 'user' => ['login' => 'communityUser']], - ['number' => 11, 'created_at' => '2025-01-02T00:00:00Z', 'merged_at' => null, 'updated_at' => '2025-01-02T00:00:00Z', 'user' => ['login' => 'employee1']], - ['number' => 12, 'created_at' => '2000-01-01T00:00:00Z', 'merged_at' => '2000-01-02T00:00:00Z', 'updated_at' => '2000-01-02T00:00:00Z', 'user' => ['login' => 'communityUser']] - ], 'http_code' => 200]; - } - return ['data' => [], 'http_code' => 200]; + protected function getGraphqlData(string $query, array $variables, array $option): array { + return ['data' => [ + 'data' => [ + 'repository' => [ + 'issues' => [ + 'nodes' => [ + [ + 'number' => 1, + 'createdAt' => '2025-01-01T00:00:00Z', + 'updatedAt' => '2025-01-05T00:00:00Z', + 'author' => ['login' => 'communityUser'], + 'timelineItems' => [ + 'nodes' => [ + [ + '__typename' => 'UnlabeledEvent', + 'label' => ['name' => '0. Needs triage'], + 'createdAt' => '2025-01-05T00:00:00Z' + ] + ] + ] + ], + [ + 'number' => 2, + 'createdAt' => '2025-01-05T00:00:00Z', + 'updatedAt' => '2025-01-06T00:00:00Z', + 'author' => ['login' => 'employee1'], + 'timelineItems' => ['nodes' => []] + ] + ] + ], + 'pullRequests' => [ + 'nodes' => [ + [ + 'number' => 10, + 'createdAt' => '2025-01-01T00:00:00Z', + 'mergedAt' => '2025-01-20T00:00:00Z', + 'updatedAt' => '2025-01-20T00:00:00Z', + 'author' => ['login' => 'communityUser'] + ], + [ + 'number' => 11, + 'createdAt' => '2025-01-02T00:00:00Z', + 'mergedAt' => null, + 'updatedAt' => '2025-01-02T00:00:00Z', + 'author' => ['login' => 'employee1'] + ], + [ + 'number' => 12, + 'createdAt' => '2000-01-01T00:00:00Z', + 'mergedAt' => '2000-01-02T00:00:00Z', + 'updatedAt' => '2000-01-02T00:00:00Z', + 'author' => ['login' => 'communityUser'] + ] + ] + ] + ] + ] + ], 'http_code' => 200]; } }; From 7ef01b9fd9f1cb67b67cfa048f9d9fce075ce214 Mon Sep 17 00:00:00 2001 From: Rello Date: Sat, 23 Aug 2025 15:56:47 +0700 Subject: [PATCH 4/8] Register GitHub Community SLA datasource --- CHANGELOG.md | 1 + lib/Controller/DatasourceController.php | 40 ++++++++++++++----------- lib/Datasource/GithubCommunitySla.php | 2 +- 3 files changed, 25 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 87230032..3ae91f28 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ ### Changed - Detect triage when removing "0. Needs triage" label and support updated-since filtering in GitHub Community SLA data source - Use GitHub GraphQL API for more efficient Community SLA queries +- Register GitHub Community SLA data source with ID 9 ### Fixed - Fix PHP 8.4 deprecation warnings #534 @[robertoschwald](https://github.com/robertoschwald) diff --git a/lib/Controller/DatasourceController.php b/lib/Controller/DatasourceController.php index 93dc3e49..f0daf054 100644 --- a/lib/Controller/DatasourceController.php +++ b/lib/Controller/DatasourceController.php @@ -12,6 +12,7 @@ use OCA\Analytics\Datasource\ExternalCsv; use OCA\Analytics\Datasource\ExternalJson; use OCA\Analytics\Datasource\Github; +use OCA\Analytics\Datasource\GithubCommunitySla; use OCA\Analytics\Datasource\LocalCsv; use OCA\Analytics\Datasource\LocalSpreadsheet; use OCA\Analytics\Datasource\LocalJson; @@ -26,7 +27,8 @@ class DatasourceController extends Controller { private $logger; - private $GithubService; + private $GithubService; + private $GithubCommunitySlaService; private $ExternalCsvService; private $RegexService; private $ExternalJsonService; @@ -47,16 +49,18 @@ class DatasourceController extends Controller { const DATASET_TYPE_REGEX = 5; const DATASET_TYPE_EXTERNAL_JSON = 6; const DATASET_TYPE_LOCAL_SPREADSHEET = 7; - const DATASET_TYPE_LOCAL_JSON = 8; + const DATASET_TYPE_LOCAL_JSON = 8; + const DATASET_TYPE_GITHUB_COMMUNITY_SLA = 9; public function __construct( string $appName, IRequest $request, LoggerInterface $logger, - Github $GithubService, - LocalCsv $LocalCsvService, - Regex $RegexService, - ExternalJson $ExternalJsonService, + Github $GithubService, + GithubCommunitySla $GithubCommunitySlaService, + LocalCsv $LocalCsvService, + Regex $RegexService, + ExternalJson $ExternalJsonService, LocalJson $LocalJsonService, ExternalCsv $ExternalCsvService, LocalSpreadsheet $LocalSpreadsheetService, @@ -67,8 +71,9 @@ public function __construct( parent::__construct($appName, $request); $this->logger = $logger; $this->ExternalCsvService = $ExternalCsvService; - $this->GithubService = $GithubService; - $this->RegexService = $RegexService; + $this->GithubService = $GithubService; + $this->GithubCommunitySlaService = $GithubCommunitySlaService; + $this->RegexService = $RegexService; $this->LocalCsvService = $LocalCsvService; $this->ExternalJsonService = $ExternalJsonService; $this->LocalJsonService = $LocalJsonService; @@ -211,15 +216,16 @@ private function getDatasources(?int $datasourceType = null) { */ private function getOwnDatasources(?int $datasourceType = null) { $dataSources = []; - $serviceMapping = [ - self::DATASET_TYPE_GIT => $this->GithubService, - self::DATASET_TYPE_LOCAL_CSV => $this->LocalCsvService, - self::DATASET_TYPE_LOCAL_SPREADSHEET => $this->LocalSpreadsheetService, - self::DATASET_TYPE_EXTERNAL_CSV => $this->ExternalCsvService, - self::DATASET_TYPE_REGEX => $this->RegexService, - self::DATASET_TYPE_EXTERNAL_JSON => $this->ExternalJsonService, - self::DATASET_TYPE_LOCAL_JSON => $this->LocalJsonService, - ]; + $serviceMapping = [ + self::DATASET_TYPE_GIT => $this->GithubService, + self::DATASET_TYPE_GITHUB_COMMUNITY_SLA => $this->GithubCommunitySlaService, + self::DATASET_TYPE_LOCAL_CSV => $this->LocalCsvService, + self::DATASET_TYPE_LOCAL_SPREADSHEET => $this->LocalSpreadsheetService, + self::DATASET_TYPE_EXTERNAL_CSV => $this->ExternalCsvService, + self::DATASET_TYPE_REGEX => $this->RegexService, + self::DATASET_TYPE_EXTERNAL_JSON => $this->ExternalJsonService, + self::DATASET_TYPE_LOCAL_JSON => $this->LocalJsonService, + ]; if ($datasourceType !== null && isset($serviceMapping[$datasourceType])) { $dataSources[$datasourceType] = $serviceMapping[$datasourceType]; diff --git a/lib/Datasource/GithubCommunitySla.php b/lib/Datasource/GithubCommunitySla.php index 20932e70..e24f3f1f 100644 --- a/lib/Datasource/GithubCommunitySla.php +++ b/lib/Datasource/GithubCommunitySla.php @@ -52,7 +52,7 @@ public function getName(): string { * @return int digit unique data source id */ public function getId(): int { - return 8; + return 9; } /** From c5b6bb5aa917b5b557e043434d72de40fe3e3a99 Mon Sep 17 00:00:00 2001 From: Rello Date: Sat, 23 Aug 2025 17:56:32 +0700 Subject: [PATCH 5/8] Add counter column and close date handling in GitHub Community SLA --- CHANGELOG.md | 1 + lib/Datasource/GithubCommunitySla.php | 16 +++++++++------- tests/Datasource/GithubCommunitySlaTest.php | 9 ++++++--- 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ae91f28..3202b85a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ - Detect triage when removing "0. Needs triage" label and support updated-since filtering in GitHub Community SLA data source - Use GitHub GraphQL API for more efficient Community SLA queries - Register GitHub Community SLA data source with ID 9 +- Include counter column and use merge/close dates for PR SLA calculation in GitHub Community SLA data source ### Fixed - Fix PHP 8.4 deprecation warnings #534 @[robertoschwald](https://github.com/robertoschwald) diff --git a/lib/Datasource/GithubCommunitySla.php b/lib/Datasource/GithubCommunitySla.php index e24f3f1f..ddda3843 100644 --- a/lib/Datasource/GithubCommunitySla.php +++ b/lib/Datasource/GithubCommunitySla.php @@ -86,7 +86,8 @@ public function readData($option): array { $this->l10n->t('Created'), $this->l10n->t('Completed'), $this->l10n->t('Days'), - $this->l10n->t('SLA met') + $this->l10n->t('SLA met'), + $this->l10n->t('Counter') ]; $data = []; @@ -118,6 +119,7 @@ public function readData($option): array { number createdAt mergedAt + closedAt updatedAt author { login } } @@ -158,7 +160,7 @@ public function readData($option): array { } $days = $this->daysBetween($issue['createdAt'], $triagedAt ?: date(DATE_ATOM)); $slaMet = $triagedAt !== '' && $days <= 14; - $data[] = [$repo, 'issue', (int)$issue['number'], $issue['createdAt'], $triagedAt, $days, $slaMet ? 1 : 0]; + $data[] = [$repo, 'issue', (int)$issue['number'], $issue['createdAt'], $triagedAt, $days, $slaMet ? 1 : 0, 1]; } foreach ($repoData['pullRequests']['nodes'] as $pr) { @@ -168,16 +170,16 @@ public function readData($option): array { if (in_array($pr['author']['login'], $this->excludedAuthors, true)) { continue; } - $mergedAt = $pr['mergedAt'] ?? ''; - $days = $this->daysBetween($pr['createdAt'], $mergedAt ?: date(DATE_ATOM)); - $slaMet = $mergedAt !== '' && $days <= 14; - $data[] = [$repo, 'pr', (int)$pr['number'], $pr['createdAt'], $mergedAt, $days, $slaMet ? 1 : 0]; + $completedAt = $pr['mergedAt'] ?? $pr['closedAt'] ?? ''; + $days = $this->daysBetween($pr['createdAt'], $completedAt ?: date(DATE_ATOM)); + $slaMet = $completedAt !== '' && $days <= 14; + $data[] = [$repo, 'pr', (int)$pr['number'], $pr['createdAt'], $completedAt, $days, $slaMet ? 1 : 0, 1]; } } return [ 'header' => $header, - 'dimensions' => array_slice($header, 0, count($header) - 1), + 'dimensions' => array_slice($header, 0, count($header) - 2), 'data' => $data, 'rawdata' => [], 'error' => 0, diff --git a/tests/Datasource/GithubCommunitySlaTest.php b/tests/Datasource/GithubCommunitySlaTest.php index 289e5929..e3f5ee97 100644 --- a/tests/Datasource/GithubCommunitySlaTest.php +++ b/tests/Datasource/GithubCommunitySlaTest.php @@ -47,7 +47,8 @@ protected function getGraphqlData(string $query, array $variables, array $option [ 'number' => 10, 'createdAt' => '2025-01-01T00:00:00Z', - 'mergedAt' => '2025-01-20T00:00:00Z', + 'mergedAt' => null, + 'closedAt' => '2025-01-20T00:00:00Z', 'updatedAt' => '2025-01-20T00:00:00Z', 'author' => ['login' => 'communityUser'] ], @@ -55,6 +56,7 @@ protected function getGraphqlData(string $query, array $variables, array $option 'number' => 11, 'createdAt' => '2025-01-02T00:00:00Z', 'mergedAt' => null, + 'closedAt' => null, 'updatedAt' => '2025-01-02T00:00:00Z', 'author' => ['login' => 'employee1'] ], @@ -62,6 +64,7 @@ protected function getGraphqlData(string $query, array $variables, array $option 'number' => 12, 'createdAt' => '2000-01-01T00:00:00Z', 'mergedAt' => '2000-01-02T00:00:00Z', + 'closedAt' => '2000-01-02T00:00:00Z', 'updatedAt' => '2000-01-02T00:00:00Z', 'author' => ['login' => 'communityUser'] ] @@ -75,7 +78,7 @@ protected function getGraphqlData(string $query, array $variables, array $option $result = $datasource->readData(['days' => 30]); $this->assertCount(2, $result['data']); - $this->assertSame(['owner/repo1','issue',1,'2025-01-01T00:00:00Z','2025-01-05T00:00:00Z',4,1], $result['data'][0]); - $this->assertSame(['owner/repo1','pr',10,'2025-01-01T00:00:00Z','2025-01-20T00:00:00Z',19,0], $result['data'][1]); + $this->assertSame(['owner/repo1','issue',1,'2025-01-01T00:00:00Z','2025-01-05T00:00:00Z',4,1,1], $result['data'][0]); + $this->assertSame(['owner/repo1','pr',10,'2025-01-01T00:00:00Z','2025-01-20T00:00:00Z',19,0,1], $result['data'][1]); } } From 6152bd3f31433c07a54dfccb7991c1fee095e3fb Mon Sep 17 00:00:00 2001 From: Rello Date: Sat, 23 Aug 2025 22:58:22 +0700 Subject: [PATCH 6/8] Expose SLA options for GitHub Community datasource --- CHANGELOG.md | 1 + lib/Datasource/GithubCommunitySla.php | 36 +++++++++++++++++---- tests/Datasource/GithubCommunitySlaTest.php | 10 +++--- 3 files changed, 37 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3202b85a..4dceaf80 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ - Cache report data in the browser (using ETag & If-None-Match) #535 - Warn about unsaved changes before leaving a report - GitHub Community SLA data source +- Configure excluded authors, repositories, and SLA days via GitHub Community SLA options ### Changed - Detect triage when removing "0. Needs triage" label and support updated-since filtering in GitHub Community SLA data source diff --git a/lib/Datasource/GithubCommunitySla.php b/lib/Datasource/GithubCommunitySla.php index ddda3843..9f8d3968 100644 --- a/lib/Datasource/GithubCommunitySla.php +++ b/lib/Datasource/GithubCommunitySla.php @@ -65,10 +65,27 @@ public function getTemplate(): array { 'name' => $this->l10n->t('Personal access token'), 'placeholder' => $this->l10n->t('optional') ]; + $template[] = [ + 'id' => 'repo', + 'name' => $this->l10n->t('Repositories'), + 'placeholder' => 'owner/repo1,owner/repo2' + ]; + $template[] = [ + 'id' => 'exclude', + 'name' => $this->l10n->t('Exclude authors'), + 'placeholder' => 'user1,user2' + ]; + $template[] = [ + 'id' => 'sla', + 'name' => $this->l10n->t('SLA days'), + 'placeholder' => '14', + 'type' => 'number' + ]; $template[] = [ 'id' => 'days', 'name' => $this->l10n->t('Updated since (days)'), - 'placeholder' => '30' + 'placeholder' => '30', + 'type' => 'number' ]; return $template; } @@ -91,10 +108,17 @@ public function readData($option): array { ]; $data = []; + $repositories = isset($option['repo']) && $option['repo'] !== '' + ? array_map('trim', explode(',', $option['repo'])) + : $this->repositories; + $excludedAuthors = isset($option['exclude']) && $option['exclude'] !== '' + ? array_map('trim', explode(',', $option['exclude'])) + : $this->excludedAuthors; + $slaDays = isset($option['sla']) && (int)$option['sla'] > 0 ? (int)$option['sla'] : 14; $daysFilter = isset($option['days']) && (int)$option['days'] > 0 ? (int)$option['days'] : 30; $sinceDate = date(DATE_ATOM, time() - ($daysFilter * 86400)); - foreach ($this->repositories as $repo) { + foreach ($repositories as $repo) { [$owner, $name] = explode('/', $repo, 2); $query = <<<'GRAPHQL' query($owner: String!, $name: String!) { @@ -145,7 +169,7 @@ public function readData($option): array { if ($issue['updatedAt'] < $sinceDate) { continue; } - if (in_array($issue['author']['login'], $this->excludedAuthors, true)) { + if (in_array($issue['author']['login'], $excludedAuthors, true)) { continue; } $triagedAt = ''; @@ -159,7 +183,7 @@ public function readData($option): array { } } $days = $this->daysBetween($issue['createdAt'], $triagedAt ?: date(DATE_ATOM)); - $slaMet = $triagedAt !== '' && $days <= 14; + $slaMet = $triagedAt !== '' && $days <= $slaDays; $data[] = [$repo, 'issue', (int)$issue['number'], $issue['createdAt'], $triagedAt, $days, $slaMet ? 1 : 0, 1]; } @@ -167,12 +191,12 @@ public function readData($option): array { if ($pr['updatedAt'] < $sinceDate) { continue; } - if (in_array($pr['author']['login'], $this->excludedAuthors, true)) { + if (in_array($pr['author']['login'], $excludedAuthors, true)) { continue; } $completedAt = $pr['mergedAt'] ?? $pr['closedAt'] ?? ''; $days = $this->daysBetween($pr['createdAt'], $completedAt ?: date(DATE_ATOM)); - $slaMet = $completedAt !== '' && $days <= 14; + $slaMet = $completedAt !== '' && $days <= $slaDays; $data[] = [$repo, 'pr', (int)$pr['number'], $pr['createdAt'], $completedAt, $days, $slaMet ? 1 : 0, 1]; } } diff --git a/tests/Datasource/GithubCommunitySlaTest.php b/tests/Datasource/GithubCommunitySlaTest.php index e3f5ee97..e5626410 100644 --- a/tests/Datasource/GithubCommunitySlaTest.php +++ b/tests/Datasource/GithubCommunitySlaTest.php @@ -9,9 +9,6 @@ class GithubCommunitySlaTest extends TestCase { public function testReadDataCalculatesSla(): void { $datasource = new class(new FakeL10N(), new NullLogger()) extends GithubCommunitySla { - protected array $repositories = ['owner/repo1']; - protected array $excludedAuthors = ['employee1']; - protected function getGraphqlData(string $query, array $variables, array $option): array { return ['data' => [ 'data' => [ @@ -76,7 +73,12 @@ protected function getGraphqlData(string $query, array $variables, array $option } }; - $result = $datasource->readData(['days' => 30]); + $result = $datasource->readData([ + 'days' => 30, + 'repo' => 'owner/repo1', + 'exclude' => 'employee1', + 'sla' => 14, + ]); $this->assertCount(2, $result['data']); $this->assertSame(['owner/repo1','issue',1,'2025-01-01T00:00:00Z','2025-01-05T00:00:00Z',4,1,1], $result['data'][0]); $this->assertSame(['owner/repo1','pr',10,'2025-01-01T00:00:00Z','2025-01-20T00:00:00Z',19,0,1], $result['data'][1]); From ea6191b58fc5fc0c9571a5b5b6692bdfa9dc8a93 Mon Sep 17 00:00:00 2001 From: Rello Date: Sun, 24 Aug 2025 12:13:40 +0700 Subject: [PATCH 7/8] Refine GitHub Community SLA calculation --- CHANGELOG.md | 2 +- lib/Datasource/GithubCommunitySla.php | 24 +++++------ tests/Datasource/GithubCommunitySlaTest.php | 44 +++++++++++++++------ 3 files changed, 46 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4dceaf80..79ca19cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ - Configure excluded authors, repositories, and SLA days via GitHub Community SLA options ### Changed -- Detect triage when removing "0. Needs triage" label and support updated-since filtering in GitHub Community SLA data source +- Treat issues as triaged when the "0. Needs triage" label is removed or the issue is closed and evaluate open items against the current date in GitHub Community SLA data source - Use GitHub GraphQL API for more efficient Community SLA queries - Register GitHub Community SLA data source with ID 9 - Include counter column and use merge/close dates for PR SLA calculation in GitHub Community SLA data source diff --git a/lib/Datasource/GithubCommunitySla.php b/lib/Datasource/GithubCommunitySla.php index 9f8d3968..27d33918 100644 --- a/lib/Datasource/GithubCommunitySla.php +++ b/lib/Datasource/GithubCommunitySla.php @@ -128,11 +128,11 @@ public function readData($option): array { number createdAt updatedAt + closedAt author { login } - timelineItems(first: 100, itemTypes: [LABELED_EVENT, UNLABELED_EVENT]) { + timelineItems(first: 100, itemTypes: [UNLABELED_EVENT]) { nodes { __typename - ... on LabeledEvent { createdAt label { name } } ... on UnlabeledEvent { createdAt label { name } } } } @@ -172,19 +172,19 @@ public function readData($option): array { if (in_array($issue['author']['login'], $excludedAuthors, true)) { continue; } - $triagedAt = ''; + $completedAt = ''; foreach ($issue['timelineItems']['nodes'] as $event) { - if ( - ($event['__typename'] === 'UnlabeledEvent' && isset($event['label']['name']) && $event['label']['name'] === '0. Needs triage') || - ($event['__typename'] === 'LabeledEvent' && isset($event['label']['name']) && $event['label']['name'] === '1. to develop') - ) { - $triagedAt = $event['createdAt']; + if ($event['__typename'] === 'UnlabeledEvent' && isset($event['label']['name']) && $event['label']['name'] === '0. Needs triage') { + $completedAt = $event['createdAt']; break; } } - $days = $this->daysBetween($issue['createdAt'], $triagedAt ?: date(DATE_ATOM)); - $slaMet = $triagedAt !== '' && $days <= $slaDays; - $data[] = [$repo, 'issue', (int)$issue['number'], $issue['createdAt'], $triagedAt, $days, $slaMet ? 1 : 0, 1]; + if ($completedAt === '' && isset($issue['closedAt']) && $issue['closedAt'] !== null) { + $completedAt = $issue['closedAt']; + } + $days = $this->daysBetween($issue['createdAt'], $completedAt ?: date(DATE_ATOM)); + $slaMet = $days <= $slaDays; + $data[] = [$repo, 'issue', (int)$issue['number'], $issue['createdAt'], $completedAt, $days, $slaMet ? 1 : 0, 1]; } foreach ($repoData['pullRequests']['nodes'] as $pr) { @@ -196,7 +196,7 @@ public function readData($option): array { } $completedAt = $pr['mergedAt'] ?? $pr['closedAt'] ?? ''; $days = $this->daysBetween($pr['createdAt'], $completedAt ?: date(DATE_ATOM)); - $slaMet = $completedAt !== '' && $days <= $slaDays; + $slaMet = $days <= $slaDays; $data[] = [$repo, 'pr', (int)$pr['number'], $pr['createdAt'], $completedAt, $days, $slaMet ? 1 : 0, 1]; } } diff --git a/tests/Datasource/GithubCommunitySlaTest.php b/tests/Datasource/GithubCommunitySlaTest.php index e5626410..19af3bd6 100644 --- a/tests/Datasource/GithubCommunitySlaTest.php +++ b/tests/Datasource/GithubCommunitySlaTest.php @@ -10,6 +10,7 @@ class GithubCommunitySlaTest extends TestCase { public function testReadDataCalculatesSla(): void { $datasource = new class(new FakeL10N(), new NullLogger()) extends GithubCommunitySla { protected function getGraphqlData(string $query, array $variables, array $option): array { + $recent = gmdate(DATE_ATOM, time() - 2 * 86400); return ['data' => [ 'data' => [ 'repository' => [ @@ -19,6 +20,7 @@ protected function getGraphqlData(string $query, array $variables, array $option 'number' => 1, 'createdAt' => '2025-01-01T00:00:00Z', 'updatedAt' => '2025-01-05T00:00:00Z', + 'closedAt' => null, 'author' => ['login' => 'communityUser'], 'timelineItems' => [ 'nodes' => [ @@ -32,8 +34,25 @@ protected function getGraphqlData(string $query, array $variables, array $option ], [ 'number' => 2, + 'createdAt' => '2025-01-01T00:00:00Z', + 'updatedAt' => '2025-01-03T00:00:00Z', + 'closedAt' => '2025-01-03T00:00:00Z', + 'author' => ['login' => 'communityUser'], + 'timelineItems' => ['nodes' => []] + ], + [ + 'number' => 3, + 'createdAt' => $recent, + 'updatedAt' => $recent, + 'closedAt' => null, + 'author' => ['login' => 'communityUser'], + 'timelineItems' => ['nodes' => []] + ], + [ + 'number' => 4, 'createdAt' => '2025-01-05T00:00:00Z', 'updatedAt' => '2025-01-06T00:00:00Z', + 'closedAt' => null, 'author' => ['login' => 'employee1'], 'timelineItems' => ['nodes' => []] ] @@ -56,14 +75,6 @@ protected function getGraphqlData(string $query, array $variables, array $option 'closedAt' => null, 'updatedAt' => '2025-01-02T00:00:00Z', 'author' => ['login' => 'employee1'] - ], - [ - 'number' => 12, - 'createdAt' => '2000-01-01T00:00:00Z', - 'mergedAt' => '2000-01-02T00:00:00Z', - 'closedAt' => '2000-01-02T00:00:00Z', - 'updatedAt' => '2000-01-02T00:00:00Z', - 'author' => ['login' => 'communityUser'] ] ] ] @@ -73,14 +84,25 @@ protected function getGraphqlData(string $query, array $variables, array $option } }; + $recent = gmdate(DATE_ATOM, time() - 2 * 86400); + $expectedRecentDays = (new \DateTime($recent))->diff(new \DateTime())->days; + $result = $datasource->readData([ - 'days' => 30, + 'days' => 10000, 'repo' => 'owner/repo1', 'exclude' => 'employee1', 'sla' => 14, ]); - $this->assertCount(2, $result['data']); + $this->assertCount(4, $result['data']); $this->assertSame(['owner/repo1','issue',1,'2025-01-01T00:00:00Z','2025-01-05T00:00:00Z',4,1,1], $result['data'][0]); - $this->assertSame(['owner/repo1','pr',10,'2025-01-01T00:00:00Z','2025-01-20T00:00:00Z',19,0,1], $result['data'][1]); + $this->assertSame(['owner/repo1','issue',2,'2025-01-01T00:00:00Z','2025-01-03T00:00:00Z',2,1,1], $result['data'][1]); + $this->assertSame('owner/repo1', $result['data'][2][0]); + $this->assertSame('issue', $result['data'][2][1]); + $this->assertSame(3, $result['data'][2][2]); + $this->assertSame($recent, $result['data'][2][3]); + $this->assertSame('', $result['data'][2][4]); + $this->assertSame($expectedRecentDays, $result['data'][2][5]); + $this->assertSame(1, $result['data'][2][6]); + $this->assertSame(['owner/repo1','pr',10,'2025-01-01T00:00:00Z','2025-01-20T00:00:00Z',19,0,1], $result['data'][3]); } } From 5d419798a823e9cf1274514a857e9a9fd84b0aed Mon Sep 17 00:00:00 2001 From: Rello Date: Mon, 25 Aug 2025 11:55:47 +0700 Subject: [PATCH 8/8] enhancements --- js/visualization.js | 61 +++++--- lib/Controller/DatasourceController.php | 66 ++++----- lib/Controller/OutputController.php | 2 +- lib/Datasource/GithubCommunitySla.php | 187 ++++++++++++++++++------ 4 files changed, 214 insertions(+), 102 deletions(-) diff --git a/js/visualization.js b/js/visualization.js index e346d30c..1f9a0d46 100644 --- a/js/visualization.js +++ b/js/visualization.js @@ -1116,7 +1116,8 @@ OCA.Analytics.Visualization = { const grouping = tg.grouping; const mode = tg.mode || 'summation'; - const valueIndex = data.data[0].length - 1; + const valueIndex1 = data.data[0].length - 2; // Second last column + const valueIndex2 = data.data[0].length - 1; // Last column if (data.data.length === 0) { return data; @@ -1161,7 +1162,8 @@ OCA.Analytics.Visualization = { return myMoment(date).format(); }; - const sums = {}; + const sums1 = {}; + const sums2 = {}; const counts = {}; data.data.forEach(row => { @@ -1182,22 +1184,26 @@ OCA.Analytics.Visualization = { const newRow = row.slice(); newRow[dimension] = newTime; - const key = newRow.slice(0, valueIndex).join('\u0001'); - const val = parseFloat(row[valueIndex]) || 0; + const key = newRow.slice(0, valueIndex1).join('\u0001'); + const val1 = parseFloat(row[valueIndex1]) || 0; + const val2 = parseFloat(row[valueIndex2]) || 0; - if (!sums[key]) { - sums[key] = val; + if (!sums1[key]) { + sums1[key] = val1; + sums2[key] = val2; counts[key] = 1; } else { - sums[key] += val; + sums1[key] += val1; + sums2[key] += val2; counts[key] += 1; } }); - data.data = Object.keys(sums).map(key => { + data.data = Object.keys(sums1).map(key => { const parts = key.split('\u0001'); - const value = mode === 'average' ? sums[key] / counts[key] : sums[key]; - return [...parts, value.toString()]; + const value1 = mode === 'average' ? sums1[key] / counts[key] : sums1[key]; + const value2 = mode === 'average' ? sums2[key] / counts[key] : sums2[key]; + return [...parts, value1.toString(), value2.toString()]; }); return data; @@ -1210,31 +1216,42 @@ OCA.Analytics.Visualization = { * @returns {Array} Updated rows */ formatDates: function (data) { - let firstRow = data[0]; let now; - for (let i = 0; i < firstRow.length; i++) { - // loop columns and check for a valid date - if (!isNaN(new Date(firstRow[i]).valueOf()) && firstRow[i] !== null && firstRow[i].length >= 19) { - // column contains a valid date - // then loop all rows for this column and convert to local time + for (let i = 0; i < data[0].length; i++) { + // Find a valid date in the column + let validDateFound = false; + for (let j = 0; j < data.length; j++) { + if (!isNaN(new Date(data[j][i]).valueOf()) && data[j][i] !== null && data[j][i].length >= 19) { + validDateFound = true; + break; + } + } + + // If a valid date is found, convert all dates in the column + if (validDateFound) { for (let j = 0; j < data.length; j++) { if (data[j][i].length === 19) { // values are assumed to have a timezone or are used as UTC data[j][i] = data[j][i] + 'Z'; } now = new Date(data[j][i]); - data[j][i] = now.getFullYear() - + "-" + (now.getMonth() < 9 ? '0' : '') + (now.getMonth() + 1) //getMonth will start with Jan = 0 - + "-" + (now.getDate() < 10 ? '0' : '') + now.getDate() - + " " + (now.getHours() < 10 ? '0' : '') + now.getHours() - + ":" + (now.getMinutes() < 10 ? '0' : '') + now.getMinutes() - + ":" + (now.getSeconds() < 10 ? '0' : '') + now.getSeconds() + if (!isNaN(now.valueOf())) { + data[j][i] = now.getFullYear() + + "-" + (now.getMonth() < 9 ? '0' : '') + (now.getMonth() + 1) + + "-" + (now.getDate() < 10 ? '0' : '') + now.getDate() + + " " + (now.getHours() < 10 ? '0' : '') + now.getHours() + + ":" + (now.getMinutes() < 10 ? '0' : '') + now.getMinutes() + + ":" + (now.getSeconds() < 10 ? '0' : '') + now.getSeconds(); + } else { + data[j][i] = ''; // Set to empty string if date is invalid + } } } } return data; }, + /** * Check a value against threshold rules and return a color style. * diff --git a/lib/Controller/DatasourceController.php b/lib/Controller/DatasourceController.php index f0daf054..e64d5b7c 100644 --- a/lib/Controller/DatasourceController.php +++ b/lib/Controller/DatasourceController.php @@ -27,8 +27,8 @@ class DatasourceController extends Controller { private $logger; - private $GithubService; - private $GithubCommunitySlaService; + private $GithubService; + private $GithubCommunitySlaService; private $ExternalCsvService; private $RegexService; private $ExternalJsonService; @@ -49,31 +49,31 @@ class DatasourceController extends Controller { const DATASET_TYPE_REGEX = 5; const DATASET_TYPE_EXTERNAL_JSON = 6; const DATASET_TYPE_LOCAL_SPREADSHEET = 7; - const DATASET_TYPE_LOCAL_JSON = 8; - const DATASET_TYPE_GITHUB_COMMUNITY_SLA = 9; + const DATASET_TYPE_LOCAL_JSON = 8; + const DATASET_TYPE_GITHUB_COMMUNITY_SLA = 9; public function __construct( - string $appName, - IRequest $request, - LoggerInterface $logger, - Github $GithubService, - GithubCommunitySla $GithubCommunitySlaService, - LocalCsv $LocalCsvService, - Regex $RegexService, - ExternalJson $ExternalJsonService, - LocalJson $LocalJsonService, - ExternalCsv $ExternalCsvService, - LocalSpreadsheet $LocalSpreadsheetService, - IL10N $l10n, - IEventDispatcher $dispatcher, - IAppConfig $appConfig, + string $appName, + IRequest $request, + LoggerInterface $logger, + Github $GithubService, + GithubCommunitySla $GithubCommunitySlaService, + LocalCsv $LocalCsvService, + Regex $RegexService, + ExternalJson $ExternalJsonService, + LocalJson $LocalJsonService, + ExternalCsv $ExternalCsvService, + LocalSpreadsheet $LocalSpreadsheetService, + IL10N $l10n, + IEventDispatcher $dispatcher, + IAppConfig $appConfig, ) { parent::__construct($appName, $request); $this->logger = $logger; $this->ExternalCsvService = $ExternalCsvService; - $this->GithubService = $GithubService; - $this->GithubCommunitySlaService = $GithubCommunitySlaService; - $this->RegexService = $RegexService; + $this->GithubService = $GithubService; + $this->GithubCommunitySlaService = $GithubCommunitySlaService; + $this->RegexService = $RegexService; $this->LocalCsvService = $LocalCsvService; $this->ExternalJsonService = $ExternalJsonService; $this->LocalJsonService = $LocalJsonService; @@ -134,7 +134,7 @@ public function getTemplates() { } /** - * Get the data from a data source; + * Get the data from a data source; * * @NoAdminRequired * @param int $datasourceId @@ -216,16 +216,16 @@ private function getDatasources(?int $datasourceType = null) { */ private function getOwnDatasources(?int $datasourceType = null) { $dataSources = []; - $serviceMapping = [ - self::DATASET_TYPE_GIT => $this->GithubService, - self::DATASET_TYPE_GITHUB_COMMUNITY_SLA => $this->GithubCommunitySlaService, - self::DATASET_TYPE_LOCAL_CSV => $this->LocalCsvService, - self::DATASET_TYPE_LOCAL_SPREADSHEET => $this->LocalSpreadsheetService, - self::DATASET_TYPE_EXTERNAL_CSV => $this->ExternalCsvService, - self::DATASET_TYPE_REGEX => $this->RegexService, - self::DATASET_TYPE_EXTERNAL_JSON => $this->ExternalJsonService, - self::DATASET_TYPE_LOCAL_JSON => $this->LocalJsonService, - ]; + $serviceMapping = [ + self::DATASET_TYPE_GIT => $this->GithubService, + self::DATASET_TYPE_GITHUB_COMMUNITY_SLA => $this->GithubCommunitySlaService, + self::DATASET_TYPE_LOCAL_CSV => $this->LocalCsvService, + self::DATASET_TYPE_LOCAL_SPREADSHEET => $this->LocalSpreadsheetService, + self::DATASET_TYPE_EXTERNAL_CSV => $this->ExternalCsvService, + self::DATASET_TYPE_REGEX => $this->RegexService, + self::DATASET_TYPE_EXTERNAL_JSON => $this->ExternalJsonService, + self::DATASET_TYPE_LOCAL_JSON => $this->LocalJsonService, + ]; if ($datasourceType !== null && isset($serviceMapping[$datasourceType])) { $dataSources[$datasourceType] = $serviceMapping[$datasourceType]; @@ -289,7 +289,7 @@ private function filterData($data, $filter) { $filtered[] = $record; } else if ($filterOption === 'IN') { preg_match_all("/'(?:[^'\\\\]|\\\\.)*'|[^,;]+/", $filterValue, $matches); - $valuesArray = array_map(function($v) { + $valuesArray = array_map(function ($v) { return trim($v, " '"); }, $matches[0]); diff --git a/lib/Controller/OutputController.php b/lib/Controller/OutputController.php index b701351b..6757b73d 100644 --- a/lib/Controller/OutputController.php +++ b/lib/Controller/OutputController.php @@ -134,7 +134,7 @@ private function returnDataWithCacheableHeader($reportMetadata) { $response = new DataResponse($result, HTTP::STATUS_OK); // only internal reports are cacheable - if ($reportMetadata['type'] === DatasourceController::DATASET_TYPE_INTERNAL_DB) { + if ($reportMetadata['type'] !== DatasourceController::DATASET_TYPE_INTERNAL_DB) { $response->addHeader('ETag', '"' . $reportMetadata['version'] . '"'); $response->addHeader('X-Analytics-Cacheable', 'true'); } else { diff --git a/lib/Datasource/GithubCommunitySla.php b/lib/Datasource/GithubCommunitySla.php index 27d33918..d116ab5e 100644 --- a/lib/Datasource/GithubCommunitySla.php +++ b/lib/Datasource/GithubCommunitySla.php @@ -29,8 +29,7 @@ class GithubCommunitySla implements IDatasource { * @var array */ protected array $repositories = [ - 'nextcloud/server', - 'nextcloud/analytics', + 'nextcloud/desktop', ]; public function __construct( @@ -120,10 +119,11 @@ public function readData($option): array { foreach ($repositories as $repo) { [$owner, $name] = explode('/', $repo, 2); - $query = <<<'GRAPHQL' -query($owner: String!, $name: String!) { + + $issuesQuery = <<<'GRAPHQL' +query($owner: String!, $name: String!, $after: String, $since: DateTime) { repository(owner: $owner, name: $name) { - issues(first: 100, orderBy: {field: UPDATED_AT, direction: DESC}, states: [OPEN, CLOSED]) { + issues(first: 100, after: $after, orderBy: {field: UPDATED_AT, direction: DESC}, states: [OPEN, CLOSED], filterBy: { since: $since }) { nodes { number createdAt @@ -135,10 +135,44 @@ public function readData($option): array { __typename ... on UnlabeledEvent { createdAt label { name } } } + pageInfo { + hasNextPage + endCursor + } + } + } + pageInfo { + hasNextPage + endCursor + } + } + } +} +GRAPHQL; + + $issueEventsPageQuery = <<<'GRAPHQL' +query($owner: String!, $name: String!, $issueNumber: Int!, $after: String) { + repository(owner: $owner, name: $name) { + issue(number: $issueNumber) { + timelineItems(first: 100, after: $after, itemTypes: [UNLABELED_EVENT]) { + nodes { + __typename + ... on UnlabeledEvent { createdAt label { name } } + } + pageInfo { + hasNextPage + endCursor } } } - pullRequests(first: 100, orderBy: {field: UPDATED_AT, direction: DESC}, states: [OPEN, MERGED, CLOSED]) { + } +} +GRAPHQL; + + $pullsQuery = <<<'GRAPHQL' +query($owner: String!, $name: String!, $after: String) { + repository(owner: $owner, name: $name) { + pullRequests(first: 100, after: $after, orderBy: {field: UPDATED_AT, direction: DESC}, states: [OPEN, MERGED, CLOSED]) { nodes { number createdAt @@ -147,58 +181,119 @@ public function readData($option): array { updatedAt author { login } } + pageInfo { + hasNextPage + endCursor + } } } } GRAPHQL; - $curlResult = $this->getGraphqlData($query, ['owner' => $owner, 'name' => $name], $option); - if ($curlResult['http_code'] < 200 || $curlResult['http_code'] >= 300 || isset($curlResult['data']['errors'])) { - return [ - 'header' => [], - 'dimensions' => [], - 'data' => $curlResult['http_code'] === 403 ? 'Rate limit exceeded' : [], - 'rawdata' => $curlResult, - 'error' => 'HTTP response code: ' . $curlResult['http_code'], - ]; - } - - $repoData = $curlResult['data']['data']['repository']; - - foreach ($repoData['issues']['nodes'] as $issue) { - if ($issue['updatedAt'] < $sinceDate) { - continue; - } - if (in_array($issue['author']['login'], $excludedAuthors, true)) { - continue; + // Fetch issues with pagination + $issuesAfter = null; + do { + $variables = ['owner' => $owner, 'name' => $name, 'after' => $issuesAfter, 'since' => $sinceDate]; + $curlResult = $this->getGraphqlData($issuesQuery, $variables, $option); + if ($curlResult['http_code'] < 200 || $curlResult['http_code'] >= 300 || isset($curlResult['data']['errors'])) { + return [ + 'header' => [], + 'dimensions' => [], + 'data' => $curlResult['http_code'] === 403 ? 'Rate limit exceeded' : [], + 'rawdata' => $curlResult, + 'error' => 'HTTP response code: ' . $curlResult['http_code'], + ]; } - $completedAt = ''; - foreach ($issue['timelineItems']['nodes'] as $event) { - if ($event['__typename'] === 'UnlabeledEvent' && isset($event['label']['name']) && $event['label']['name'] === '0. Needs triage') { - $completedAt = $event['createdAt']; - break; + $repoData = $curlResult['data']['data']['repository']; + $issuesEdge = $repoData['issues']; + + foreach ($issuesEdge['nodes'] as $issue) { + if ($issue['createdAt'] < $sinceDate) { + continue; + } + if (in_array($issue['author']['login'], $excludedAuthors, true)) { + continue; } + $completedAt = ''; + $events = $issue['timelineItems']['nodes']; + $eventsPageInfo = $issue['timelineItems']['pageInfo']; + // Search for "0. Needs triage" label with case-insensitive comparison + $foundTriage = false; + foreach ($events as $event) { + if ($event['__typename'] === 'UnlabeledEvent' && isset($event['label']['name']) && strcasecmp($event['label']['name'], '0. Needs triage') === 0) { + $completedAt = $event['createdAt']; + $foundTriage = true; + break; + } + } + // If not found and there are more timeline pages, paginate through timeline items + $afterTimeline = $eventsPageInfo['endCursor'] ?? null; + while (!$foundTriage && $eventsPageInfo['hasNextPage'] && $afterTimeline !== null) { + $variablesEvents = ['owner' => $owner, 'name' => $name, 'issueNumber' => (int)$issue['number'], 'after' => $afterTimeline]; + $eventsResult = $this->getGraphqlData($issueEventsPageQuery, $variablesEvents, $option); + if ($eventsResult['http_code'] < 200 || $eventsResult['http_code'] >= 300 || isset($eventsResult['data']['errors'])) { + break; + } + $eventsPage = $eventsResult['data']['data']['repository']['issue']['timelineItems']; + foreach ($eventsPage['nodes'] as $event) { + if ($event['__typename'] === 'UnlabeledEvent' && isset($event['label']['name']) && strcasecmp($event['label']['name'], '0. Needs triage') === 0) { + $completedAt = $event['createdAt']; + $foundTriage = true; + break 2; + } + } + $eventsPageInfo = $eventsPage['pageInfo']; + $afterTimeline = $eventsPageInfo['endCursor'] ?? null; + } + if ($completedAt === '' && isset($issue['closedAt']) && $issue['closedAt'] !== null) { + $completedAt = $issue['closedAt']; + } + $days = $this->daysBetween($issue['createdAt'], $completedAt ?: date(DATE_ATOM)); + $slaMet = $days <= $slaDays; + $data[] = [$repo, 'issue', (int)$issue['number'], $issue['createdAt'], $completedAt, $days, $slaMet ? 1 : 0, 1]; } - if ($completedAt === '' && isset($issue['closedAt']) && $issue['closedAt'] !== null) { - $completedAt = $issue['closedAt']; + $issuesAfter = $issuesEdge['pageInfo']['endCursor'] ?? null; + } while ($issuesEdge['pageInfo']['hasNextPage']); + + // Fetch pull requests with pagination + $prsAfter = null; + $continuePaging = true; + do { + $variables = ['owner' => $owner, 'name' => $name, 'after' => $prsAfter]; + $curlResult = $this->getGraphqlData($pullsQuery, $variables, $option); + if ($curlResult['http_code'] < 200 || $curlResult['http_code'] >= 300 || isset($curlResult['data']['errors'])) { + return [ + 'header' => [], + 'dimensions' => [], + 'data' => $curlResult['http_code'] === 403 ? 'Rate limit exceeded' : [], + 'rawdata' => $curlResult, + 'error' => 'HTTP response code: ' . $curlResult['http_code'], + ]; } - $days = $this->daysBetween($issue['createdAt'], $completedAt ?: date(DATE_ATOM)); - $slaMet = $days <= $slaDays; - $data[] = [$repo, 'issue', (int)$issue['number'], $issue['createdAt'], $completedAt, $days, $slaMet ? 1 : 0, 1]; - } + $repoData = $curlResult['data']['data']['repository']; + $prsEdge = $repoData['pullRequests']; - foreach ($repoData['pullRequests']['nodes'] as $pr) { - if ($pr['updatedAt'] < $sinceDate) { - continue; + $pageHasRecent = false; + foreach ($prsEdge['nodes'] as $pr) { + $isRecent = ($pr['updatedAt'] >= $sinceDate); + if (!$isRecent) { + continue; + } + $pageHasRecent = true; + if (in_array($pr['author']['login'], $excludedAuthors, true)) { + continue; + } + $completedAt = $pr['mergedAt'] ?? $pr['closedAt'] ?? ''; + $days = $this->daysBetween($pr['createdAt'], $completedAt ?: date(DATE_ATOM)); + $slaMet = $days <= $slaDays; + $data[] = [$repo, 'pr', (int)$pr['number'], $pr['createdAt'], $completedAt, $days, $slaMet ? 1 : 0, 1]; } - if (in_array($pr['author']['login'], $excludedAuthors, true)) { - continue; + if (!$pageHasRecent) { + $continuePaging = false; } - $completedAt = $pr['mergedAt'] ?? $pr['closedAt'] ?? ''; - $days = $this->daysBetween($pr['createdAt'], $completedAt ?: date(DATE_ATOM)); - $slaMet = $days <= $slaDays; - $data[] = [$repo, 'pr', (int)$pr['number'], $pr['createdAt'], $completedAt, $days, $slaMet ? 1 : 0, 1]; - } + $prsAfter = $prsEdge['pageInfo']['endCursor'] ?? null; + $morePrPages = ($prsEdge['pageInfo']['hasNextPage'] ?? false) && $continuePaging; + } while ($morePrPages); } return [