diff --git a/CHANGELOG.md b/CHANGELOG.md index 6367538e..79ca19cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,14 @@ ### Added - Cache report data in the browser (using ETag & If-None-Match) #535 - Warn about unsaved changes before leaving a report +- GitHub Community SLA data source +- Configure excluded authors, repositories, and SLA days via GitHub Community SLA options + +### Changed +- Treat issues as triaged when the "0. Needs triage" label is removed or the issue is closed and evaluate open items against the current date in GitHub Community SLA data source +- Use GitHub GraphQL API for more efficient Community SLA queries +- Register GitHub Community SLA data source with ID 9 +- Include counter column and use merge/close dates for PR SLA calculation in GitHub Community SLA data source ### Fixed - Fix PHP 8.4 deprecation warnings #534 @[robertoschwald](https://github.com/robertoschwald) diff --git a/js/visualization.js b/js/visualization.js index e346d30c..1f9a0d46 100644 --- a/js/visualization.js +++ b/js/visualization.js @@ -1116,7 +1116,8 @@ OCA.Analytics.Visualization = { const grouping = tg.grouping; const mode = tg.mode || 'summation'; - const valueIndex = data.data[0].length - 1; + const valueIndex1 = data.data[0].length - 2; // Second last column + const valueIndex2 = data.data[0].length - 1; // Last column if (data.data.length === 0) { return data; @@ -1161,7 +1162,8 @@ OCA.Analytics.Visualization = { return myMoment(date).format(); }; - const sums = {}; + const sums1 = {}; + const sums2 = {}; const counts = {}; data.data.forEach(row => { @@ -1182,22 +1184,26 @@ OCA.Analytics.Visualization = { const newRow = row.slice(); newRow[dimension] = newTime; - const key = newRow.slice(0, valueIndex).join('\u0001'); - const val = parseFloat(row[valueIndex]) || 0; + const key = newRow.slice(0, valueIndex1).join('\u0001'); + const val1 = parseFloat(row[valueIndex1]) || 0; + const val2 = parseFloat(row[valueIndex2]) || 0; - if (!sums[key]) { - sums[key] = val; + if (!sums1[key]) { + sums1[key] = val1; + sums2[key] = val2; counts[key] = 1; } else { - sums[key] += val; + sums1[key] += val1; + sums2[key] += val2; counts[key] += 1; } }); - data.data = Object.keys(sums).map(key => { + data.data = Object.keys(sums1).map(key => { const parts = key.split('\u0001'); - const value = mode === 'average' ? sums[key] / counts[key] : sums[key]; - return [...parts, value.toString()]; + const value1 = mode === 'average' ? sums1[key] / counts[key] : sums1[key]; + const value2 = mode === 'average' ? sums2[key] / counts[key] : sums2[key]; + return [...parts, value1.toString(), value2.toString()]; }); return data; @@ -1210,31 +1216,42 @@ OCA.Analytics.Visualization = { * @returns {Array} Updated rows */ formatDates: function (data) { - let firstRow = data[0]; let now; - for (let i = 0; i < firstRow.length; i++) { - // loop columns and check for a valid date - if (!isNaN(new Date(firstRow[i]).valueOf()) && firstRow[i] !== null && firstRow[i].length >= 19) { - // column contains a valid date - // then loop all rows for this column and convert to local time + for (let i = 0; i < data[0].length; i++) { + // Find a valid date in the column + let validDateFound = false; + for (let j = 0; j < data.length; j++) { + if (!isNaN(new Date(data[j][i]).valueOf()) && data[j][i] !== null && data[j][i].length >= 19) { + validDateFound = true; + break; + } + } + + // If a valid date is found, convert all dates in the column + if (validDateFound) { for (let j = 0; j < data.length; j++) { if (data[j][i].length === 19) { // values are assumed to have a timezone or are used as UTC data[j][i] = data[j][i] + 'Z'; } now = new Date(data[j][i]); - data[j][i] = now.getFullYear() - + "-" + (now.getMonth() < 9 ? '0' : '') + (now.getMonth() + 1) //getMonth will start with Jan = 0 - + "-" + (now.getDate() < 10 ? '0' : '') + now.getDate() - + " " + (now.getHours() < 10 ? '0' : '') + now.getHours() - + ":" + (now.getMinutes() < 10 ? '0' : '') + now.getMinutes() - + ":" + (now.getSeconds() < 10 ? '0' : '') + now.getSeconds() + if (!isNaN(now.valueOf())) { + data[j][i] = now.getFullYear() + + "-" + (now.getMonth() < 9 ? '0' : '') + (now.getMonth() + 1) + + "-" + (now.getDate() < 10 ? '0' : '') + now.getDate() + + " " + (now.getHours() < 10 ? '0' : '') + now.getHours() + + ":" + (now.getMinutes() < 10 ? '0' : '') + now.getMinutes() + + ":" + (now.getSeconds() < 10 ? '0' : '') + now.getSeconds(); + } else { + data[j][i] = ''; // Set to empty string if date is invalid + } } } } return data; }, + /** * Check a value against threshold rules and return a color style. * diff --git a/lib/Controller/DatasourceController.php b/lib/Controller/DatasourceController.php index 93dc3e49..e64d5b7c 100644 --- a/lib/Controller/DatasourceController.php +++ b/lib/Controller/DatasourceController.php @@ -12,6 +12,7 @@ use OCA\Analytics\Datasource\ExternalCsv; use OCA\Analytics\Datasource\ExternalJson; use OCA\Analytics\Datasource\Github; +use OCA\Analytics\Datasource\GithubCommunitySla; use OCA\Analytics\Datasource\LocalCsv; use OCA\Analytics\Datasource\LocalSpreadsheet; use OCA\Analytics\Datasource\LocalJson; @@ -27,6 +28,7 @@ class DatasourceController extends Controller { private $logger; private $GithubService; + private $GithubCommunitySlaService; private $ExternalCsvService; private $RegexService; private $ExternalJsonService; @@ -48,26 +50,29 @@ class DatasourceController extends Controller { const DATASET_TYPE_EXTERNAL_JSON = 6; const DATASET_TYPE_LOCAL_SPREADSHEET = 7; const DATASET_TYPE_LOCAL_JSON = 8; + const DATASET_TYPE_GITHUB_COMMUNITY_SLA = 9; public function __construct( - string $appName, - IRequest $request, - LoggerInterface $logger, - Github $GithubService, - LocalCsv $LocalCsvService, - Regex $RegexService, - ExternalJson $ExternalJsonService, - LocalJson $LocalJsonService, - ExternalCsv $ExternalCsvService, - LocalSpreadsheet $LocalSpreadsheetService, - IL10N $l10n, - IEventDispatcher $dispatcher, - IAppConfig $appConfig, + string $appName, + IRequest $request, + LoggerInterface $logger, + Github $GithubService, + GithubCommunitySla $GithubCommunitySlaService, + LocalCsv $LocalCsvService, + Regex $RegexService, + ExternalJson $ExternalJsonService, + LocalJson $LocalJsonService, + ExternalCsv $ExternalCsvService, + LocalSpreadsheet $LocalSpreadsheetService, + IL10N $l10n, + IEventDispatcher $dispatcher, + IAppConfig $appConfig, ) { parent::__construct($appName, $request); $this->logger = $logger; $this->ExternalCsvService = $ExternalCsvService; $this->GithubService = $GithubService; + $this->GithubCommunitySlaService = $GithubCommunitySlaService; $this->RegexService = $RegexService; $this->LocalCsvService = $LocalCsvService; $this->ExternalJsonService = $ExternalJsonService; @@ -129,7 +134,7 @@ public function getTemplates() { } /** - * Get the data from a data source; + * Get the data from a data source; * * @NoAdminRequired * @param int $datasourceId @@ -213,6 +218,7 @@ private function getOwnDatasources(?int $datasourceType = null) { $dataSources = []; $serviceMapping = [ self::DATASET_TYPE_GIT => $this->GithubService, + self::DATASET_TYPE_GITHUB_COMMUNITY_SLA => $this->GithubCommunitySlaService, self::DATASET_TYPE_LOCAL_CSV => $this->LocalCsvService, self::DATASET_TYPE_LOCAL_SPREADSHEET => $this->LocalSpreadsheetService, self::DATASET_TYPE_EXTERNAL_CSV => $this->ExternalCsvService, @@ -283,7 +289,7 @@ private function filterData($data, $filter) { $filtered[] = $record; } else if ($filterOption === 'IN') { preg_match_all("/'(?:[^'\\\\]|\\\\.)*'|[^,;]+/", $filterValue, $matches); - $valuesArray = array_map(function($v) { + $valuesArray = array_map(function ($v) { return trim($v, " '"); }, $matches[0]); diff --git a/lib/Controller/OutputController.php b/lib/Controller/OutputController.php index b701351b..6757b73d 100644 --- a/lib/Controller/OutputController.php +++ b/lib/Controller/OutputController.php @@ -134,7 +134,7 @@ private function returnDataWithCacheableHeader($reportMetadata) { $response = new DataResponse($result, HTTP::STATUS_OK); // only internal reports are cacheable - if ($reportMetadata['type'] === DatasourceController::DATASET_TYPE_INTERNAL_DB) { + if ($reportMetadata['type'] !== DatasourceController::DATASET_TYPE_INTERNAL_DB) { $response->addHeader('ETag', '"' . $reportMetadata['version'] . '"'); $response->addHeader('X-Analytics-Cacheable', 'true'); } else { diff --git a/lib/Datasource/GithubCommunitySla.php b/lib/Datasource/GithubCommunitySla.php new file mode 100644 index 00000000..d116ab5e --- /dev/null +++ b/lib/Datasource/GithubCommunitySla.php @@ -0,0 +1,342 @@ + + */ + protected array $excludedAuthors = [ + 'alice', + 'bob', + ]; + + /** + * Repositories to analyse in owner/repo format + * @var array + */ + protected array $repositories = [ + 'nextcloud/desktop', + ]; + + public function __construct( + IL10N $l10n, + LoggerInterface $logger + ) { + $this->l10n = $l10n; + $this->logger = $logger; + } + + /** + * @return string Display Name of the data source + */ + public function getName(): string { + return 'GitHub Community SLA'; + } + + /** + * @return int digit unique data source id + */ + public function getId(): int { + return 9; + } + + /** + * @return array available options of the data source + */ + public function getTemplate(): array { + $template = []; + $template[] = [ + 'id' => 'token', + 'name' => $this->l10n->t('Personal access token'), + 'placeholder' => $this->l10n->t('optional') + ]; + $template[] = [ + 'id' => 'repo', + 'name' => $this->l10n->t('Repositories'), + 'placeholder' => 'owner/repo1,owner/repo2' + ]; + $template[] = [ + 'id' => 'exclude', + 'name' => $this->l10n->t('Exclude authors'), + 'placeholder' => 'user1,user2' + ]; + $template[] = [ + 'id' => 'sla', + 'name' => $this->l10n->t('SLA days'), + 'placeholder' => '14', + 'type' => 'number' + ]; + $template[] = [ + 'id' => 'days', + 'name' => $this->l10n->t('Updated since (days)'), + 'placeholder' => '30', + 'type' => 'number' + ]; + return $template; + } + + /** + * Read the Data + * @param $option + * @return array available options of the data source + */ + public function readData($option): array { + $header = [ + $this->l10n->t('Repository'), + $this->l10n->t('Type'), + $this->l10n->t('Number'), + $this->l10n->t('Created'), + $this->l10n->t('Completed'), + $this->l10n->t('Days'), + $this->l10n->t('SLA met'), + $this->l10n->t('Counter') + ]; + $data = []; + + $repositories = isset($option['repo']) && $option['repo'] !== '' + ? array_map('trim', explode(',', $option['repo'])) + : $this->repositories; + $excludedAuthors = isset($option['exclude']) && $option['exclude'] !== '' + ? array_map('trim', explode(',', $option['exclude'])) + : $this->excludedAuthors; + $slaDays = isset($option['sla']) && (int)$option['sla'] > 0 ? (int)$option['sla'] : 14; + $daysFilter = isset($option['days']) && (int)$option['days'] > 0 ? (int)$option['days'] : 30; + $sinceDate = date(DATE_ATOM, time() - ($daysFilter * 86400)); + + foreach ($repositories as $repo) { + [$owner, $name] = explode('/', $repo, 2); + + $issuesQuery = <<<'GRAPHQL' +query($owner: String!, $name: String!, $after: String, $since: DateTime) { + repository(owner: $owner, name: $name) { + issues(first: 100, after: $after, orderBy: {field: UPDATED_AT, direction: DESC}, states: [OPEN, CLOSED], filterBy: { since: $since }) { + nodes { + number + createdAt + updatedAt + closedAt + author { login } + timelineItems(first: 100, itemTypes: [UNLABELED_EVENT]) { + nodes { + __typename + ... on UnlabeledEvent { createdAt label { name } } + } + pageInfo { + hasNextPage + endCursor + } + } + } + pageInfo { + hasNextPage + endCursor + } + } + } +} +GRAPHQL; + + $issueEventsPageQuery = <<<'GRAPHQL' +query($owner: String!, $name: String!, $issueNumber: Int!, $after: String) { + repository(owner: $owner, name: $name) { + issue(number: $issueNumber) { + timelineItems(first: 100, after: $after, itemTypes: [UNLABELED_EVENT]) { + nodes { + __typename + ... on UnlabeledEvent { createdAt label { name } } + } + pageInfo { + hasNextPage + endCursor + } + } + } + } +} +GRAPHQL; + + $pullsQuery = <<<'GRAPHQL' +query($owner: String!, $name: String!, $after: String) { + repository(owner: $owner, name: $name) { + pullRequests(first: 100, after: $after, orderBy: {field: UPDATED_AT, direction: DESC}, states: [OPEN, MERGED, CLOSED]) { + nodes { + number + createdAt + mergedAt + closedAt + updatedAt + author { login } + } + pageInfo { + hasNextPage + endCursor + } + } + } +} +GRAPHQL; + + // Fetch issues with pagination + $issuesAfter = null; + do { + $variables = ['owner' => $owner, 'name' => $name, 'after' => $issuesAfter, 'since' => $sinceDate]; + $curlResult = $this->getGraphqlData($issuesQuery, $variables, $option); + if ($curlResult['http_code'] < 200 || $curlResult['http_code'] >= 300 || isset($curlResult['data']['errors'])) { + return [ + 'header' => [], + 'dimensions' => [], + 'data' => $curlResult['http_code'] === 403 ? 'Rate limit exceeded' : [], + 'rawdata' => $curlResult, + 'error' => 'HTTP response code: ' . $curlResult['http_code'], + ]; + } + $repoData = $curlResult['data']['data']['repository']; + $issuesEdge = $repoData['issues']; + + foreach ($issuesEdge['nodes'] as $issue) { + if ($issue['createdAt'] < $sinceDate) { + continue; + } + if (in_array($issue['author']['login'], $excludedAuthors, true)) { + continue; + } + $completedAt = ''; + $events = $issue['timelineItems']['nodes']; + $eventsPageInfo = $issue['timelineItems']['pageInfo']; + // Search for "0. Needs triage" label with case-insensitive comparison + $foundTriage = false; + foreach ($events as $event) { + if ($event['__typename'] === 'UnlabeledEvent' && isset($event['label']['name']) && strcasecmp($event['label']['name'], '0. Needs triage') === 0) { + $completedAt = $event['createdAt']; + $foundTriage = true; + break; + } + } + // If not found and there are more timeline pages, paginate through timeline items + $afterTimeline = $eventsPageInfo['endCursor'] ?? null; + while (!$foundTriage && $eventsPageInfo['hasNextPage'] && $afterTimeline !== null) { + $variablesEvents = ['owner' => $owner, 'name' => $name, 'issueNumber' => (int)$issue['number'], 'after' => $afterTimeline]; + $eventsResult = $this->getGraphqlData($issueEventsPageQuery, $variablesEvents, $option); + if ($eventsResult['http_code'] < 200 || $eventsResult['http_code'] >= 300 || isset($eventsResult['data']['errors'])) { + break; + } + $eventsPage = $eventsResult['data']['data']['repository']['issue']['timelineItems']; + foreach ($eventsPage['nodes'] as $event) { + if ($event['__typename'] === 'UnlabeledEvent' && isset($event['label']['name']) && strcasecmp($event['label']['name'], '0. Needs triage') === 0) { + $completedAt = $event['createdAt']; + $foundTriage = true; + break 2; + } + } + $eventsPageInfo = $eventsPage['pageInfo']; + $afterTimeline = $eventsPageInfo['endCursor'] ?? null; + } + if ($completedAt === '' && isset($issue['closedAt']) && $issue['closedAt'] !== null) { + $completedAt = $issue['closedAt']; + } + $days = $this->daysBetween($issue['createdAt'], $completedAt ?: date(DATE_ATOM)); + $slaMet = $days <= $slaDays; + $data[] = [$repo, 'issue', (int)$issue['number'], $issue['createdAt'], $completedAt, $days, $slaMet ? 1 : 0, 1]; + } + $issuesAfter = $issuesEdge['pageInfo']['endCursor'] ?? null; + } while ($issuesEdge['pageInfo']['hasNextPage']); + + // Fetch pull requests with pagination + $prsAfter = null; + $continuePaging = true; + do { + $variables = ['owner' => $owner, 'name' => $name, 'after' => $prsAfter]; + $curlResult = $this->getGraphqlData($pullsQuery, $variables, $option); + if ($curlResult['http_code'] < 200 || $curlResult['http_code'] >= 300 || isset($curlResult['data']['errors'])) { + return [ + 'header' => [], + 'dimensions' => [], + 'data' => $curlResult['http_code'] === 403 ? 'Rate limit exceeded' : [], + 'rawdata' => $curlResult, + 'error' => 'HTTP response code: ' . $curlResult['http_code'], + ]; + } + $repoData = $curlResult['data']['data']['repository']; + $prsEdge = $repoData['pullRequests']; + + $pageHasRecent = false; + foreach ($prsEdge['nodes'] as $pr) { + $isRecent = ($pr['updatedAt'] >= $sinceDate); + if (!$isRecent) { + continue; + } + $pageHasRecent = true; + if (in_array($pr['author']['login'], $excludedAuthors, true)) { + continue; + } + $completedAt = $pr['mergedAt'] ?? $pr['closedAt'] ?? ''; + $days = $this->daysBetween($pr['createdAt'], $completedAt ?: date(DATE_ATOM)); + $slaMet = $days <= $slaDays; + $data[] = [$repo, 'pr', (int)$pr['number'], $pr['createdAt'], $completedAt, $days, $slaMet ? 1 : 0, 1]; + } + if (!$pageHasRecent) { + $continuePaging = false; + } + $prsAfter = $prsEdge['pageInfo']['endCursor'] ?? null; + $morePrPages = ($prsEdge['pageInfo']['hasNextPage'] ?? false) && $continuePaging; + } while ($morePrPages); + } + + return [ + 'header' => $header, + 'dimensions' => array_slice($header, 0, count($header) - 2), + 'data' => $data, + 'rawdata' => [], + 'error' => 0, + ]; + } + + protected function getGraphqlData(string $query, array $variables, array $option): array { + $ch = curl_init('https://api.github.com/graphql'); + if ($ch !== false) { + curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false); + curl_setopt($ch, CURLOPT_HEADER, false); + curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true); + curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); + curl_setopt($ch, CURLOPT_USERAGENT, 'Mozilla/5.0'); + curl_setopt($ch, CURLOPT_POST, true); + curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode(['query' => $query, 'variables' => $variables])); + $headers = [ + 'Content-Type: application/json', + 'User-Agent: AnalyticsApp' + ]; + if (isset($option['token']) && $option['token'] !== '') { + $headers[] = 'Authorization: bearer ' . $option['token']; + } + curl_setopt($ch, CURLOPT_HTTPHEADER, $headers); + $curlResult = curl_exec($ch); + $http_code = curl_getinfo($ch, CURLINFO_HTTP_CODE); + curl_close($ch); + } else { + $curlResult = ''; + $http_code = 500; + } + $curlResult = json_decode($curlResult, true); + return ['data' => $curlResult, 'http_code' => $http_code]; + } + + private function daysBetween(string $from, string $to): int { + $start = new \DateTime($from); + $end = new \DateTime($to); + return $start->diff($end)->days; + } +} diff --git a/tests/Datasource/GithubCommunitySlaTest.php b/tests/Datasource/GithubCommunitySlaTest.php new file mode 100644 index 00000000..19af3bd6 --- /dev/null +++ b/tests/Datasource/GithubCommunitySlaTest.php @@ -0,0 +1,108 @@ + [ + 'data' => [ + 'repository' => [ + 'issues' => [ + 'nodes' => [ + [ + 'number' => 1, + 'createdAt' => '2025-01-01T00:00:00Z', + 'updatedAt' => '2025-01-05T00:00:00Z', + 'closedAt' => null, + 'author' => ['login' => 'communityUser'], + 'timelineItems' => [ + 'nodes' => [ + [ + '__typename' => 'UnlabeledEvent', + 'label' => ['name' => '0. Needs triage'], + 'createdAt' => '2025-01-05T00:00:00Z' + ] + ] + ] + ], + [ + 'number' => 2, + 'createdAt' => '2025-01-01T00:00:00Z', + 'updatedAt' => '2025-01-03T00:00:00Z', + 'closedAt' => '2025-01-03T00:00:00Z', + 'author' => ['login' => 'communityUser'], + 'timelineItems' => ['nodes' => []] + ], + [ + 'number' => 3, + 'createdAt' => $recent, + 'updatedAt' => $recent, + 'closedAt' => null, + 'author' => ['login' => 'communityUser'], + 'timelineItems' => ['nodes' => []] + ], + [ + 'number' => 4, + 'createdAt' => '2025-01-05T00:00:00Z', + 'updatedAt' => '2025-01-06T00:00:00Z', + 'closedAt' => null, + 'author' => ['login' => 'employee1'], + 'timelineItems' => ['nodes' => []] + ] + ] + ], + 'pullRequests' => [ + 'nodes' => [ + [ + 'number' => 10, + 'createdAt' => '2025-01-01T00:00:00Z', + 'mergedAt' => null, + 'closedAt' => '2025-01-20T00:00:00Z', + 'updatedAt' => '2025-01-20T00:00:00Z', + 'author' => ['login' => 'communityUser'] + ], + [ + 'number' => 11, + 'createdAt' => '2025-01-02T00:00:00Z', + 'mergedAt' => null, + 'closedAt' => null, + 'updatedAt' => '2025-01-02T00:00:00Z', + 'author' => ['login' => 'employee1'] + ] + ] + ] + ] + ] + ], 'http_code' => 200]; + } + }; + + $recent = gmdate(DATE_ATOM, time() - 2 * 86400); + $expectedRecentDays = (new \DateTime($recent))->diff(new \DateTime())->days; + + $result = $datasource->readData([ + 'days' => 10000, + 'repo' => 'owner/repo1', + 'exclude' => 'employee1', + 'sla' => 14, + ]); + $this->assertCount(4, $result['data']); + $this->assertSame(['owner/repo1','issue',1,'2025-01-01T00:00:00Z','2025-01-05T00:00:00Z',4,1,1], $result['data'][0]); + $this->assertSame(['owner/repo1','issue',2,'2025-01-01T00:00:00Z','2025-01-03T00:00:00Z',2,1,1], $result['data'][1]); + $this->assertSame('owner/repo1', $result['data'][2][0]); + $this->assertSame('issue', $result['data'][2][1]); + $this->assertSame(3, $result['data'][2][2]); + $this->assertSame($recent, $result['data'][2][3]); + $this->assertSame('', $result['data'][2][4]); + $this->assertSame($expectedRecentDays, $result['data'][2][5]); + $this->assertSame(1, $result['data'][2][6]); + $this->assertSame(['owner/repo1','pr',10,'2025-01-01T00:00:00Z','2025-01-20T00:00:00Z',19,0,1], $result['data'][3]); + } +}