From cadb0af66abcc324cdbf72f95346c3a55ae7dc37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20L=C3=B3pez?= <50665615+flopez7@users.noreply.github.com> Date: Tue, 12 May 2026 16:29:37 +0200 Subject: [PATCH 1/6] [Job Launcher] Job creation to be job type agnostic (#3897) Co-authored-by: flopez7 --- .../assignment/assignment.service.spec.ts | 10 +- .../modules/assignment/assignment.service.ts | 8 +- .../server/src/modules/job/job.dto.ts | 1 - .../src/modules/job/job.service.spec.ts | 7 - .../server/src/modules/job/job.service.ts | 31 +- .../server/src/modules/web3/web3.service.ts | 1 - .../src/common/interfaces/job.ts | 1 - .../src/modules/job/job.service.spec.ts | 26 +- .../src/modules/job/job.service.ts | 20 +- .../recording-oracle/test/constants.ts | 1 - .../Jobs/Create/CvatJobRequestForm.tsx | 33 +- .../src/components/Jobs/Create/helpers.ts | 1 + .../src/components/Jobs/Create/schema.ts | 4 + .../job-launcher/client/src/constants/cvat.ts | 2 + .../job-launcher/client/src/services/job.ts | 81 +- .../job-launcher/client/src/types/index.ts | 45 +- .../apps/job-launcher/server/.env.example | 10 - .../server/src/common/config/config.module.ts | 6 - .../server/src/common/config/env-schema.ts | 14 - .../src/common/config/slack-config.service.ts | 17 - .../common/config/vision-config.service.ts | 51 -- .../src/common/config/web3-config.service.ts | 8 - .../server/src/common/constants/errors.ts | 26 - .../server/src/common/constants/index.ts | 14 +- .../src/common/enums/content-moderation.ts | 7 - .../server/src/common/enums/cron-job.ts | 1 - .../server/src/common/enums/gcv.ts | 9 - .../server/src/common/enums/job.ts | 3 - .../server/src/common/utils/gcstorage.spec.ts | 191 ----- .../server/src/common/utils/gcstorage.ts | 193 ----- .../server/src/common/utils/storage.ts | 8 +- .../server/src/database/database.module.ts | 2 - .../1774453578372-removeContentModeration.ts | 151 ++++ .../content-moderation-request.entity.ts | 31 - .../content-moderation-request.repository.ts | 94 --- .../content-moderation.dto.ts | 17 - .../content-moderation.interface.ts | 5 - .../content-moderation.module.ts | 27 - .../gcv-content-moderation.service.spec.ts | 772 ------------------ .../gcv-content-moderation.service.ts | 507 ------------ .../src/modules/cron-job/cron-job.module.ts | 2 - .../modules/cron-job/cron-job.service.spec.ts | 140 +--- .../src/modules/cron-job/cron-job.service.ts | 46 +- .../server/src/modules/job/fixtures.ts | 51 +- .../src/modules/job/job.controller.spec.ts | 257 +++--- .../server/src/modules/job/job.controller.ts | 65 +- .../server/src/modules/job/job.dto.ts | 114 +-- .../server/src/modules/job/job.entity.ts | 8 - .../server/src/modules/job/job.interface.ts | 81 -- .../server/src/modules/job/job.module.ts | 2 - .../server/src/modules/job/job.repository.ts | 8 +- .../src/modules/job/job.service.spec.ts | 338 ++++---- .../server/src/modules/job/job.service.ts | 101 ++- .../server/src/modules/manifest/fixtures.ts | 50 +- .../src/modules/manifest/manifest.dto.ts | 5 - .../src/modules/manifest/manifest.module.ts | 12 +- .../modules/manifest/manifest.service.spec.ts | 374 +++------ .../src/modules/manifest/manifest.service.ts | 269 +----- .../routing-protocol.interface.ts | 29 - .../routing-protocol.module.ts | 11 - .../routing-protocol.service.spec.ts | 498 ----------- .../routing-protocol.service.ts | 259 ------ .../src/modules/web3/web3.service.spec.ts | 21 - .../webhook/webhook.controller.spec.ts | 9 - .../job-launcher/server/test/constants.ts | 162 +--- .../server/src/common/types/manifest.ts | 1 - .../escrow-completion/fixtures/fortune.ts | 1 - 67 files changed, 827 insertions(+), 4523 deletions(-) create mode 100644 packages/apps/job-launcher/client/src/constants/cvat.ts delete mode 100644 packages/apps/job-launcher/server/src/common/config/slack-config.service.ts delete mode 100644 packages/apps/job-launcher/server/src/common/config/vision-config.service.ts delete mode 100644 packages/apps/job-launcher/server/src/common/enums/content-moderation.ts delete mode 100644 packages/apps/job-launcher/server/src/common/enums/gcv.ts delete mode 100644 packages/apps/job-launcher/server/src/common/utils/gcstorage.spec.ts delete mode 100644 packages/apps/job-launcher/server/src/common/utils/gcstorage.ts create mode 100644 packages/apps/job-launcher/server/src/database/migrations/1774453578372-removeContentModeration.ts delete mode 100644 packages/apps/job-launcher/server/src/modules/content-moderation/content-moderation-request.entity.ts delete mode 100644 packages/apps/job-launcher/server/src/modules/content-moderation/content-moderation-request.repository.ts delete mode 100644 packages/apps/job-launcher/server/src/modules/content-moderation/content-moderation.dto.ts delete mode 100644 packages/apps/job-launcher/server/src/modules/content-moderation/content-moderation.interface.ts delete mode 100644 packages/apps/job-launcher/server/src/modules/content-moderation/content-moderation.module.ts delete mode 100644 packages/apps/job-launcher/server/src/modules/content-moderation/gcv-content-moderation.service.spec.ts delete mode 100644 packages/apps/job-launcher/server/src/modules/content-moderation/gcv-content-moderation.service.ts delete mode 100644 packages/apps/job-launcher/server/src/modules/routing-protocol/routing-protocol.interface.ts delete mode 100644 packages/apps/job-launcher/server/src/modules/routing-protocol/routing-protocol.module.ts delete mode 100644 packages/apps/job-launcher/server/src/modules/routing-protocol/routing-protocol.service.spec.ts delete mode 100644 packages/apps/job-launcher/server/src/modules/routing-protocol/routing-protocol.service.ts diff --git a/packages/apps/fortune/exchange-oracle/server/src/modules/assignment/assignment.service.spec.ts b/packages/apps/fortune/exchange-oracle/server/src/modules/assignment/assignment.service.spec.ts index 0144bac268..ea909e9642 100644 --- a/packages/apps/fortune/exchange-oracle/server/src/modules/assignment/assignment.service.spec.ts +++ b/packages/apps/fortune/exchange-oracle/server/src/modules/assignment/assignment.service.spec.ts @@ -102,7 +102,6 @@ describe('AssignmentService', () => { requesterTitle: 'Example Title', requesterDescription: 'Example Description', submissionsRequired: 5, - fundAmount: 100, }; beforeAll(async () => { @@ -128,6 +127,7 @@ describe('AssignmentService', () => { .mockResolvedValue(null); jest.spyOn(assignmentRepository, 'countByJobId').mockResolvedValue(0); jest.spyOn(jobService, 'getManifest').mockResolvedValue(manifest); + jest.spyOn(jobService, 'getRewardAmount').mockResolvedValue(20); (Escrow__factory.connect as any).mockImplementation(() => ({ duration: jest .fn() @@ -150,13 +150,18 @@ describe('AssignmentService', () => { workerAddress: workerAddress, status: AssignmentStatus.ACTIVE, expiresAt: expect.any(Date), - rewardAmount: manifest.fundAmount / manifest.submissionsRequired, + rewardAmount: 20, }); expect(jobService.getManifest).toHaveBeenCalledWith( chainId, escrowAddress, MOCK_MANIFEST_URL, ); + expect(jobService.getRewardAmount).toHaveBeenCalledWith( + chainId, + escrowAddress, + manifest.submissionsRequired, + ); }); it('should reassign user who has previously canceled', async () => { @@ -371,7 +376,6 @@ describe('AssignmentService', () => { requesterTitle: 'Example Title', requesterDescription: 'Example Description', submissionsRequired: 5, - fundAmount: 100, }; jest.spyOn(jobService, 'getManifest').mockResolvedValue(manifest); diff --git a/packages/apps/fortune/exchange-oracle/server/src/modules/assignment/assignment.service.ts b/packages/apps/fortune/exchange-oracle/server/src/modules/assignment/assignment.service.ts index 40fc11371c..824f688199 100644 --- a/packages/apps/fortune/exchange-oracle/server/src/modules/assignment/assignment.service.ts +++ b/packages/apps/fortune/exchange-oracle/server/src/modules/assignment/assignment.service.ts @@ -79,6 +79,11 @@ export class AssignmentService { data.escrowAddress, jobEntity.manifestUrl, ); + const rewardAmount = await this.jobService.getRewardAmount( + data.chainId, + data.escrowAddress, + manifest.submissionsRequired, + ); // Check if all required qualifications are present const userQualificationsSet = new Set(jwtUser.qualifications); @@ -110,8 +115,7 @@ export class AssignmentService { newAssignmentEntity.job = jobEntity; newAssignmentEntity.workerAddress = jwtUser.address; newAssignmentEntity.status = AssignmentStatus.ACTIVE; - newAssignmentEntity.rewardAmount = - manifest.fundAmount / manifest.submissionsRequired; + newAssignmentEntity.rewardAmount = rewardAmount; newAssignmentEntity.expiresAt = expirationDate; return this.assignmentRepository.createUnique(newAssignmentEntity); } diff --git a/packages/apps/fortune/exchange-oracle/server/src/modules/job/job.dto.ts b/packages/apps/fortune/exchange-oracle/server/src/modules/job/job.dto.ts index 106c506564..3a32f1f92a 100644 --- a/packages/apps/fortune/exchange-oracle/server/src/modules/job/job.dto.ts +++ b/packages/apps/fortune/exchange-oracle/server/src/modules/job/job.dto.ts @@ -20,7 +20,6 @@ export class ManifestDto { requesterTitle: string; requesterDescription: string; submissionsRequired: number; - fundAmount: number; qualifications?: string[]; } diff --git a/packages/apps/fortune/exchange-oracle/server/src/modules/job/job.service.spec.ts b/packages/apps/fortune/exchange-oracle/server/src/modules/job/job.service.spec.ts index d95fbda020..ce2922f246 100644 --- a/packages/apps/fortune/exchange-oracle/server/src/modules/job/job.service.spec.ts +++ b/packages/apps/fortune/exchange-oracle/server/src/modules/job/job.service.spec.ts @@ -381,7 +381,6 @@ describe('JobService', () => { requesterTitle: 'Example Title', requesterDescription: 'Example Description', submissionsRequired: 5, - fundAmount: 100, }; jest.spyOn(jobService, 'getManifest').mockResolvedValue(manifest); @@ -481,7 +480,6 @@ describe('JobService', () => { requesterTitle: 'Example Title', requesterDescription: 'Example Description', submissionsRequired: 5, - fundAmount: 100, }; jest @@ -537,7 +535,6 @@ describe('JobService', () => { requesterTitle: 'Example Title', requesterDescription: 'Example Description', submissionsRequired: 1, - fundAmount: 100, }; assignment.status = AssignmentStatus.ACTIVE; @@ -569,7 +566,6 @@ describe('JobService', () => { requesterTitle: 'Example Title', requesterDescription: 'Example Description', submissionsRequired: 5, - fundAmount: 100, }; assignment.status = AssignmentStatus.ACTIVE; @@ -605,7 +601,6 @@ describe('JobService', () => { requesterTitle: 'Example Title', requesterDescription: 'Example Description', submissionsRequired: 5, - fundAmount: 100, }; downloadFileFromUrlMock.mockResolvedValue(JSON.stringify(manifest)); @@ -626,7 +621,6 @@ describe('JobService', () => { requesterTitle: 'Example Title', requesterDescription: 'Example Description', submissionsRequired: 5, - fundAmount: 100, }; downloadFileFromUrlMock.mockResolvedValue('encrypted-content'); @@ -650,7 +644,6 @@ describe('JobService', () => { requesterTitle: 'Example Title', requesterDescription: 'Example Description', submissionsRequired: 5, - fundAmount: 100, }; downloadFileFromUrlMock.mockResolvedValue(manifest); diff --git a/packages/apps/fortune/exchange-oracle/server/src/modules/job/job.service.ts b/packages/apps/fortune/exchange-oracle/server/src/modules/job/job.service.ts index 1c60b0df81..cdc708cf51 100644 --- a/packages/apps/fortune/exchange-oracle/server/src/modules/job/job.service.ts +++ b/packages/apps/fortune/exchange-oracle/server/src/modules/job/job.service.ts @@ -7,10 +7,11 @@ import { Encryption, EncryptionUtils, EscrowClient, + EscrowUtils, } from '@human-protocol/sdk'; import { Inject, Injectable } from '@nestjs/common'; +import { ethers } from 'ethers'; -import { downloadFileFromUrl } from '../../common/utils/storage'; import { PGPConfigService } from '../../common/config/pgp-config.service'; import { ErrorAssignment, ErrorJob } from '../../common/constant/errors'; import { SortDirection } from '../../common/enums/collection'; @@ -30,6 +31,7 @@ import { } from '../../common/errors'; import { ISolution } from '../../common/interfaces/job'; import { PageDto } from '../../common/pagination/pagination.dto'; +import { downloadFileFromUrl } from '../../common/utils/storage'; import { AssignmentEntity } from '../assignment/assignment.entity'; import { AssignmentRepository } from '../assignment/assignment.repository'; import { StorageService } from '../storage/storage.service'; @@ -186,7 +188,11 @@ export class JobService { data.sortField === JobSortField.REWARD_AMOUNT ) { job.rewardAmount = ( - manifest.fundAmount / manifest.submissionsRequired + await this.getRewardAmount( + entity.chainId, + entity.escrowAddress, + manifest.submissionsRequired, + ) ).toString(); } if (data.fields?.includes(JobFieldName.RewardToken)) { @@ -411,4 +417,25 @@ export class JobService { return manifest; } + + public async getRewardAmount( + chainId: number, + escrowAddress: string, + submissionsRequired: number, + ): Promise { + const escrow = await EscrowUtils.getEscrow(chainId, escrowAddress); + if (!escrow) { + throw new NotFoundError(ErrorJob.NotFound); + } + + const decimals = await HMToken__factory.connect( + escrow.token, + this.web3Service.getSigner(chainId), + ).decimals(); + + return ( + Number(ethers.formatUnits(escrow.totalFundedAmount, decimals)) / + submissionsRequired + ); + } } diff --git a/packages/apps/fortune/exchange-oracle/server/src/modules/web3/web3.service.ts b/packages/apps/fortune/exchange-oracle/server/src/modules/web3/web3.service.ts index 99fb7413da..e4b5646762 100644 --- a/packages/apps/fortune/exchange-oracle/server/src/modules/web3/web3.service.ts +++ b/packages/apps/fortune/exchange-oracle/server/src/modules/web3/web3.service.ts @@ -19,7 +19,6 @@ export class Web3Service { private signers: { [key: number]: Wallet } = {}; readonly signerAddress: string; - readonly currentWeb3Env: string; constructor( private readonly web3ConfigService: Web3ConfigService, diff --git a/packages/apps/fortune/recording-oracle/src/common/interfaces/job.ts b/packages/apps/fortune/recording-oracle/src/common/interfaces/job.ts index dd67a36aa4..05b023fec6 100644 --- a/packages/apps/fortune/recording-oracle/src/common/interfaces/job.ts +++ b/packages/apps/fortune/recording-oracle/src/common/interfaces/job.ts @@ -4,7 +4,6 @@ export interface IManifest { submissionsRequired: number; requesterTitle: string; requesterDescription: string; - fundAmount: string; requestType: JobRequestType; } diff --git a/packages/apps/fortune/recording-oracle/src/modules/job/job.service.spec.ts b/packages/apps/fortune/recording-oracle/src/modules/job/job.service.spec.ts index e9094e4d21..b1409a1efa 100644 --- a/packages/apps/fortune/recording-oracle/src/modules/job/job.service.spec.ts +++ b/packages/apps/fortune/recording-oracle/src/modules/job/job.service.spec.ts @@ -34,7 +34,6 @@ import { StorageService } from '../storage/storage.service'; import { Web3Service } from '../web3/web3.service'; import { WebhookDto } from '../webhook/webhook.dto'; import { JobService } from './job.service'; -import { HMToken__factory } from '@human-protocol/core/typechain-types'; import { downloadFileFromUrl } from '@/common/utils/storage'; jest.mock('minio', () => { @@ -61,6 +60,11 @@ jest.mock('@human-protocol/sdk', () => ({ EscrowClient: { build: jest.fn().mockImplementation(() => ({})), }, + EscrowUtils: { + getEscrow: jest.fn().mockResolvedValue({ + totalFundedAmount: 8n, + }), + }, KVStoreUtils: { get: jest.fn(), getPublicKey: jest.fn().mockResolvedValue('publicKey'), @@ -130,13 +134,6 @@ describe('JobService', () => { }); describe('processJobSolution', () => { - beforeAll(() => { - const decimalsMock = jest.fn().mockResolvedValue(18); - const tokenContractMock = { decimals: decimalsMock }; - jest - .spyOn(HMToken__factory, 'connect') - .mockReturnValue(tokenContractMock as any); - }); afterEach(() => { jest.clearAllMocks(); }); @@ -241,7 +238,6 @@ describe('JobService', () => { submissionsRequired: 2, requesterTitle: MOCK_REQUESTER_TITLE, requesterDescription: MOCK_REQUESTER_DESCRIPTION, - fundAmount: '10', requestType: JobRequestType.FORTUNE, }; @@ -303,7 +299,6 @@ describe('JobService', () => { submissionsRequired: 2, requesterTitle: MOCK_REQUESTER_TITLE, requesterDescription: MOCK_REQUESTER_DESCRIPTION, - fundAmount: '10', requestType: JobRequestType.FORTUNE, }; @@ -316,7 +311,6 @@ describe('JobService', () => { .fn() .mockResolvedValue('http://example.com/results'), storeResults: jest.fn().mockResolvedValue(true), - getTokenAddress: jest.fn().mockResolvedValue(MOCK_ADDRESS), }; (EscrowClient.build as jest.Mock).mockResolvedValue(escrowClient); @@ -370,7 +364,6 @@ describe('JobService', () => { getManifest: jest.fn().mockResolvedValue('http://example.com/manifest'), getIntermediateResultsUrl: jest.fn().mockResolvedValue(''), storeResults: jest.fn().mockResolvedValue(true), - getTokenAddress: jest.fn().mockResolvedValue(MOCK_ADDRESS), }; (EscrowClient.build as jest.Mock).mockResolvedValue(escrowClient); @@ -378,7 +371,6 @@ describe('JobService', () => { submissionsRequired: 3, requesterTitle: MOCK_REQUESTER_TITLE, requesterDescription: MOCK_REQUESTER_DESCRIPTION, - fundAmount: '10', requestType: JobRequestType.FORTUNE, }; @@ -426,7 +418,6 @@ describe('JobService', () => { .fn() .mockResolvedValue('http://existing-solutions'), storeResults: jest.fn().mockResolvedValue(true), - getTokenAddress: jest.fn().mockResolvedValue(MOCK_ADDRESS), }; (EscrowClient.build as jest.Mock).mockResolvedValue(escrowClient); @@ -438,7 +429,6 @@ describe('JobService', () => { submissionsRequired: 2, requesterTitle: MOCK_REQUESTER_TITLE, requesterDescription: MOCK_REQUESTER_DESCRIPTION, - fundAmount: '10', requestType: JobRequestType.FORTUNE, }; @@ -504,7 +494,6 @@ describe('JobService', () => { .fn() .mockResolvedValue('http://existing-solutions'), storeResults: jest.fn().mockResolvedValue(true), - getTokenAddress: jest.fn().mockResolvedValue(MOCK_ADDRESS), }; (EscrowClient.build as jest.Mock).mockResolvedValue(escrowClient); @@ -512,7 +501,6 @@ describe('JobService', () => { submissionsRequired: 4, requesterTitle: MOCK_REQUESTER_TITLE, requesterDescription: MOCK_REQUESTER_DESCRIPTION, - fundAmount: '10', requestType: JobRequestType.FORTUNE, }; @@ -594,7 +582,6 @@ describe('JobService', () => { .fn() .mockResolvedValue('http://existing-solutions'), storeResults: jest.fn().mockResolvedValue(true), - getTokenAddress: jest.fn().mockResolvedValue(MOCK_ADDRESS), }; (EscrowClient.build as jest.Mock).mockResolvedValue(escrowClient); KVStoreUtils.get = jest @@ -606,7 +593,6 @@ describe('JobService', () => { submissionsRequired: 3, requesterTitle: MOCK_REQUESTER_TITLE, requesterDescription: MOCK_REQUESTER_DESCRIPTION, - fundAmount: '10', requestType: JobRequestType.FORTUNE, }; @@ -676,7 +662,6 @@ describe('JobService', () => { .fn() .mockResolvedValue('http://existing-solutions'), storeResults: jest.fn().mockResolvedValue(true), - getTokenAddress: jest.fn().mockResolvedValue(MOCK_ADDRESS), }; (EscrowClient.build as jest.Mock).mockResolvedValue(escrowClient); KVStoreUtils.get = jest @@ -688,7 +673,6 @@ describe('JobService', () => { submissionsRequired: 3, requesterTitle: MOCK_REQUESTER_TITLE, requesterDescription: MOCK_REQUESTER_DESCRIPTION, - fundAmount: '10', requestType: JobRequestType.FORTUNE, }; diff --git a/packages/apps/fortune/recording-oracle/src/modules/job/job.service.ts b/packages/apps/fortune/recording-oracle/src/modules/job/job.service.ts index cccaa60244..5d348d5057 100644 --- a/packages/apps/fortune/recording-oracle/src/modules/job/job.service.ts +++ b/packages/apps/fortune/recording-oracle/src/modules/job/job.service.ts @@ -3,6 +3,7 @@ import { EscrowStatus, KVStoreKeys, KVStoreUtils, + EscrowUtils, } from '@human-protocol/sdk'; import { HttpService } from '@nestjs/axios'; import { Inject, Injectable } from '@nestjs/common'; @@ -25,7 +26,6 @@ import { SolutionEventData, WebhookDto, } from '../webhook/webhook.dto'; -import { HMToken__factory } from '@human-protocol/core/typechain-types'; @Injectable() export class JobService { @@ -121,7 +121,7 @@ export class JobService { } const manifestUrl = await escrowClient.getManifest(webhook.escrowAddress); - const { submissionsRequired, requestType, fundAmount }: IManifest = + const { submissionsRequired, requestType }: IManifest = await this.storageService.download(manifestUrl); if (!submissionsRequired || !requestType) { @@ -186,16 +186,16 @@ export class JobService { s.solution === lastExchangeSolution.solution, ); - const tokenAddress = await escrowClient.getTokenAddress( + const escrow = await EscrowUtils.getEscrow( + webhook.chainId, webhook.escrowAddress, ); - const tokenContract = HMToken__factory.connect( - tokenAddress, - this.web3Service.getSigner(webhook.chainId), - ); - const decimals = await tokenContract.decimals(); - const fundAmountInWei = ethers.parseUnits(fundAmount.toString(), decimals); - const amountToReserve = fundAmountInWei / BigInt(submissionsRequired); + if (!escrow) { + throw new ValidationError('Escrow not found'); + } + + const amountToReserve = + escrow.totalFundedAmount / BigInt(submissionsRequired); await escrowClient.storeResults( webhook.escrowAddress, diff --git a/packages/apps/fortune/recording-oracle/test/constants.ts b/packages/apps/fortune/recording-oracle/test/constants.ts index 88e8b55de9..51488c045b 100644 --- a/packages/apps/fortune/recording-oracle/test/constants.ts +++ b/packages/apps/fortune/recording-oracle/test/constants.ts @@ -37,7 +37,6 @@ export const MOCK_MANIFEST: IManifest = { submissionsRequired: 2, requesterTitle: 'Fortune', requesterDescription: 'Some desc', - fundAmount: '8', requestType: JobRequestType.FORTUNE, }; export const MOCK_ENCRYPTION_PRIVATE_KEY = 'private-key'; diff --git a/packages/apps/job-launcher/client/src/components/Jobs/Create/CvatJobRequestForm.tsx b/packages/apps/job-launcher/client/src/components/Jobs/Create/CvatJobRequestForm.tsx index 10c649ee37..71094902e1 100644 --- a/packages/apps/job-launcher/client/src/components/Jobs/Create/CvatJobRequestForm.tsx +++ b/packages/apps/job-launcher/client/src/components/Jobs/Create/CvatJobRequestForm.tsx @@ -99,6 +99,7 @@ export const CvatJobRequestForm = () => { gtPath, userGuide, accuracyTarget, + jobBounty, }: ReturnType) => { let bp = undefined; if (type === CvatJobType.IMAGE_BOXES_FROM_POINTS) { @@ -154,7 +155,8 @@ export const CvatJobRequestForm = () => { path: gtPath, }, userGuide, - accuracyTarget, + accuracyTarget: Number(accuracyTarget), + jobBounty: Number(jobBounty), }, }); goToNextStep(); @@ -830,6 +832,35 @@ export const CvatJobRequestForm = () => { /> + + + + setFieldValue('jobBounty', e.target.value) + } + onBlur={handleBlur} + error={touched.jobBounty && Boolean(errors.jobBounty)} + helperText={errors.jobBounty} + InputProps={{ + endAdornment: ( + + + + + + ), + }} + /> + + diff --git a/packages/apps/job-launcher/client/src/components/Jobs/Create/helpers.ts b/packages/apps/job-launcher/client/src/components/Jobs/Create/helpers.ts index cd1fe2e2b0..6ccc3ba2b4 100644 --- a/packages/apps/job-launcher/client/src/components/Jobs/Create/helpers.ts +++ b/packages/apps/job-launcher/client/src/components/Jobs/Create/helpers.ts @@ -24,6 +24,7 @@ export const mapCvatFormValues = ( : [], userGuide: cvatRequest?.userGuide || '', accuracyTarget: cvatRequest?.accuracyTarget || 80, + jobBounty: cvatRequest?.jobBounty || 0, dataProvider: cvatRequest?.data?.dataset?.provider || StorageProviders.AWS, dataRegion: (cvatRequest?.data?.dataset?.region as AWSRegions | GCSRegions) || '', diff --git a/packages/apps/job-launcher/client/src/components/Jobs/Create/schema.ts b/packages/apps/job-launcher/client/src/components/Jobs/Create/schema.ts index 85884c5fba..d904d72557 100644 --- a/packages/apps/job-launcher/client/src/components/Jobs/Create/schema.ts +++ b/packages/apps/job-launcher/client/src/components/Jobs/Create/schema.ts @@ -19,6 +19,10 @@ export const CvatJobRequestValidationSchema = Yup.object().shape({ .required('Accuracy target is required') .moreThan(0, 'Accuracy target must be greater than 0') .max(100, 'Accuracy target must be less than or equal to 100'), + jobBounty: Yup.number() + .typeError('Job bounty is required') + .required('Job bounty is required') + .moreThan(0, 'Job bounty must be greater than 0'), qualifications: Yup.array().of(Yup.object()), }); diff --git a/packages/apps/job-launcher/client/src/constants/cvat.ts b/packages/apps/job-launcher/client/src/constants/cvat.ts new file mode 100644 index 0000000000..deb5bd61e7 --- /dev/null +++ b/packages/apps/job-launcher/client/src/constants/cvat.ts @@ -0,0 +1,2 @@ +export const CVAT_JOB_SIZE = 10; +export const CVAT_VAL_SIZE = 2; diff --git a/packages/apps/job-launcher/client/src/services/job.ts b/packages/apps/job-launcher/client/src/services/job.ts index ceb719f492..a6d2c52184 100644 --- a/packages/apps/job-launcher/client/src/services/job.ts +++ b/packages/apps/job-launcher/client/src/services/job.ts @@ -1,16 +1,71 @@ import { ChainId } from '@human-protocol/sdk'; +import { CVAT_JOB_SIZE, CVAT_VAL_SIZE } from '../constants/cvat'; import { - CreateFortuneJobRequest, - CreateCvatJobRequest, + CreateJobRequest, FortuneRequest, CvatRequest, JobStatus, JobDetailsResponse, FortuneFinalResult, + FortuneManifest, + CvatManifest, + JobType, + StorageProviders, } from '../types'; import api from '../utils/api'; import { getFilenameFromContentDisposition } from '../utils/string'; +const buildFortuneManifest = (data: FortuneRequest): FortuneManifest => ({ + submissionsRequired: Number(data.fortunesRequested), + requesterTitle: data.title, + requesterDescription: data.description, + requestType: JobType.FORTUNE, + qualifications: data.qualifications, +}); + +const buildBucketUrl = ({ + provider, + region, + bucketName, + path, +}: CvatRequest['data']['dataset']) => { + if (provider === StorageProviders.AWS) { + return `https://${bucketName}.s3.${region}.amazonaws.com${ + path ? `/${path.replace(/\/$/, '')}` : '' + }`; + } + + return `https://${bucketName}.storage.googleapis.com${path ? `/${path}` : ''}`; +}; + +const buildCvatManifest = (data: CvatRequest): CvatManifest => ({ + data: { + dataUrl: buildBucketUrl(data.data.dataset), + ...(data.data.points && { + pointsUrl: buildBucketUrl(data.data.points), + }), + ...(data.data.boxes && { + boxesUrl: buildBucketUrl(data.data.boxes), + }), + }, + annotation: { + labels: data.labels, + description: data.description, + userGuide: data.userGuide, + type: data.type, + jobSize: CVAT_JOB_SIZE, + ...(data.qualifications?.length && { + qualifications: data.qualifications, + }), + }, + validation: { + minQuality: Number(data.accuracyTarget) / 100, + valSize: CVAT_VAL_SIZE, + gtUrl: buildBucketUrl(data.groundTruth), + }, + jobBounty: String(data.jobBounty), +}); + export const createFortuneJob = async ( chainId: number, data: FortuneRequest, @@ -18,17 +73,16 @@ export const createFortuneJob = async ( paymentAmount: number | string, escrowFundToken: string, ) => { - const body: CreateFortuneJobRequest = { + const body: CreateJobRequest = { chainId, - submissionsRequired: Number(data.fortunesRequested), - requesterTitle: data.title, - requesterDescription: data.description, + requestType: JobType.FORTUNE, paymentCurrency, paymentAmount: Number(paymentAmount), escrowFundToken, qualifications: data.qualifications, + manifest: buildFortuneManifest(data), }; - await api.post('/job/fortune', body); + await api.post('/job', body); }; export const createCvatJob = async ( @@ -38,21 +92,16 @@ export const createCvatJob = async ( paymentAmount: number | string, escrowFundToken: string, ) => { - const body: CreateCvatJobRequest = { + const body: CreateJobRequest = { chainId, - requesterDescription: data.description, + requestType: data.type, paymentCurrency, paymentAmount: Number(paymentAmount), escrowFundToken, - data: data.data, - labels: data.labels, - minQuality: Number(data.accuracyTarget) / 100, - groundTruth: data.groundTruth, - userGuide: data.userGuide, - type: data.type, qualifications: data.qualifications, + manifest: buildCvatManifest(data), }; - await api.post('/job/cvat', body); + await api.post('/job', body); }; export const getJobList = async ({ diff --git a/packages/apps/job-launcher/client/src/types/index.ts b/packages/apps/job-launcher/client/src/types/index.ts index 0ca3d5a11b..2ca082c754 100644 --- a/packages/apps/job-launcher/client/src/types/index.ts +++ b/packages/apps/job-launcher/client/src/types/index.ts @@ -39,30 +39,46 @@ export type FiatPaymentRequest = { paymentMethodId: string; }; -export type CreateFortuneJobRequest = { - chainId: number; +export type FortuneManifest = { submissionsRequired: number; requesterTitle: string; requesterDescription: string; - paymentCurrency: string; - paymentAmount: number; - escrowFundToken: string; + requestType: JobType.FORTUNE; qualifications?: string[]; }; -export type CreateCvatJobRequest = { +export type JobRequestType = JobType.FORTUNE | JobType.HCAPTCHA | CvatJobType; + +export type CreateJobRequest> = { chainId: number; - requesterDescription: string; - qualifications?: string[]; + requestType: JobRequestType; paymentCurrency: string; paymentAmount: number; escrowFundToken: string; - data: CvatData; - labels: Label[]; - minQuality: number; - groundTruth: CvatDataSource; - userGuide: string; - type: CvatJobType; + qualifications?: string[]; + manifest: TManifest; +}; + +export type CvatManifest = { + data: { + dataUrl: string; + pointsUrl?: string; + boxesUrl?: string; + }; + annotation: { + labels: Label[]; + description: string; + userGuide: string; + type: CvatJobType; + jobSize: number; + qualifications?: string[]; + }; + validation: { + minQuality: number; + valSize: number; + gtUrl: string; + }; + jobBounty: string; }; export enum CreateJobStep { @@ -215,6 +231,7 @@ export type CvatRequest = { groundTruth: CvatDataSource; userGuide: string; accuracyTarget: number; + jobBounty: number; }; export type JobRequest = { diff --git a/packages/apps/job-launcher/server/.env.example b/packages/apps/job-launcher/server/.env.example index 55590e127e..1541e5f6d8 100644 --- a/packages/apps/job-launcher/server/.env.example +++ b/packages/apps/job-launcher/server/.env.example @@ -89,13 +89,3 @@ PAYMENT_PROVIDER_APP_INFO_URL=http://local.app # Sendgrid SENDGRID_API_KEY=sendgrid-disabled - -# Vision -GOOGLE_PROJECT_ID=disabled -GOOGLE_PRIVATE_KEY=disabled -GOOGLE_CLIENT_EMAIL=disabled -GCV_MODERATION_RESULTS_FILES_PATH=disabled -GCV_MODERATION_RESULTS_BUCKET=disabled - -# Slack -SLACK_ABUSE_NOTIFICATION_WEBHOOK_URL=disabled diff --git a/packages/apps/job-launcher/server/src/common/config/config.module.ts b/packages/apps/job-launcher/server/src/common/config/config.module.ts index 82692b8851..189136e888 100644 --- a/packages/apps/job-launcher/server/src/common/config/config.module.ts +++ b/packages/apps/job-launcher/server/src/common/config/config.module.ts @@ -11,8 +11,6 @@ import { S3ConfigService } from './s3-config.service'; import { SendgridConfigService } from './sendgrid-config.service'; import { PaymentProviderConfigService } from './payment-provider-config.service'; import { Web3ConfigService } from './web3-config.service'; -import { SlackConfigService } from './slack-config.service'; -import { VisionConfigService } from './vision-config.service'; @Global() @Module({ @@ -28,8 +26,6 @@ import { VisionConfigService } from './vision-config.service'; CvatConfigService, PGPConfigService, NetworkConfigService, - SlackConfigService, - VisionConfigService, ], exports: [ ConfigService, @@ -43,8 +39,6 @@ import { VisionConfigService } from './vision-config.service'; CvatConfigService, PGPConfigService, NetworkConfigService, - SlackConfigService, - VisionConfigService, ], }) export class EnvConfigModule {} diff --git a/packages/apps/job-launcher/server/src/common/config/env-schema.ts b/packages/apps/job-launcher/server/src/common/config/env-schema.ts index 195f9becb1..edd5b41a68 100644 --- a/packages/apps/job-launcher/server/src/common/config/env-schema.ts +++ b/packages/apps/job-launcher/server/src/common/config/env-schema.ts @@ -30,7 +30,6 @@ export const envValidator = Joi.object({ GAS_PRICE_MULTIPLIER: Joi.number(), APPROVE_AMOUNT_USD: Joi.number(), REPUTATION_ORACLE_ADDRESS: Joi.string().required(), - REPUTATION_ORACLES: Joi.string().required(), CVAT_EXCHANGE_ORACLE_ADDRESS: Joi.string().required(), CVAT_RECORDING_ORACLE_ADDRESS: Joi.string().required(), HCAPTCHA_ORACLE_ADDRESS: Joi.string().required(), @@ -67,11 +66,6 @@ export const envValidator = Joi.object({ SENDGRID_API_KEY: Joi.string().required(), SENDGRID_FROM_EMAIL: Joi.string(), SENDGRID_FROM_NAME: Joi.string(), - // CVAT - CVAT_JOB_SIZE: Joi.string(), - CVAT_MAX_TIME: Joi.string(), - CVAT_VAL_SIZE: Joi.string(), - CVAT_SKELETONS_JOB_SIZE_MULTIPLIER: Joi.string(), //PGP PGP_ENCRYPT: Joi.boolean(), PGP_PRIVATE_KEY: Joi.string().optional(), @@ -82,12 +76,4 @@ export const envValidator = Joi.object({ //COIN API KEYS RATE_CACHE_TIME: Joi.number().optional(), COINGECKO_API_KEY: Joi.string().optional(), - // Google - GOOGLE_PROJECT_ID: Joi.string().required(), - GOOGLE_PRIVATE_KEY: Joi.string().required(), - GOOGLE_CLIENT_EMAIL: Joi.string().required(), - GCV_MODERATION_RESULTS_FILES_PATH: Joi.string().required(), - GCV_MODERATION_RESULTS_BUCKET: Joi.string().required(), - // Slack - SLACK_ABUSE_NOTIFICATION_WEBHOOK_URL: Joi.string().required(), }); diff --git a/packages/apps/job-launcher/server/src/common/config/slack-config.service.ts b/packages/apps/job-launcher/server/src/common/config/slack-config.service.ts deleted file mode 100644 index 4bccf51aee..0000000000 --- a/packages/apps/job-launcher/server/src/common/config/slack-config.service.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { Injectable } from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; - -@Injectable() -export class SlackConfigService { - constructor(private configService: ConfigService) {} - - /** - * The abuse notification webhook URL for sending messages to a Slack channel. - * Required - */ - get abuseNotificationWebhookUrl(): string { - return this.configService.getOrThrow( - 'SLACK_ABUSE_NOTIFICATION_WEBHOOK_URL', - ); - } -} diff --git a/packages/apps/job-launcher/server/src/common/config/vision-config.service.ts b/packages/apps/job-launcher/server/src/common/config/vision-config.service.ts deleted file mode 100644 index 3d39b184b6..0000000000 --- a/packages/apps/job-launcher/server/src/common/config/vision-config.service.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { Injectable } from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; - -@Injectable() -export class VisionConfigService { - constructor(private configService: ConfigService) {} - - /** - * The Google Cloud Storage (GCS) path name where temporary async moderation results will be saved. - * Required - */ - get moderationResultsFilesPath(): string { - return this.configService.getOrThrow( - 'GCV_MODERATION_RESULTS_FILES_PATH', - ); - } - - /** - * The Google Cloud Storage (GCS) bucket name where moderation results will be saved. - * Required - */ - get moderationResultsBucket(): string { - return this.configService.getOrThrow( - 'GCV_MODERATION_RESULTS_BUCKET', - ); - } - - /** - * The project ID for connecting to the Google Cloud Vision API. - * Required - */ - get projectId(): string { - return this.configService.getOrThrow('GOOGLE_PROJECT_ID'); - } - - /** - * The private key for authenticating with the Google Cloud Vision API. - * Required - */ - get privateKey(): string { - return this.configService.getOrThrow('GOOGLE_PRIVATE_KEY'); - } - - /** - * The client email used for authenticating requests to the Google Cloud Vision API. - * Required - */ - get clientEmail(): string { - return this.configService.getOrThrow('GOOGLE_CLIENT_EMAIL'); - } -} diff --git a/packages/apps/job-launcher/server/src/common/config/web3-config.service.ts b/packages/apps/job-launcher/server/src/common/config/web3-config.service.ts index 9c5806f0f4..41011149fb 100644 --- a/packages/apps/job-launcher/server/src/common/config/web3-config.service.ts +++ b/packages/apps/job-launcher/server/src/common/config/web3-config.service.ts @@ -37,14 +37,6 @@ export class Web3ConfigService { return this.configService.getOrThrow('REPUTATION_ORACLE_ADDRESS'); } - /** - * List of reputation oracle addresses, typically comma-separated. - * Required - */ - get reputationOracles(): string { - return this.configService.getOrThrow('REPUTATION_ORACLES'); - } - /** * URI for the hCaptcha recording oracle service. * Required diff --git a/packages/apps/job-launcher/server/src/common/constants/errors.ts b/packages/apps/job-launcher/server/src/common/constants/errors.ts index a12a28d52a..e1e59e2a90 100644 --- a/packages/apps/job-launcher/server/src/common/constants/errors.ts +++ b/packages/apps/job-launcher/server/src/common/constants/errors.ts @@ -26,23 +26,6 @@ export enum ErrorJob { NoRefundFound = 'No refund found for this escrow', } -/** - * Represents error messages associated with a job moderation. - */ -export enum ErrorContentModeration { - ErrorProcessingDataset = 'Error processing dataset', - InappropriateContent = 'Job cannot be processed due to inappropriate content', - ContentModerationFailed = 'Job cannot be processed due to failure in content moderation', - NoDestinationURIFound = 'No destination URI found in the response', - InvalidBucketUrl = 'Invalid bucket URL', - DataMustBeStoredInGCS = 'Data must be stored in Google Cloud Storage', - NoResultsFound = 'No results found', - ResultsParsingFailed = 'Results parsing failed', - JobModerationFailed = 'Job moderation failed', - ProcessContentModerationRequestFailed = 'Process content moderation request failed', - CompleteContentModerationFailed = 'Complete content moderation failed', -} - /** * Represents error messages associated to webhook. */ @@ -166,15 +149,6 @@ export enum ErrorWeb3 { ReputationOracleUrlNotSet = 'Reputation oracle URL not set', } -/** - * Represents error messages related to routing protocol. - */ -export enum ErrorRoutingProtocol { - ReputationOracleNotFound = 'The specified Reputation Oracle address is not found in the set of available oracles. Ensure the address is correct and check available oracles for this network.', - ExchangeOracleNotFound = 'The specified Exchange Oracle address is not found in the set of available oracles. Ensure the address is correct and part of the available oracle pool.', - RecordingOracleNotFound = 'The specified Recording Oracle address is not found in the set of available oracles. Ensure the address is correct and part of the available oracle pool.', -} - /** * Represents error messages related to send grid. */ diff --git a/packages/apps/job-launcher/server/src/common/constants/index.ts b/packages/apps/job-launcher/server/src/common/constants/index.ts index 72ada66583..7bbb8b9a5e 100644 --- a/packages/apps/job-launcher/server/src/common/constants/index.ts +++ b/packages/apps/job-launcher/server/src/common/constants/index.ts @@ -66,6 +66,14 @@ export const LOGOUT_PATH = '/auth/logout'; export const MUTEX_TIMEOUT = 2000; //ms -export const GS_PROTOCOL = 'gs://'; -export const GCV_CONTENT_MODERATION_ASYNC_BATCH_SIZE = 100; -export const GCV_CONTENT_MODERATION_BATCH_SIZE_PER_TASK = 2000; +/** + * Regex for GCS URL in subdomain format: https://.storage.googleapis.com/ + */ +export const GCS_HTTP_REGEX_SUBDOMAIN = + /^https:\/\/([a-zA-Z0-9\-.]+)\.storage\.googleapis\.com\/?(.*)$/; + +/** + * Regex for GCS URL in path-based format: https://storage.googleapis.com// + */ +export const GCS_HTTP_REGEX_PATH_BASED = + /^https:\/\/storage\.googleapis\.com\/([^/]+)\/?(.*)$/; diff --git a/packages/apps/job-launcher/server/src/common/enums/content-moderation.ts b/packages/apps/job-launcher/server/src/common/enums/content-moderation.ts deleted file mode 100644 index d772b1774f..0000000000 --- a/packages/apps/job-launcher/server/src/common/enums/content-moderation.ts +++ /dev/null @@ -1,7 +0,0 @@ -export enum ContentModerationRequestStatus { - PENDING = 'pending', - PROCESSED = 'processed', - POSITIVE_ABUSE = 'positive_abuse', - PASSED = 'passed', - FAILED = 'failed', -} diff --git a/packages/apps/job-launcher/server/src/common/enums/cron-job.ts b/packages/apps/job-launcher/server/src/common/enums/cron-job.ts index 6cb71352ec..da9fd197f3 100644 --- a/packages/apps/job-launcher/server/src/common/enums/cron-job.ts +++ b/packages/apps/job-launcher/server/src/common/enums/cron-job.ts @@ -1,5 +1,4 @@ export enum CronJobType { - ContentModeration = 'content-moderation', CreateEscrow = 'create-escrow', CancelEscrow = 'cancel-escrow', ProcessPendingWebhook = 'process-pending-webhook', diff --git a/packages/apps/job-launcher/server/src/common/enums/gcv.ts b/packages/apps/job-launcher/server/src/common/enums/gcv.ts deleted file mode 100644 index 600b82abae..0000000000 --- a/packages/apps/job-launcher/server/src/common/enums/gcv.ts +++ /dev/null @@ -1,9 +0,0 @@ -export enum ContentModerationLevel { - VERY_LIKELY = 'VERY_LIKELY', - LIKELY = 'LIKELY', - POSSIBLE = 'POSSIBLE', -} - -export enum ContentModerationFeature { - SAFE_SEARCH_DETECTION = 'SAFE_SEARCH_DETECTION', -} diff --git a/packages/apps/job-launcher/server/src/common/enums/job.ts b/packages/apps/job-launcher/server/src/common/enums/job.ts index e2ad6ca78e..db16799d11 100644 --- a/packages/apps/job-launcher/server/src/common/enums/job.ts +++ b/packages/apps/job-launcher/server/src/common/enums/job.ts @@ -1,8 +1,5 @@ export enum JobStatus { PAID = 'paid', - UNDER_MODERATION = 'under_moderation', - MODERATION_PASSED = 'moderation_passed', - POSSIBLE_ABUSE_IN_REVIEW = 'possible_abuse_in_review', LAUNCHED = 'launched', PARTIAL = 'partial', COMPLETED = 'completed', diff --git a/packages/apps/job-launcher/server/src/common/utils/gcstorage.spec.ts b/packages/apps/job-launcher/server/src/common/utils/gcstorage.spec.ts deleted file mode 100644 index 5100ed5e18..0000000000 --- a/packages/apps/job-launcher/server/src/common/utils/gcstorage.spec.ts +++ /dev/null @@ -1,191 +0,0 @@ -import { - constructGcsPath, - convertToGCSPath, - convertToHttpUrl, - isGCSBucketUrl, -} from './gcstorage'; -import { ErrorBucket } from '../constants/errors'; - -describe('Google Cloud Storage utils', () => { - describe('isGCSBucketUrl', () => { - it('should return true for a valid GCS HTTP URL', () => { - expect( - isGCSBucketUrl( - 'https://valid-bucket-with-file.storage.googleapis.com/object.jpg', - ), - ).toBe(true); - expect( - isGCSBucketUrl('https://valid-bucket.storage.googleapis.com/'), - ).toBe(true); - expect( - isGCSBucketUrl('https://valid-bucket.storage.googleapis.com'), - ).toBe(true); - }); - - it('should return true for a valid GCS gs:// URL', () => { - expect(isGCSBucketUrl('gs://valid-bucket-with-file/object.jpg')).toBe( - true, - ); - expect(isGCSBucketUrl('gs://valid-bucket/')).toBe(true); - expect(isGCSBucketUrl('gs://valid-bucket')).toBe(true); - }); - - it('should return false for an invalid GCS HTTP URL', () => { - expect(isGCSBucketUrl('https://invalid-url.com/object.jpg')).toBe(false); - }); - - it('should return false for an invalid gs:// URL', () => { - expect(isGCSBucketUrl('gs:/invalid-bucket/object.jpg')).toBe(false); - }); - - it('should return false for a completely invalid URL', () => { - expect(isGCSBucketUrl('randomstring')).toBe(false); - }); - - it('should return false for a GCS URL with an invalid bucket name', () => { - expect(isGCSBucketUrl('https://_invalid.storage.googleapis.com')).toBe( - false, - ); - expect(isGCSBucketUrl('gs://sh.storage.googleapis.com')).toBe(false); - expect(isGCSBucketUrl('https://test-.storage.googleapis.com')).toBe( - false, - ); - expect(isGCSBucketUrl('https://-test.storage.googleapis.com')).toBe( - false, - ); - }); - }); - - describe('convertToGCSPath', () => { - it('should convert a valid GCS HTTP URL to a gs:// path', () => { - expect( - convertToGCSPath( - 'https://valid-bucket.storage.googleapis.com/object.jpg', - ), - ).toBe('gs://valid-bucket/object.jpg'); - }); - - it('should convert a valid GCS HTTP URL without an object path to a gs:// bucket path', () => { - expect( - convertToGCSPath('https://valid-bucket.storage.googleapis.com'), - ).toBe('gs://valid-bucket'); - - expect( - convertToGCSPath('https://valid-bucket.storage.googleapis.com/'), - ).toBe('gs://valid-bucket'); - }); - - it('should throw a Error for an invalid GCS URL', () => { - expect(() => - convertToGCSPath('https://invalid-url.com/object.jpg'), - ).toThrow(new Error(ErrorBucket.InvalidGCSUrl)); - }); - - it('should throw a Error for a URL with an invalid bucket name', () => { - expect(() => - convertToGCSPath('https://invalid_bucket.storage.googleapis.com'), - ).toThrow(new Error(ErrorBucket.InvalidGCSUrl)); - }); - }); - - describe('convertToHttpUrl', () => { - it('should convert a gs:// path to a valid HTTP URL', () => { - const result = convertToHttpUrl('gs://valid-bucket/object.jpg'); - expect(result).toBe( - 'https://valid-bucket.storage.googleapis.com/object.jpg', - ); - }); - - it('should convert a gs:// bucket path without an object to an HTTP bucket URL', () => { - expect(convertToHttpUrl('gs://valid-bucket/')).toBe( - 'https://valid-bucket.storage.googleapis.com/', - ); - expect(convertToHttpUrl('gs://valid-bucket')).toBe( - 'https://valid-bucket.storage.googleapis.com/', - ); - }); - - it('should throw a Error for an invalid gs:// path', () => { - expect(() => convertToHttpUrl('invalid-gcs-path')).toThrow( - new Error(ErrorBucket.InvalidGCSUrl), - ); - }); - - it('should throw a Error if the gs:// format is incorrect', () => { - expect(() => convertToHttpUrl('gs:/missing-slash/object.jpg')).toThrow( - new Error(ErrorBucket.InvalidGCSUrl), - ); - }); - - it('should throw a Error for an invalid bucket name in gs:// path', () => { - expect(() => convertToHttpUrl('gs://_invalid/object.jpg')).toThrow( - new Error(ErrorBucket.InvalidGCSUrl), - ); - expect(() => convertToHttpUrl('gs://test-/object.jpg')).toThrow( - new Error(ErrorBucket.InvalidGCSUrl), - ); - }); - }); - - describe('constructGcsPath', () => { - it('should correctly construct a GCS path with multiple segments', () => { - expect(constructGcsPath('my-bucket', 'folder', 'file.jpg')).toBe( - 'gs://my-bucket/folder/file.jpg', - ); - }); - - it('should handle leading and trailing slashes properly', () => { - expect(constructGcsPath('my-bucket/', '/folder/', '/file.jpg')).toBe( - 'gs://my-bucket/folder/file.jpg', - ); - }); - - it('should remove extra slashes and normalize path segments', () => { - expect( - constructGcsPath('my-bucket', '///folder///', '///file.jpg///'), - ).toBe('gs://my-bucket/folder/file.jpg'); - }); - - it('should handle cases where no additional paths are provided', () => { - expect(constructGcsPath('my-bucket')).toBe('gs://my-bucket'); - }); - - it('should handle empty segments gracefully', () => { - expect(constructGcsPath('my-bucket', '', 'file.jpg')).toBe( - 'gs://my-bucket/file.jpg', - ); - }); - - it('should construct a path with nested directories correctly', () => { - expect( - constructGcsPath('my-bucket', 'folder1', 'folder2', 'file.jpg'), - ).toBe('gs://my-bucket/folder1/folder2/file.jpg'); - }); - - it('should not add an extra slash if the base path already ends with one', () => { - expect(constructGcsPath('my-bucket/', 'file.jpg')).toBe( - 'gs://my-bucket/file.jpg', - ); - }); - - it('should correctly handle a single trailing slash in the base path', () => { - expect(constructGcsPath('my-bucket/', '')).toBe('gs://my-bucket'); - }); - - it('should correctly handle a bucket name that already includes gs://', () => { - expect(constructGcsPath('gs://my-bucket', 'folder', 'file.jpg')).toBe( - 'gs://my-bucket/folder/file.jpg', - ); - }); - - it('should correctly handle a bucket name with gs:// and a trailing slash', () => { - expect(constructGcsPath('gs://my-bucket/', 'folder', 'file.jpg')).toBe( - 'gs://my-bucket/folder/file.jpg', - ); - }); - - it('should handle paths that contain only slashes', () => { - expect(constructGcsPath('my-bucket', '/', '/')).toBe('gs://my-bucket'); - }); - }); -}); diff --git a/packages/apps/job-launcher/server/src/common/utils/gcstorage.ts b/packages/apps/job-launcher/server/src/common/utils/gcstorage.ts deleted file mode 100644 index 87f57e9086..0000000000 --- a/packages/apps/job-launcher/server/src/common/utils/gcstorage.ts +++ /dev/null @@ -1,193 +0,0 @@ -import { isURL } from 'validator'; -import { GS_PROTOCOL } from '../constants'; -import { ErrorBucket } from '../constants/errors'; - -// Step 1: Define your regular expressions, bucket validation, and URL validation helpers - -/** - * Regex for GCS URL in subdomain format: https://.storage.googleapis.com/ - */ -export const GCS_HTTP_REGEX_SUBDOMAIN = - /^https:\/\/([a-zA-Z0-9\-.]+)\.storage\.googleapis\.com\/?(.*)$/; - -/** - * Regex for GCS URL in path-based format: https://storage.googleapis.com// - */ -export const GCS_HTTP_REGEX_PATH_BASED = - /^https:\/\/storage\.googleapis\.com\/([^/]+)\/?(.*)$/; - -/** - * Regex for GCS URI format: gs:/// - */ -export const GCS_GS_REGEX = /^gs:\/\/([a-zA-Z0-9\-.]+)\/?(.*)$/; - -/** - * Regex that ensures the bucket name follows Google Cloud Storage (GCS) naming rules: - * - Must be between 3 and 63 characters long. - * - Can contain lowercase letters, numbers, dashes (`-`), and dots (`.`). - * - Cannot begin or end with a dash (`-`). - * - Cannot have consecutive periods (`..`). - * - Cannot resemble an IP address (e.g., "192.168.1.1"). - */ -const BUCKET_NAME_REGEX = /^[a-z0-9][a-z0-9-]{1,61}[a-z0-9]$/; - -// Step 2: Implement the main validation function - -/** - * Validates if a given URL is a valid Google Cloud Storage URL. - * - * Supports: - * - Subdomain format: https://.storage.googleapis.com[/] - * - Path-based format: https://storage.googleapis.com/[/] - * - GCS URI format: gs://[/] - * - * @param url - The URL to validate. - * @returns {boolean} - Returns true if the URL is valid, otherwise false. - */ -export function isGCSBucketUrl(url: string): boolean { - // 1) Quickly check if it's a valid URL in general - if (!isValidUrl(url)) { - return false; - } - - // 2) Try subdomain-based regex first - let httpMatch = url.match(GCS_HTTP_REGEX_SUBDOMAIN); - - // 3) If that fails, try path-based regex - if (!httpMatch) { - httpMatch = url.match(GCS_HTTP_REGEX_PATH_BASED); - } - - // 4) Also check if it matches the gs:// scheme - const gsMatch = url.match(GCS_GS_REGEX); - - // 5) If any HTTP or GS regex matched - if (httpMatch || gsMatch) { - // For HTTP matches, the bucket is captured in group [1]. - // For GS matches, it's also in group [1]. - const bucketName = httpMatch ? httpMatch[1] : gsMatch ? gsMatch[1] : null; - - if (!bucketName || !isValidBucketName(bucketName)) { - return false; - } - - return true; - } - - return false; -} - -/** - * Validates a URL to check if it is a valid Google Cloud Storage URL. - * This function ensures the URL is well-formed and its protocol is one of: - * - `http:` (HTTP URL) - * - `https:` (HTTPS URL) - * - `gs:` (Google Cloud Storage URI) - * - * @param maybeUrl The URL string to be validated. - * @returns A boolean indicating whether the URL is valid and has an allowed protocol. - */ -export function isValidUrl(maybeUrl: string): boolean { - try { - const url = new URL(maybeUrl); - if (url.protocol === 'gs:') { - return true; - } else { - return isURL(maybeUrl, { - require_protocol: true, - protocols: ['http', 'https'], - }); - } - } catch { - return false; - } -} - -/** - * Validates a Google Cloud Storage bucket name. - * GCS requires bucket names to: - * - Be 3-63 characters long - * - Contain only lowercase letters, numbers, dashes - * - Not start or end with a dash - */ -function isValidBucketName(bucket: string): boolean { - return BUCKET_NAME_REGEX.test(bucket); -} - -/** - * Converts a valid Google Cloud Storage HTTP URL to a GCS path. - * - * @param url - The HTTP URL to convert. - * @returns {string} - The converted GCS path. - * @throws Error - If the URL is not a valid GCS URL. - */ -export function convertToGCSPath(url: string): string { - if (!isGCSBucketUrl(url)) { - throw new Error(ErrorBucket.InvalidGCSUrl); - } - - let match = url.match(GCS_HTTP_REGEX_SUBDOMAIN); - let bucketName: string | null = null; - let objectPath: string | null = null; - - if (match) { - bucketName = match[1]; - objectPath = match[2] || ''; - } else { - match = url.match(GCS_HTTP_REGEX_PATH_BASED); - if (match) { - bucketName = match[1]; - objectPath = match[2] || ''; - } - } - - if (!bucketName) { - throw new Error(ErrorBucket.InvalidGCSUrl); - } - - let gcsPath = `gs://${bucketName}`; - if (objectPath) { - gcsPath += `/${objectPath}`; - } - return gcsPath; -} - -/** - * Converts a GCS path to a valid Google Cloud Storage HTTP URL. - * - * @param gcsPath - The GCS path to convert (e.g., "gs://bucket-name/object-path"). - * @returns {string} - The converted HTTP URL. - * @throws Error - If the GCS path is not valid. - */ -export function convertToHttpUrl(gcsPath: string): string { - if (!isGCSBucketUrl(gcsPath)) { - throw new Error(ErrorBucket.InvalidGCSUrl); - } - - const match = gcsPath.match(GCS_GS_REGEX); - - const bucketName = match![1]; - const objectPath = match![2] || ''; - - return `https://${bucketName}.storage.googleapis.com/${objectPath}`; -} - -/** - * Constructs a GCS path with a variable number of segments. - * - * @param bucket - The GCS bucket name (without `gs://`). - * @param paths - Additional path segments to append. - * @returns {string} - The constructed GCS path. - */ -export function constructGcsPath(bucket: string, ...paths: string[]): string { - const cleanBucket = bucket.replace(/^gs:\/\//, '').replace(/\/+$/, ''); - - const fullPath = paths - .map((segment) => segment.replace(/^\/+|\/+$/g, '')) - .filter((segment) => segment) - .join('/'); - - return fullPath - ? `${GS_PROTOCOL}${cleanBucket}/${fullPath}` - : `${GS_PROTOCOL}${cleanBucket}`; -} diff --git a/packages/apps/job-launcher/server/src/common/utils/storage.ts b/packages/apps/job-launcher/server/src/common/utils/storage.ts index ec7100b4ae..fbe86382ec 100644 --- a/packages/apps/job-launcher/server/src/common/utils/storage.ts +++ b/packages/apps/job-launcher/server/src/common/utils/storage.ts @@ -2,14 +2,14 @@ import { HttpStatus } from '@nestjs/common'; import axios, { AxiosError } from 'axios'; import { parseString } from 'xml2js'; import { StorageDataDto } from '../../modules/job/job.dto'; +import { + GCS_HTTP_REGEX_PATH_BASED, + GCS_HTTP_REGEX_SUBDOMAIN, +} from '../constants'; import { ErrorBucket } from '../constants/errors'; import { CvatJobType, JobRequestType } from '../enums/job'; import { AWSRegions, StorageProviders } from '../enums/storage'; import { ValidationError } from '../errors'; -import { - GCS_HTTP_REGEX_PATH_BASED, - GCS_HTTP_REGEX_SUBDOMAIN, -} from './gcstorage'; import { formatAxiosError } from './http'; function parseXml(xml: string): Promise { diff --git a/packages/apps/job-launcher/server/src/database/database.module.ts b/packages/apps/job-launcher/server/src/database/database.module.ts index 66d72e30d2..9e8d36a174 100644 --- a/packages/apps/job-launcher/server/src/database/database.module.ts +++ b/packages/apps/job-launcher/server/src/database/database.module.ts @@ -8,7 +8,6 @@ import { UserEntity } from '../modules/user/user.entity'; import { TypeOrmLoggerModule, TypeOrmLoggerService } from './typeorm'; import { JobEntity } from '../modules/job/job.entity'; -import { ContentModerationRequestEntity } from '../modules/content-moderation/content-moderation-request.entity'; import { PaymentEntity } from '../modules/payment/payment.entity'; import { DatabaseConfigService } from '../common/config/database-config.service'; import { ApiKeyEntity } from '../modules/auth/apikey.entity'; @@ -40,7 +39,6 @@ import { WhitelistEntity } from '../modules/whitelist/whitelist.entity'; ApiKeyEntity, UserEntity, JobEntity, - ContentModerationRequestEntity, PaymentEntity, WebhookEntity, CronJobEntity, diff --git a/packages/apps/job-launcher/server/src/database/migrations/1774453578372-removeContentModeration.ts b/packages/apps/job-launcher/server/src/database/migrations/1774453578372-removeContentModeration.ts new file mode 100644 index 0000000000..b9ec19d063 --- /dev/null +++ b/packages/apps/job-launcher/server/src/database/migrations/1774453578372-removeContentModeration.ts @@ -0,0 +1,151 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class RemoveContentModeration1774453578372 implements MigrationInterface { + name = 'RemoveContentModeration1774453578372'; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(` + UPDATE "hmt"."jobs" + SET "status" = 'paid' + WHERE "status" IN ('moderation_passed', 'under_moderation') + `); + await queryRunner.query(` + UPDATE "hmt"."jobs" + SET "status" = 'failed' + WHERE "status" = 'possible_abuse_in_review' + `); + await queryRunner.query(` + DELETE FROM "hmt"."cron-jobs" + WHERE "cron_job_type" = 'content-moderation' + `); + await queryRunner.query(` + ALTER TABLE "hmt"."content-moderation-requests" + DROP CONSTRAINT IF EXISTS "FK_d4f313caf54945a83b00abc02af" + `); + await queryRunner.query(` + DROP TABLE IF EXISTS "hmt"."content-moderation-requests" + `); + await queryRunner.query(` + DROP TYPE IF EXISTS "hmt"."content-moderation-requests_status_enum" + `); + await queryRunner.query(` + ALTER TYPE "hmt"."jobs_status_enum" + RENAME TO "jobs_status_enum_old" + `); + await queryRunner.query(` + CREATE TYPE "hmt"."jobs_status_enum" AS ENUM( + 'paid', + 'launched', + 'partial', + 'completed', + 'failed', + 'to_cancel', + 'canceling', + 'canceled' + ) + `); + await queryRunner.query(` + ALTER TABLE "hmt"."jobs" + ALTER COLUMN "status" TYPE "hmt"."jobs_status_enum" USING "status"::"text"::"hmt"."jobs_status_enum" + `); + await queryRunner.query(` + DROP TYPE "hmt"."jobs_status_enum_old" + `); + await queryRunner.query(` + ALTER TYPE "hmt"."cron-jobs_cron_job_type_enum" + RENAME TO "cron-jobs_cron_job_type_enum_old" + `); + await queryRunner.query(` + CREATE TYPE "hmt"."cron-jobs_cron_job_type_enum" AS ENUM( + 'create-escrow', + 'cancel-escrow', + 'process-pending-webhook', + 'sync-job-statuses', + 'abuse' + ) + `); + await queryRunner.query(` + ALTER TABLE "hmt"."cron-jobs" + ALTER COLUMN "cron_job_type" TYPE "hmt"."cron-jobs_cron_job_type_enum" USING "cron_job_type"::"text"::"hmt"."cron-jobs_cron_job_type_enum" + `); + await queryRunner.query(` + DROP TYPE "hmt"."cron-jobs_cron_job_type_enum_old" + `); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(` + CREATE TYPE "hmt"."content-moderation-requests_status_enum" AS ENUM( + 'pending', + 'processed', + 'positive_abuse', + 'passed', + 'failed' + ) + `); + await queryRunner.query(` + CREATE TABLE "hmt"."content-moderation-requests" ( + "id" SERIAL NOT NULL, + "created_at" TIMESTAMP WITH TIME ZONE NOT NULL, + "updated_at" TIMESTAMP WITH TIME ZONE NOT NULL, + "data_url" character varying NOT NULL, + "from" integer NOT NULL, + "to" integer NOT NULL, + "status" "hmt"."content-moderation-requests_status_enum" NOT NULL, + "job_id" integer NOT NULL, + CONSTRAINT "PK_e81154211cbfb9f8dcd56158313" PRIMARY KEY ("id") + ) + `); + await queryRunner.query(` + CREATE TYPE "hmt"."cron-jobs_cron_job_type_enum_old" AS ENUM( + 'abuse', + 'cancel-escrow', + 'content-moderation', + 'create-escrow', + 'process-pending-webhook', + 'sync-job-statuses' + ) + `); + await queryRunner.query(` + ALTER TABLE "hmt"."cron-jobs" + ALTER COLUMN "cron_job_type" TYPE "hmt"."cron-jobs_cron_job_type_enum_old" USING "cron_job_type"::"text"::"hmt"."cron-jobs_cron_job_type_enum_old" + `); + await queryRunner.query(` + DROP TYPE "hmt"."cron-jobs_cron_job_type_enum" + `); + await queryRunner.query(` + ALTER TYPE "hmt"."cron-jobs_cron_job_type_enum_old" + RENAME TO "cron-jobs_cron_job_type_enum" + `); + await queryRunner.query(` + CREATE TYPE "hmt"."jobs_status_enum_old" AS ENUM( + 'canceled', + 'canceling', + 'completed', + 'failed', + 'launched', + 'moderation_passed', + 'paid', + 'partial', + 'possible_abuse_in_review', + 'to_cancel', + 'under_moderation' + ) + `); + await queryRunner.query(` + ALTER TABLE "hmt"."jobs" + ALTER COLUMN "status" TYPE "hmt"."jobs_status_enum_old" USING "status"::"text"::"hmt"."jobs_status_enum_old" + `); + await queryRunner.query(` + DROP TYPE "hmt"."jobs_status_enum" + `); + await queryRunner.query(` + ALTER TYPE "hmt"."jobs_status_enum_old" + RENAME TO "jobs_status_enum" + `); + await queryRunner.query(` + ALTER TABLE "hmt"."content-moderation-requests" + ADD CONSTRAINT "FK_d4f313caf54945a83b00abc02af" FOREIGN KEY ("job_id") REFERENCES "hmt"."jobs"("id") ON DELETE NO ACTION ON UPDATE NO ACTION + `); + } +} diff --git a/packages/apps/job-launcher/server/src/modules/content-moderation/content-moderation-request.entity.ts b/packages/apps/job-launcher/server/src/modules/content-moderation/content-moderation-request.entity.ts deleted file mode 100644 index 70a268f4a7..0000000000 --- a/packages/apps/job-launcher/server/src/modules/content-moderation/content-moderation-request.entity.ts +++ /dev/null @@ -1,31 +0,0 @@ -import { Column, Entity, ManyToOne } from 'typeorm'; -import { NS } from '../../common/constants'; -import { BaseEntity } from '../../database/base.entity'; -import { ContentModerationRequestStatus } from '../../common/enums/content-moderation'; -import { JobEntity } from '../job/job.entity'; - -@Entity({ schema: NS, name: 'content-moderation-requests' }) -export class ContentModerationRequestEntity extends BaseEntity { - @Column({ type: 'varchar', nullable: false }) - public dataUrl: string; - - @Column({ type: 'int', nullable: false }) - public from: number; - - @Column({ type: 'int', nullable: false }) - public to: number; - - @Column({ - type: 'enum', - enum: ContentModerationRequestStatus, - }) - public status: ContentModerationRequestStatus; - - @ManyToOne(() => JobEntity, (job) => job.contentModerationRequests, { - eager: true, - }) - job: JobEntity; - - @Column({ type: 'int', nullable: false }) - public jobId: number; -} diff --git a/packages/apps/job-launcher/server/src/modules/content-moderation/content-moderation-request.repository.ts b/packages/apps/job-launcher/server/src/modules/content-moderation/content-moderation-request.repository.ts deleted file mode 100644 index 179b6c1092..0000000000 --- a/packages/apps/job-launcher/server/src/modules/content-moderation/content-moderation-request.repository.ts +++ /dev/null @@ -1,94 +0,0 @@ -import { Injectable } from '@nestjs/common'; -import { DataSource } from 'typeorm'; -import { ServerConfigService } from '../../common/config/server-config.service'; -import { SortDirection } from '../../common/enums/collection'; -import { ContentModerationRequestStatus } from '../../common/enums/content-moderation'; -import { BaseRepository } from '../../database/base.repository'; -import { ContentModerationRequestEntity } from './content-moderation-request.entity'; -import { QueryFailedError } from 'typeorm'; -import { handleQueryFailedError } from '../../common/errors'; - -@Injectable() -export class ContentModerationRequestRepository extends BaseRepository { - constructor( - private readonly dataSource: DataSource, - public readonly serverConfigService: ServerConfigService, - ) { - super(ContentModerationRequestEntity, dataSource); - } - - /** - * Finds all requests for a given job, ordered by createdAt desc. - */ - public async findByJobId( - jobId: number, - ): Promise { - try { - return this.find({ - where: { job: { id: jobId } }, - order: { createdAt: SortDirection.DESC }, - relations: ['job', 'job.contentModerationRequests'], - }); - } catch (error) { - if (error instanceof QueryFailedError) { - throw handleQueryFailedError(error); - } - throw error; - } - } - - /** - * Finds requests matching a jobId & status, in descending order by createdAt. - */ - public async findByJobIdAndStatus( - jobId: number, - status: ContentModerationRequestStatus, - ): Promise { - try { - return this.find({ - where: { job: { id: jobId }, status }, - order: { createdAt: SortDirection.DESC }, - relations: ['job', 'job.contentModerationRequests'], - }); - } catch (error) { - if (error instanceof QueryFailedError) { - throw handleQueryFailedError(error); - } - throw error; - } - } - - /** - * Creates multiple new requests in one call. - */ - public async createRequests( - requests: ContentModerationRequestEntity[], - ): Promise { - try { - return await this.save(requests); - } catch (error) { - if (error instanceof QueryFailedError) { - throw handleQueryFailedError(error); - } - throw error; - } - } - - /** - * Updates the status of a single request. - */ - public async updateStatus( - request: ContentModerationRequestEntity, - newStatus: ContentModerationRequestStatus, - ): Promise { - try { - request.status = newStatus; - await this.updateOne(request); - } catch (error) { - if (error instanceof QueryFailedError) { - throw handleQueryFailedError(error); - } - throw error; - } - } -} diff --git a/packages/apps/job-launcher/server/src/modules/content-moderation/content-moderation.dto.ts b/packages/apps/job-launcher/server/src/modules/content-moderation/content-moderation.dto.ts deleted file mode 100644 index eca7f1452a..0000000000 --- a/packages/apps/job-launcher/server/src/modules/content-moderation/content-moderation.dto.ts +++ /dev/null @@ -1,17 +0,0 @@ -export class ModerationResultDto { - adult: string; - violence: string; - racy: string; - spoof: string; - medical: string; -} - -export class ImageModerationResultDto { - imageUrl: string; - moderationResult: ModerationResultDto; -} - -export class DataModerationResultDto { - positiveAbuseResults: ImageModerationResultDto[]; - possibleAbuseResults: ImageModerationResultDto[]; -} diff --git a/packages/apps/job-launcher/server/src/modules/content-moderation/content-moderation.interface.ts b/packages/apps/job-launcher/server/src/modules/content-moderation/content-moderation.interface.ts deleted file mode 100644 index 7e4b518ba7..0000000000 --- a/packages/apps/job-launcher/server/src/modules/content-moderation/content-moderation.interface.ts +++ /dev/null @@ -1,5 +0,0 @@ -import { JobEntity } from '../job/job.entity'; - -export interface IContentModeratorService { - moderateJob(jobEntity: JobEntity): Promise; -} diff --git a/packages/apps/job-launcher/server/src/modules/content-moderation/content-moderation.module.ts b/packages/apps/job-launcher/server/src/modules/content-moderation/content-moderation.module.ts deleted file mode 100644 index b7ef50a7af..0000000000 --- a/packages/apps/job-launcher/server/src/modules/content-moderation/content-moderation.module.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { Global, Module } from '@nestjs/common'; -import { ConfigModule } from '@nestjs/config'; -import { TypeOrmModule } from '@nestjs/typeorm'; -import { JobModule } from '../job/job.module'; -import { ContentModerationRequestEntity } from './content-moderation-request.entity'; -import { ContentModerationRequestRepository } from './content-moderation-request.repository'; -import { GCVContentModerationService } from './gcv-content-moderation.service'; -import { JobEntity } from '../job/job.entity'; -import { JobRepository } from '../job/job.repository'; -import { ManifestModule } from '../manifest/manifest.module'; - -@Global() -@Module({ - imports: [ - TypeOrmModule.forFeature([ContentModerationRequestEntity, JobEntity]), - ConfigModule, - JobModule, - ManifestModule, - ], - providers: [ - ContentModerationRequestRepository, - JobRepository, - GCVContentModerationService, - ], - exports: [GCVContentModerationService], -}) -export class ContentModerationModule {} diff --git a/packages/apps/job-launcher/server/src/modules/content-moderation/gcv-content-moderation.service.spec.ts b/packages/apps/job-launcher/server/src/modules/content-moderation/gcv-content-moderation.service.spec.ts deleted file mode 100644 index 84bce6d2ef..0000000000 --- a/packages/apps/job-launcher/server/src/modules/content-moderation/gcv-content-moderation.service.spec.ts +++ /dev/null @@ -1,772 +0,0 @@ -jest.mock('@google-cloud/storage'); -jest.mock('@google-cloud/vision'); -jest.mock('../../common/utils/slack', () => ({ - sendSlackNotification: jest.fn(), -})); -jest.mock('../../common/utils/storage', () => ({ - ...jest.requireActual('../../common/utils/storage'), - listObjectsInBucket: jest.fn(), -})); - -import { faker } from '@faker-js/faker'; -import { Storage } from '@google-cloud/storage'; -import { ImageAnnotatorClient } from '@google-cloud/vision'; -import { Test, TestingModule } from '@nestjs/testing'; - -import { SlackConfigService } from '../../common/config/slack-config.service'; -import { VisionConfigService } from '../../common/config/vision-config.service'; -import { ErrorContentModeration } from '../../common/constants/errors'; -import { ContentModerationRequestStatus } from '../../common/enums/content-moderation'; -import { ContentModerationLevel } from '../../common/enums/gcv'; -import { JobStatus } from '../../common/enums/job'; -import { sendSlackNotification } from '../../common/utils/slack'; -import { listObjectsInBucket } from '../../common/utils/storage'; -import { JobEntity } from '../job/job.entity'; -import { JobRepository } from '../job/job.repository'; -import { ManifestService } from '../manifest/manifest.service'; -import { ContentModerationRequestEntity } from './content-moderation-request.entity'; -import { ContentModerationRequestRepository } from './content-moderation-request.repository'; -import { GCVContentModerationService } from './gcv-content-moderation.service'; - -describe('GCVContentModerationService', () => { - let service: GCVContentModerationService; - - let jobRepository: JobRepository; - let contentModerationRequestRepository: ContentModerationRequestRepository; - let slackConfigService: SlackConfigService; - let manifestService: ManifestService; - let jobEntity: JobEntity; - - const mockStorage = { - bucket: jest.fn().mockReturnValue({ - getFiles: jest.fn(), - file: jest.fn().mockReturnValue({ - createWriteStream: jest.fn(() => ({ end: jest.fn() })), - getSignedUrl: jest.fn(), - download: jest.fn(), - }), - }), - }; - const mockVisionClient = { - asyncBatchAnnotateImages: jest.fn(), - }; - - beforeAll(async () => { - (Storage as unknown as jest.Mock).mockImplementation(() => mockStorage); - (ImageAnnotatorClient as unknown as jest.Mock).mockImplementation( - () => mockVisionClient, - ); - - const module: TestingModule = await Test.createTestingModule({ - providers: [ - GCVContentModerationService, - { - provide: JobRepository, - useValue: { - updateOne: jest.fn(), - }, - }, - { - provide: ContentModerationRequestRepository, - useValue: { - findByJobId: jest.fn(), - findByJobIdAndStatus: jest.fn(), - updateOne: jest.fn(), - }, - }, - { - provide: VisionConfigService, - useValue: { - projectId: faker.string.uuid(), - privateKey: faker.string.alphanumeric(40), - clientEmail: faker.internet.email(), - moderationResultsBucket: faker.word.sample(), - moderationResultsFilesPath: faker.word.sample(), - }, - }, - { - provide: SlackConfigService, - useValue: { - abuseNotificationWebhookUrl: faker.internet.url(), - }, - }, - { - provide: ManifestService, - useValue: { - downloadManifest: jest.fn(), - }, - }, - ], - }).compile(); - service = module.get( - GCVContentModerationService, - ); - jobRepository = module.get(JobRepository); - contentModerationRequestRepository = - module.get( - ContentModerationRequestRepository, - ); - slackConfigService = module.get(SlackConfigService); - manifestService = module.get(ManifestService); - - jobEntity = { - id: faker.number.int(), - status: JobStatus.PAID, - manifestUrl: faker.internet.url(), - } as JobEntity; - }); - - afterEach(() => { - jest.clearAllMocks(); - }); - - describe('moderateJob (public)', () => { - it('should call createModerationRequests, processModerationRequests, parseModerationRequests, finalizeJob in order', async () => { - const createModerationRequestsSpy = jest - .spyOn(service, 'createModerationRequests') - .mockResolvedValueOnce(undefined); - const processModerationRequestsSpy = jest - .spyOn(service, 'processModerationRequests') - .mockResolvedValueOnce(undefined); - const parseModerationRequestsSpy = jest - .spyOn(service, 'parseModerationRequests') - .mockResolvedValueOnce(undefined); - const finalizeJobSpy = jest - .spyOn(service, 'finalizeJob') - .mockResolvedValueOnce(undefined); - - await service.moderateJob(jobEntity); - - expect(createModerationRequestsSpy).toHaveBeenCalledWith(jobEntity); - expect(processModerationRequestsSpy).toHaveBeenCalledWith(jobEntity); - expect(parseModerationRequestsSpy).toHaveBeenCalledWith(jobEntity); - expect(finalizeJobSpy).toHaveBeenCalledWith(jobEntity); - }); - - it('should propagate an error if createModerationRequests fails', async () => { - jest - .spyOn(service, 'createModerationRequests') - .mockRejectedValueOnce( - new Error('Simulated createModerationRequests error'), - ); - - await expect(service.moderateJob(jobEntity)).rejects.toThrow( - 'Simulated createModerationRequests error', - ); - }); - }); - - describe('createModerationRequests', () => { - it('should return if job status not PAID or UNDER_MODERATION', async () => { - jobEntity.status = JobStatus.CANCELED; - - await (service as any).createModerationRequests(jobEntity); - expect(jobRepository.updateOne).not.toHaveBeenCalled(); - }); - - it('should set job to MODERATION_PASSED if data_url is missing or invalid', async () => { - jobEntity.status = JobStatus.PAID; - (manifestService.downloadManifest as jest.Mock).mockResolvedValueOnce({ - data: { data_url: null }, - }); - - await (service as any).createModerationRequests(jobEntity); - expect(jobEntity.status).toBe(JobStatus.MODERATION_PASSED); - expect(jobRepository.updateOne).toHaveBeenCalledWith(jobEntity); - }); - - it('should do nothing if no valid files found in GCS', async () => { - jobEntity.status = JobStatus.PAID; - (manifestService.downloadManifest as jest.Mock).mockResolvedValueOnce({ - data: { - data_url: `gs://${faker.word.sample({ length: { min: 5, max: 10 } })}`, - }, - }); - - (listObjectsInBucket as jest.Mock).mockResolvedValueOnce([]); - await (service as any).createModerationRequests(jobEntity); - - expect(jobRepository.updateOne).not.toHaveBeenCalled(); - }); - - it('should create new requests in PENDING and set job to UNDER_MODERATION', async () => { - jobEntity.status = JobStatus.PAID; - (manifestService.downloadManifest as jest.Mock).mockResolvedValueOnce({ - data: { - data_url: `gs://${faker.word.sample({ length: { min: 5, max: 10 } })}`, - }, - }); - - (listObjectsInBucket as jest.Mock).mockResolvedValueOnce([ - `${faker.word.sample()}.jpg`, - `${faker.word.sample()}.jpg`, - `${faker.word.sample()}.jpg`, - ]); - ( - contentModerationRequestRepository.findByJobId as jest.Mock - ).mockResolvedValueOnce([]); - - await (service as any).createModerationRequests(jobEntity); - - expect(jobEntity.status).toBe(JobStatus.UNDER_MODERATION); - expect(jobRepository.updateOne).toHaveBeenCalledWith(jobEntity); - }); - - it('should throw if an error occurs in creation logic', async () => { - jobEntity.status = JobStatus.PAID; - (manifestService.downloadManifest as jest.Mock).mockResolvedValueOnce({ - data: { - data_url: `gs://${faker.word.sample({ length: { min: 5, max: 10 } })}`, - }, - }); - (listObjectsInBucket as jest.Mock).mockResolvedValueOnce([ - `${faker.word.sample()}.jpg`, - `${faker.word.sample()}.jpg`, - `${faker.word.sample()}.jpg`, - ]); - ( - contentModerationRequestRepository.findByJobId as jest.Mock - ).mockRejectedValueOnce(new Error('DB error')); - - await expect( - (service as any).createModerationRequests(jobEntity), - ).rejects.toThrow('DB error'); - }); - }); - - describe('processModerationRequests', () => { - it('should process all PENDING requests (success)', async () => { - const pendingRequest = { - id: faker.number.int(), - } as ContentModerationRequestEntity; - - ( - contentModerationRequestRepository.findByJobIdAndStatus as jest.Mock - ).mockResolvedValueOnce([pendingRequest]); - const processSingleRequestSpy = jest - .spyOn(service, 'processSingleRequest') - .mockResolvedValueOnce(undefined); - - await (service as any).processModerationRequests(jobEntity); - expect(processSingleRequestSpy).toHaveBeenCalledWith(pendingRequest); - }); - - it('should mark request as FAILED if processSingleRequest throws', async () => { - const pendingRequest = { - id: faker.number.int(), - } as ContentModerationRequestEntity; - - ( - contentModerationRequestRepository.findByJobIdAndStatus as jest.Mock - ).mockResolvedValueOnce([pendingRequest]); - jest - .spyOn(service, 'processSingleRequest') - .mockRejectedValueOnce(new Error('Processing error')); - - await (service as any).processModerationRequests(jobEntity); - - expect(contentModerationRequestRepository.updateOne).toHaveBeenCalledWith( - expect.objectContaining({ - id: pendingRequest.id, - status: ContentModerationRequestStatus.FAILED, - }), - ); - }); - - it('should throw if findByJobIdAndStatus fails', async () => { - ( - contentModerationRequestRepository.findByJobIdAndStatus as jest.Mock - ).mockRejectedValueOnce(new Error('getRequests error')); - - await expect( - (service as any).processModerationRequests(jobEntity), - ).rejects.toThrow('getRequests error'); - }); - }); - - describe('parseModerationRequests', () => { - it('should parse all PROCESSED requests (success)', async () => { - const processedRequest = { - id: faker.number.int(), - } as ContentModerationRequestEntity; - - ( - contentModerationRequestRepository.findByJobIdAndStatus as jest.Mock - ).mockResolvedValueOnce([processedRequest]); - const parseSingleRequestSpy = jest - .spyOn(service, 'parseSingleRequest') - .mockResolvedValueOnce(undefined); - - await (service as any).parseModerationRequests(jobEntity); - expect(parseSingleRequestSpy).toHaveBeenCalledWith(processedRequest); - }); - - it('should mark request as FAILED if parseSingleRequest throws', async () => { - const processedRequest = { - id: faker.number.int(), - } as ContentModerationRequestEntity; - - ( - contentModerationRequestRepository.findByJobIdAndStatus as jest.Mock - ).mockResolvedValueOnce([processedRequest]); - jest - .spyOn(service, 'parseSingleRequest') - .mockRejectedValueOnce(new Error('Parsing error')); - - await (service as any).parseModerationRequests(jobEntity); - expect(contentModerationRequestRepository.updateOne).toHaveBeenCalledWith( - expect.objectContaining({ - id: processedRequest.id, - status: ContentModerationRequestStatus.FAILED, - }), - ); - }); - - it('should throw if findByJobIdAndStatus fails', async () => { - ( - contentModerationRequestRepository.findByJobIdAndStatus as jest.Mock - ).mockRejectedValueOnce(new Error('getRequests error')); - - await expect( - (service as any).parseModerationRequests(jobEntity), - ).rejects.toThrow('getRequests error'); - }); - }); - - describe('finalizeJob', () => { - it('should do nothing if any requests are still PENDING or PROCESSED', async () => { - jobEntity.contentModerationRequests = []; - ( - contentModerationRequestRepository.findByJobId as jest.Mock - ).mockResolvedValueOnce([ - { status: ContentModerationRequestStatus.PROCESSED }, - ]); - - await (service as any).finalizeJob(jobEntity); - expect(jobRepository.updateOne).not.toHaveBeenCalled(); - }); - - it('should set job to MODERATION_PASSED if all requests passed', async () => { - jobEntity.contentModerationRequests = []; - ( - contentModerationRequestRepository.findByJobId as jest.Mock - ).mockResolvedValueOnce([ - { status: ContentModerationRequestStatus.PASSED }, - { status: ContentModerationRequestStatus.PASSED }, - ]); - - await (service as any).finalizeJob(jobEntity); - expect(jobEntity.status).toBe(JobStatus.MODERATION_PASSED); - expect(jobRepository.updateOne).toHaveBeenCalledWith(jobEntity); - }); - - it('should set job to POSSIBLE_ABUSE_IN_REVIEW if any request is flagged', async () => { - jobEntity.contentModerationRequests = []; - ( - contentModerationRequestRepository.findByJobId as jest.Mock - ).mockResolvedValueOnce([ - { status: ContentModerationRequestStatus.POSITIVE_ABUSE }, - ]); - - await (service as any).finalizeJob(jobEntity); - expect(jobEntity.status).toBe(JobStatus.POSSIBLE_ABUSE_IN_REVIEW); - expect(jobRepository.updateOne).toHaveBeenCalledWith(jobEntity); - }); - - it('should throw if DB call fails', async () => { - jobEntity.contentModerationRequests = []; - ( - contentModerationRequestRepository.findByJobId as jest.Mock - ).mockRejectedValueOnce(new Error('DB error')); - - await expect((service as any).finalizeJob(jobEntity)).rejects.toThrow( - 'DB error', - ); - }); - }); - - describe('processSingleRequest', () => { - it('should slice valid files, call asyncBatchAnnotateImages, set status PROCESSED', async () => { - const fakerBucket = faker.word.sample({ length: { min: 5, max: 10 } }); - const requestEntity: ContentModerationRequestEntity = { - id: faker.number.int(), - dataUrl: `https://${fakerBucket}.storage.googleapis.com`, - from: 1, - to: 2, - job: jobEntity, - } as any; - - const file1 = `${faker.word.sample()}.jpg`; - const file2 = `${faker.word.sample()}.jpg`; - const file3 = `${faker.word.sample()}.jpg`; - jest - .spyOn(service, 'getValidFiles') - .mockResolvedValueOnce([file1, file2, file3]); - const asyncBatchSpy = jest - .spyOn(service, 'asyncBatchAnnotateImages') - .mockResolvedValueOnce(undefined); - - await (service as any).processSingleRequest(requestEntity); - - expect(asyncBatchSpy).toHaveBeenCalledWith( - [`gs://${fakerBucket}/${file1}`, `gs://${fakerBucket}/${file2}`], - `moderation-results-${requestEntity.job.id}-${requestEntity.id}`, - ); - expect(contentModerationRequestRepository.updateOne).toHaveBeenCalledWith( - expect.objectContaining({ - id: requestEntity.id, - status: ContentModerationRequestStatus.PROCESSED, - }), - ); - }); - - it('should throw if asyncBatchAnnotateImages fails', async () => { - const requestEntity: ContentModerationRequestEntity = { - id: faker.number.int(), - dataUrl: `https://${faker.word.sample({ length: { min: 5, max: 10 } })}.storage.googleapis.com`, - from: 1, - to: 2, - job: jobEntity, - } as any; - - jest - .spyOn(service, 'getValidFiles') - .mockResolvedValueOnce([`${faker.word.sample()}.jpg`]); - jest - .spyOn(service, 'asyncBatchAnnotateImages') - .mockRejectedValueOnce(new Error('Vision error')); - - await expect( - (service as any).processSingleRequest(requestEntity), - ).rejects.toThrow('Vision error'); - }); - }); - - describe('asyncBatchAnnotateImages', () => { - it('should call visionClient.asyncBatchAnnotateImages successfully', async () => { - const mockOperation = { - promise: jest.fn().mockResolvedValueOnce([ - { - outputConfig: { gcsDestination: { uri: faker.internet.url() } }, - }, - ]), - }; - mockVisionClient.asyncBatchAnnotateImages.mockResolvedValueOnce([ - mockOperation, - ]); - - await (service as any).asyncBatchAnnotateImages( - ['img1', 'img2'], - 'my-file', - ); - expect(mockVisionClient.asyncBatchAnnotateImages).toHaveBeenCalledWith( - expect.objectContaining({ requests: expect.any(Array) }), - ); - }); - - it('should throw Error if vision call fails', async () => { - mockVisionClient.asyncBatchAnnotateImages.mockRejectedValueOnce( - new Error('Vision failure'), - ); - - await expect( - (service as any).asyncBatchAnnotateImages([], 'my-file'), - ).rejects.toThrow(Error); - }); - }); - - describe('parseSingleRequest', () => { - it('should set POSITIVE_ABUSE if positiveAbuseResults found', async () => { - const requestEntity: ContentModerationRequestEntity = { - id: faker.number.int(), - job: jobEntity, - } as any; - jest - .spyOn(service, 'collectModerationResults') - .mockResolvedValueOnce([ - { imageUrl: 'abuse.jpg', moderationResult: 'adult' }, - ]); - jest - .spyOn(service, 'handleAbuseLinks') - .mockResolvedValueOnce(undefined); - - await (service as any).parseSingleRequest(requestEntity); - expect(service['handleAbuseLinks']).toHaveBeenCalled(); - expect(requestEntity.status).toBe( - ContentModerationRequestStatus.POSITIVE_ABUSE, - ); - expect(contentModerationRequestRepository.updateOne).toHaveBeenCalledWith( - expect.objectContaining({ - status: ContentModerationRequestStatus.POSITIVE_ABUSE, - }), - ); - }); - - it('should set PASSED if no abuse found', async () => { - const requestEntity = { - id: faker.number.int(), - job: jobEntity, - } as ContentModerationRequestEntity; - jest - .spyOn(service, 'collectModerationResults') - .mockResolvedValueOnce({ - positiveAbuseResults: [], - possibleAbuseResults: [], - }); - - await (service as any).parseSingleRequest(requestEntity); - expect(requestEntity.status).toBe(ContentModerationRequestStatus.PASSED); - expect(contentModerationRequestRepository.updateOne).toHaveBeenCalledWith( - expect.objectContaining({ - status: ContentModerationRequestStatus.PASSED, - }), - ); - }); - - it('should set FAILED if collectModerationResults throws', async () => { - const requestEntity = { - id: faker.number.int(), - job: jobEntity, - } as ContentModerationRequestEntity; - jest - .spyOn(service, 'collectModerationResults') - .mockRejectedValueOnce(new Error('Collect error')); - - await expect( - (service as any).parseSingleRequest(requestEntity), - ).rejects.toThrow('Collect error'); - expect(requestEntity.status).toBe(ContentModerationRequestStatus.FAILED); - }); - }); - - describe('collectModerationResults', () => { - it('should throw ControlledError if no GCS files found', async () => { - (mockStorage.bucket as any).mockReturnValueOnce({ - getFiles: jest.fn().mockResolvedValueOnce([]), - }); - - await expect( - (service as any).collectModerationResults('some-file'), - ).rejects.toThrow(ErrorContentModeration.NoResultsFound); - }); - - it('should parse each file and accumulate responses, then categorize', async () => { - (mockStorage.bucket as any).mockReturnValueOnce({ - getFiles: jest.fn().mockResolvedValueOnce([ - [ - { - name: `${faker.word.sample()}.json`, - download: jest.fn().mockResolvedValueOnce([ - Buffer.from( - JSON.stringify({ - responses: [ - { - safeSearchAnnotation: { - adult: ContentModerationLevel.LIKELY, - }, - }, - ], - }), - ), - ]), - }, - { - name: `${faker.word.sample()}.json`, - download: jest.fn().mockResolvedValueOnce([ - Buffer.from( - JSON.stringify({ - responses: [ - { - safeSearchAnnotation: { - violence: ContentModerationLevel.POSSIBLE, - }, - }, - ], - }), - ), - ]), - }, - ], - ]), - }); - - jest - .spyOn(service, 'categorizeModerationResults') - .mockReturnValueOnce({ - positiveAbuseResults: [], - possibleAbuseResults: [], - }); - - const result = await (service as any).collectModerationResults( - faker.word.sample(), - ); - expect((service as any).categorizeModerationResults).toHaveBeenCalledWith( - expect.arrayContaining([ - { safeSearchAnnotation: { adult: ContentModerationLevel.LIKELY } }, - { - safeSearchAnnotation: { violence: ContentModerationLevel.POSSIBLE }, - }, - ]), - ); - expect(result).toHaveProperty('positiveAbuseResults'); - expect(result).toHaveProperty('possibleAbuseResults'); - }); - - it('should throw ControlledError if an error occurs', async () => { - (mockStorage.bucket as any).mockReturnValueOnce({ - getFiles: jest.fn().mockRejectedValueOnce(new Error('GCS error')), - }); - - await expect( - (service as any).collectModerationResults(faker.word.sample()), - ).rejects.toThrow(ErrorContentModeration.ResultsParsingFailed); - }); - }); - - describe('categorizeModerationResults', () => { - it('should split results into positiveAbuse and possibleAbuse', () => { - const responses = [ - { - safeSearchAnnotation: { adult: ContentModerationLevel.LIKELY }, - context: { - uri: `gs://${faker.word.sample({ length: { min: 5, max: 10 } })}/${faker.word.sample()}`, - }, - }, - { - safeSearchAnnotation: { violence: ContentModerationLevel.POSSIBLE }, - context: { - uri: `gs://${faker.word.sample({ length: { min: 5, max: 10 } })}/${faker.word.sample()}`, - }, - }, - ]; - const results = (service as any).categorizeModerationResults(responses); - expect(results).toHaveLength(2); - expect(results[0]).toHaveProperty('imageUrl'); - expect(results[0]).toHaveProperty('moderationResult'); - expect(results[1]).toHaveProperty('imageUrl'); - expect(results[1]).toHaveProperty('moderationResult'); - expect(results[0].moderationResult).toBe('adult'); - expect(results[1].moderationResult).toBe('violence'); - }); - - it('should ignore entries with no safeSearchAnnotation', () => { - const responses = [ - { - safeSearchAnnotation: null, - context: { - uri: `gs://${faker.word.sample({ length: { min: 5, max: 10 } })}/${faker.word.sample()}`, - }, - }, - ]; - const results = (service as any).categorizeModerationResults(responses); - expect(results).toHaveLength(0); - }); - }); - - describe('handleAbuseLinks', () => { - it('should upload text file and send Slack message for confirmed abuse', async () => { - const mockSignedUrl = faker.internet.url(); - (mockStorage.bucket as any).mockReturnValueOnce({ - file: jest.fn().mockReturnValueOnce({ - createWriteStream: jest.fn(() => ({ end: jest.fn() })), - getSignedUrl: jest.fn().mockResolvedValueOnce([mockSignedUrl]), - }), - }); - - await (service as any).handleAbuseLinks( - [faker.internet.url()], - faker.word.sample(), - faker.number.int(), - faker.number.int(), - true, - ); - expect(sendSlackNotification).toHaveBeenCalledWith( - slackConfigService.abuseNotificationWebhookUrl, - expect.stringContaining(mockSignedUrl), - ); - }); - - it('should handle possible abuse similarly', async () => { - const mockSignedUrl = faker.internet.url(); - (mockStorage.bucket as any).mockReturnValueOnce({ - file: jest.fn().mockReturnValueOnce({ - createWriteStream: jest.fn(() => ({ end: jest.fn() })), - getSignedUrl: jest.fn().mockResolvedValueOnce([mockSignedUrl]), - }), - }); - - await (service as any).handleAbuseLinks( - [faker.internet.url()], - faker.word.sample(), - faker.number.int(), - faker.number.int(), - false, - ); - expect(sendSlackNotification).toHaveBeenCalledWith( - slackConfigService.abuseNotificationWebhookUrl, - expect.stringContaining(mockSignedUrl), - ); - }); - - it('should throw if getSignedUrl fails', async () => { - (mockStorage.bucket as any).mockReturnValueOnce({ - file: jest.fn().mockReturnValueOnce({ - createWriteStream: jest.fn(() => ({ end: jest.fn() })), - getSignedUrl: jest - .fn() - .mockRejectedValueOnce(new Error('Signed URL error')), - }), - }); - - await expect( - (service as any).handleAbuseLinks( - [], - faker.word.sample(), - faker.number.int(), - faker.number.int(), - true, - ), - ).rejects.toThrow('Signed URL error'); - }); - }); - - describe('getValidFiles', () => { - it('should return cached files if present', async () => { - const dataUrl = `gs://${faker.word.sample({ length: { min: 5, max: 10 } })}/data`; - const file1 = `${faker.word.sample()}.jpg`; - const file2 = `${faker.word.sample()}.png`; - (service as any).bucketListCache.set(dataUrl, [file1, file2]); - - const result = await (service as any).getValidFiles(dataUrl); - expect(result).toEqual([file1, file2]); - expect(listObjectsInBucket).not.toHaveBeenCalled(); - }); - - it('should fetch from GCS if not cached, filter out directories, and cache', async () => { - const dataUrl = `gs://${faker.word.sample({ length: { min: 5, max: 10 } })}/data`; - const file1 = `${faker.word.sample()}.jpg`; - const file2 = `${faker.word.sample()}.png`; - (listObjectsInBucket as jest.Mock).mockResolvedValueOnce([ - file1, - 'subdir/', - file2, - ]); - - const result = await (service as any).getValidFiles(dataUrl); - expect(result).toEqual([file1, file2]); - - expect((service as any).bucketListCache.get(dataUrl)).toEqual(result); - }); - - it('should throw if listObjectsInBucket fails', async () => { - const dataUrl = `gs://${faker.word.sample({ length: { min: 5, max: 10 } })}/fail`; - (listObjectsInBucket as jest.Mock).mockRejectedValueOnce( - new Error('List objects error'), - ); - - await expect((service as any).getValidFiles(dataUrl)).rejects.toThrow( - 'List objects error', - ); - }); - }); -}); diff --git a/packages/apps/job-launcher/server/src/modules/content-moderation/gcv-content-moderation.service.ts b/packages/apps/job-launcher/server/src/modules/content-moderation/gcv-content-moderation.service.ts deleted file mode 100644 index f1a422c51d..0000000000 --- a/packages/apps/job-launcher/server/src/modules/content-moderation/gcv-content-moderation.service.ts +++ /dev/null @@ -1,507 +0,0 @@ -import { Storage } from '@google-cloud/storage'; -import { ImageAnnotatorClient, protos } from '@google-cloud/vision'; -import { Injectable } from '@nestjs/common'; -import NodeCache from 'node-cache'; -import { SlackConfigService } from '../../common/config/slack-config.service'; -import { VisionConfigService } from '../../common/config/vision-config.service'; -import { - GCV_CONTENT_MODERATION_ASYNC_BATCH_SIZE, - GCV_CONTENT_MODERATION_BATCH_SIZE_PER_TASK, -} from '../../common/constants'; -import { ErrorContentModeration } from '../../common/constants/errors'; -import { ContentModerationRequestStatus } from '../../common/enums/content-moderation'; -import { - ContentModerationFeature, - ContentModerationLevel, -} from '../../common/enums/gcv'; -import { JobStatus } from '../../common/enums/job'; -import { - constructGcsPath, - convertToGCSPath, - convertToHttpUrl, - isGCSBucketUrl, -} from '../../common/utils/gcstorage'; -import { sendSlackNotification } from '../../common/utils/slack'; -import { listObjectsInBucket } from '../../common/utils/storage'; -import { JobEntity } from '../job/job.entity'; -import { JobRepository } from '../job/job.repository'; -import { CvatManifestDto } from '../manifest/manifest.dto'; -import { ManifestService } from '../manifest/manifest.service'; -import { ContentModerationRequestEntity } from './content-moderation-request.entity'; -import { ContentModerationRequestRepository } from './content-moderation-request.repository'; -import { ModerationResultDto } from './content-moderation.dto'; -import { IContentModeratorService } from './content-moderation.interface'; -import logger from '../../logger'; - -@Injectable() -export class GCVContentModerationService implements IContentModeratorService { - private readonly logger = logger.child({ - context: GCVContentModerationService.name, - }); - - private visionClient: ImageAnnotatorClient; - private storage: Storage; - - /** - * Cache of GCS object listings by dataUrl - * Key: dataUrl string, Value: array of valid file names - */ - private bucketListCache: NodeCache; - - constructor( - private readonly jobRepository: JobRepository, - private readonly contentModerationRequestRepository: ContentModerationRequestRepository, - private readonly visionConfigService: VisionConfigService, - private readonly slackConfigService: SlackConfigService, - private readonly manifestService: ManifestService, - ) { - this.visionClient = new ImageAnnotatorClient({ - projectId: this.visionConfigService.projectId, - credentials: { - private_key: this.visionConfigService.privateKey, - client_email: this.visionConfigService.clientEmail, - }, - }); - - this.storage = new Storage({ - projectId: this.visionConfigService.projectId, - credentials: { - private_key: this.visionConfigService.privateKey, - client_email: this.visionConfigService.clientEmail, - }, - }); - - // Initialize cache with expiration time of 60 minutes and check period of 15 minutes - this.bucketListCache = new NodeCache({ - stdTTL: 30 * 60, - checkperiod: 15 * 60, - }); - } - - /** - * Single public method orchestrating all steps in order - */ - public async moderateJob(jobEntity: JobEntity): Promise { - await this.createModerationRequests(jobEntity); - await this.processModerationRequests(jobEntity); - await this.parseModerationRequests(jobEntity); - await this.finalizeJob(jobEntity); - } - - /** - * 1) If no requests exist for this job, create them in PENDING. - */ - private async createModerationRequests(jobEntity: JobEntity): Promise { - if ( - jobEntity.status !== JobStatus.PAID && - jobEntity.status !== JobStatus.UNDER_MODERATION - ) { - return; - } - - try { - const manifest = (await this.manifestService.downloadManifest( - jobEntity.manifestUrl, - jobEntity.requestType, - )) as CvatManifestDto; - const dataUrl = manifest?.data?.data_url; - - if (!dataUrl || !isGCSBucketUrl(dataUrl)) { - jobEntity.status = JobStatus.MODERATION_PASSED; - await this.jobRepository.updateOne(jobEntity); - return; - } - - const validFiles = await this.getValidFiles(dataUrl); - if (validFiles.length === 0) return; - - const existingRequests = - await this.contentModerationRequestRepository.findByJobId(jobEntity.id); - - const newRequests: ContentModerationRequestEntity[] = []; - - for ( - let i = 0; - i < validFiles.length; - i += GCV_CONTENT_MODERATION_BATCH_SIZE_PER_TASK - ) { - const from = i + 1; - const to = Math.min( - i + GCV_CONTENT_MODERATION_BATCH_SIZE_PER_TASK, - validFiles.length, - ); - - const request = existingRequests.some( - (req) => req.from === from && req.to === to, - ); - - if (!request) { - newRequests.push( - Object.assign(new ContentModerationRequestEntity(), { - dataUrl, - from, - to, - status: ContentModerationRequestStatus.PENDING, - job: jobEntity, - }), - ); - } - } - - if (newRequests.length > 0) { - jobEntity.contentModerationRequests = [ - ...(jobEntity.contentModerationRequests || []), - ...newRequests, - ]; - jobEntity.status = JobStatus.UNDER_MODERATION; - await this.jobRepository.updateOne(jobEntity); - } - } catch (error) { - this.logger.error('Error creating requests for job', { - error, - jobId: jobEntity.id, - }); - throw error; - } - } - - /** - * 2) Process all PENDING requests -> call GCV. Mark them PROCESSED if success. - * Parallelized with Promise.all for performance. - */ - private async processModerationRequests(jobEntity: JobEntity): Promise { - try { - const requests = - await this.contentModerationRequestRepository.findByJobIdAndStatus( - jobEntity.id, - ContentModerationRequestStatus.PENDING, - ); - await Promise.all( - requests.map(async (requestEntity) => { - try { - await this.processSingleRequest(requestEntity); - } catch (error) { - this.logger.error('Error processing moderation request', { - moderationRequestId: requestEntity.id, - jobId: jobEntity.id, - error, - }); - - requestEntity.status = ContentModerationRequestStatus.FAILED; - await this.contentModerationRequestRepository.updateOne( - requestEntity, - ); - } - }), - ); - } catch (error) { - this.logger.error('Error processing moderation requests', { - error, - jobId: jobEntity.id, - }); - - throw error; - } - } - - /** - * 3) Parse results for requests in PROCESSED -> set to PASSED, POSSIBLE_ABUSE, or POSITIVE_ABUSE - * Also parallelized with Promise.all. - */ - private async parseModerationRequests(jobEntity: JobEntity): Promise { - try { - const requests = - await this.contentModerationRequestRepository.findByJobIdAndStatus( - jobEntity.id, - ContentModerationRequestStatus.PROCESSED, - ); - - await Promise.all( - requests.map(async (requestEntity) => { - try { - await this.parseSingleRequest(requestEntity); - } catch (error) { - this.logger.error('Error parsing moderation request', { - moderationRequestId: requestEntity.id, - jobId: jobEntity.id, - error, - }); - - requestEntity.status = ContentModerationRequestStatus.FAILED; - await this.contentModerationRequestRepository.updateOne( - requestEntity, - ); - } - }), - ); - } catch (error) { - this.logger.error('Error parsing moderation results', { - jobId: jobEntity.id, - error, - }); - throw error; - } - } - - /** - * 4) If all requests are done, set job to MODERATION_PASSED or POSSIBLE_ABUSE_IN_REVIEW - */ - private async finalizeJob(jobEntity: JobEntity): Promise { - try { - // We'll try to use the jobEntity if it has requests loaded. Otherwise, fallback to DB. - const allRequests = jobEntity.contentModerationRequests?.length - ? jobEntity.contentModerationRequests - : await this.contentModerationRequestRepository.findByJobId( - jobEntity.id, - ); - - const incomplete = allRequests.some( - (r) => - r.status === ContentModerationRequestStatus.PENDING || - r.status === ContentModerationRequestStatus.PROCESSED, - ); - if (incomplete) return; - - let allPassed = true; - for (const req of allRequests) { - if ( - req.status === ContentModerationRequestStatus.FAILED || - req.status === ContentModerationRequestStatus.POSITIVE_ABUSE - ) { - allPassed = false; - } - } - - if (allPassed) { - jobEntity.status = JobStatus.MODERATION_PASSED; - await this.jobRepository.updateOne(jobEntity); - } else { - jobEntity.status = JobStatus.POSSIBLE_ABUSE_IN_REVIEW; - await this.jobRepository.updateOne(jobEntity); - } - } catch (error) { - this.logger.error('Error finalizing moderation job', { - jobId: jobEntity.id, - error, - }); - throw error; - } - } - - /** - * Actually calls GCV. Mark requestEntity => PROCESSED on success. - */ - private async processSingleRequest( - requestEntity: ContentModerationRequestEntity, - ): Promise { - const validFiles = await this.getValidFiles(requestEntity.dataUrl); - const filesToProcess = validFiles.slice( - requestEntity.from - 1, - requestEntity.to, - ); - const gcDataUrl = convertToGCSPath(requestEntity.dataUrl); - const imageUrls = filesToProcess.map( - (fileName) => `${gcDataUrl}/${fileName.split('/').pop()}`, - ); - - const fileName = `moderation-results-${requestEntity.job.id}-${requestEntity.id}`; - - await this.asyncBatchAnnotateImages(imageUrls, fileName); - - requestEntity.status = ContentModerationRequestStatus.PROCESSED; - await this.contentModerationRequestRepository.updateOne(requestEntity); - } - - /** - * Calls GCV's asyncBatchAnnotateImages with SAFE_SEARCH_DETECTION - */ - private async asyncBatchAnnotateImages( - imageUrls: string[], - fileName: string, - ): Promise { - const request = imageUrls.map((url) => ({ - image: { source: { imageUri: url } }, - features: [{ type: ContentModerationFeature.SAFE_SEARCH_DETECTION }], - })); - - const outputUri = constructGcsPath( - this.visionConfigService.moderationResultsBucket, - this.visionConfigService.moderationResultsFilesPath, - fileName + '-', - ); - - const requestPayload: protos.google.cloud.vision.v1.IAsyncBatchAnnotateImagesRequest = - { - requests: request, - outputConfig: { - gcsDestination: { uri: outputUri }, - batchSize: GCV_CONTENT_MODERATION_ASYNC_BATCH_SIZE, - }, - }; - - try { - const [operation] = - await this.visionClient.asyncBatchAnnotateImages(requestPayload); - const [filesResponse] = await operation.promise(); - this.logger.debug('Output written to GCS', { - url: filesResponse?.outputConfig?.gcsDestination?.uri, - }); - } catch (error) { - this.logger.error('Error analyzing images', error); - throw new Error(ErrorContentModeration.ContentModerationFailed); - } - } - - /** - * Parse a single PROCESSED request => sets it to PASSED or POSITIVE_ABUSE - */ - private async parseSingleRequest( - requestEntity: ContentModerationRequestEntity, - ): Promise { - try { - const fileName = `moderation-results-${requestEntity.job.id}-${requestEntity.id}`; - const moderationResults = await this.collectModerationResults(fileName); - - if (moderationResults.length > 0) { - await this.handleAbuseLinks( - moderationResults, - fileName, - requestEntity.id, - requestEntity.job.id, - ); - requestEntity.status = ContentModerationRequestStatus.POSITIVE_ABUSE; - } else { - requestEntity.status = ContentModerationRequestStatus.PASSED; - } - } catch (err) { - requestEntity.status = ContentModerationRequestStatus.FAILED; - throw err; - } - await this.contentModerationRequestRepository.updateOne(requestEntity); - } - - /** - * Downloads GCS results, categorizes them into positiveAbuse / possibleAbuse - */ - private async collectModerationResults(fileName: string) { - try { - const bucketPrefix = `${this.visionConfigService.moderationResultsFilesPath}/${fileName}`; - const bucketName = this.visionConfigService.moderationResultsBucket; - const bucket = this.storage.bucket(bucketName); - - const [files] = await bucket.getFiles({ prefix: bucketPrefix }); - if (!files || files.length === 0) { - throw new Error(ErrorContentModeration.NoResultsFound); - } - - const allResponses = []; - for (const file of files) { - const [content] = await file.download(); - const jsonString = content.toString('utf-8'); - const parsed = JSON.parse(jsonString); - - if (Array.isArray(parsed.responses)) { - allResponses.push(...parsed.responses); - } - } - return this.categorizeModerationResults(allResponses); - } catch (error) { - if (error.message === ErrorContentModeration.NoResultsFound) { - throw error; - } - this.logger.error('Error collecting moderation results', error); - throw new Error(ErrorContentModeration.ResultsParsingFailed); - } - } - - /** - * Processes the results from the Google Cloud Vision API and categorizes them based on moderation levels - */ - private categorizeModerationResults( - results: protos.google.cloud.vision.v1.IAnnotateImageResponse[], - ) { - const relevantLevels = [ - ContentModerationLevel.VERY_LIKELY, - ContentModerationLevel.LIKELY, - ContentModerationLevel.POSSIBLE, - ]; - - return results - .map((response) => { - const safeSearch = response.safeSearchAnnotation as ModerationResultDto; - if (!safeSearch) return null; - - const imageUrl = convertToHttpUrl(response.context?.uri ?? ''); - - const flaggedCategory = Object.keys(new ModerationResultDto()).find( - (field) => - relevantLevels.includes( - safeSearch[ - field as keyof ModerationResultDto - ] as ContentModerationLevel, - ), - ); - - if (!flaggedCategory) { - return null; - } - - return { - imageUrl, - moderationResult: flaggedCategory, - }; - }) - - .filter( - (item): item is { imageUrl: string; moderationResult: string } => - !!item, - ); - } - - /** - * Uploads a small text file listing the abuse-related images, then sends Slack notification - */ - private async handleAbuseLinks( - images: { - imageUrl: string; - moderationResult: string; - }[], - fileName: string, - requestId: number, - jobId: number, - ): Promise { - const bucketName = this.visionConfigService.moderationResultsBucket; - const resultsFileName = `${fileName}.txt`; - const file = this.storage.bucket(bucketName).file(resultsFileName); - const stream = file.createWriteStream({ resumable: false }); - stream.end(JSON.stringify(images)); - - const [signedUrl] = await file.getSignedUrl({ - action: 'read', - expires: Date.now() + 60 * 60 * 24 * 1000, - }); - const consoleUrl = `https://console.cloud.google.com/storage/browser/${bucketName}?prefix=${resultsFileName}`; - const message = `Images may contain abusive content. Request ${requestId}, job ${jobId}.\n\n**Results File:** <${signedUrl}|Download Here>\n**Google Cloud Console:** <${consoleUrl}|View in Console>\n\nEnsure you download the file before the link expires, or access it directly via GCS.`; - - await sendSlackNotification( - this.slackConfigService.abuseNotificationWebhookUrl, - message, - ); - } - - /** - * Caches GCS object listings so we don't repeatedly call listObjectsInBucket for the same dataUrl - */ - private async getValidFiles(dataUrl: string): Promise { - const cacheEntry = this.bucketListCache.get(dataUrl); - if (cacheEntry) { - return cacheEntry; - } - - const allFiles = await listObjectsInBucket(new URL(dataUrl)); - const validFiles = allFiles.filter((f) => f && !f.endsWith('/')); - this.bucketListCache.set(dataUrl, validFiles); - - return validFiles; - } -} diff --git a/packages/apps/job-launcher/server/src/modules/cron-job/cron-job.module.ts b/packages/apps/job-launcher/server/src/modules/cron-job/cron-job.module.ts index 97c54ae941..f51bd37967 100644 --- a/packages/apps/job-launcher/server/src/modules/cron-job/cron-job.module.ts +++ b/packages/apps/job-launcher/server/src/modules/cron-job/cron-job.module.ts @@ -12,14 +12,12 @@ import { WebhookRepository } from '../webhook/webhook.repository'; import { JobEntity } from '../job/job.entity'; import { JobRepository } from '../job/job.repository'; import { ConfigModule } from '@nestjs/config'; -import { ContentModerationModule } from '../content-moderation/content-moderation.module'; @Global() @Module({ imports: [ TypeOrmModule.forFeature([CronJobEntity, JobEntity]), ConfigModule, - ContentModerationModule, JobModule, PaymentModule, Web3Module, diff --git a/packages/apps/job-launcher/server/src/modules/cron-job/cron-job.service.spec.ts b/packages/apps/job-launcher/server/src/modules/cron-job/cron-job.service.spec.ts index 660fab11e1..74d298b02a 100644 --- a/packages/apps/job-launcher/server/src/modules/cron-job/cron-job.service.spec.ts +++ b/packages/apps/job-launcher/server/src/modules/cron-job/cron-job.service.spec.ts @@ -27,13 +27,8 @@ import { } from '../../../test/constants'; import { NetworkConfigService } from '../../common/config/network-config.service'; import { ServerConfigService } from '../../common/config/server-config.service'; -import { SlackConfigService } from '../../common/config/slack-config.service'; -import { VisionConfigService } from '../../common/config/vision-config.service'; import { Web3ConfigService } from '../../common/config/web3-config.service'; -import { - ErrorContentModeration, - ErrorCronJob, -} from '../../common/constants/errors'; +import { ErrorCronJob } from '../../common/constants/errors'; import { CronJobType } from '../../common/enums/cron-job'; import { CvatJobType, @@ -44,8 +39,6 @@ import { import { WebhookStatus } from '../../common/enums/webhook'; import { ConflictError } from '../../common/errors'; import logger from '../../logger'; -import { ContentModerationRequestRepository } from '../content-moderation/content-moderation-request.repository'; -import { GCVContentModerationService } from '../content-moderation/gcv-content-moderation.service'; import { JobEntity } from '../job/job.entity'; import { JobRepository } from '../job/job.repository'; import { JobService } from '../job/job.service'; @@ -54,7 +47,6 @@ import { PaymentRepository } from '../payment/payment.repository'; import { PaymentService } from '../payment/payment.service'; import { QualificationService } from '../qualification/qualification.service'; import { RateService } from '../rate/rate.service'; -import { RoutingProtocolService } from '../routing-protocol/routing-protocol.service'; import { StorageService } from '../storage/storage.service'; import { Web3Service } from '../web3/web3.service'; import { WebhookEntity } from '../webhook/webhook.entity'; @@ -77,7 +69,6 @@ describe('CronJobService', () => { storageService: StorageService, jobService: JobService, paymentService: PaymentService, - contentModerationService: GCVContentModerationService, jobRepository: JobRepository; const signerMock = { @@ -111,23 +102,11 @@ describe('CronJobService', () => { }, }, JobService, - GCVContentModerationService, WebhookService, Encryption, ServerConfigService, Web3ConfigService, NetworkConfigService, - { - provide: VisionConfigService, - useValue: { - projectId: 'test-project-id', - privateKey: 'test-private-key', - clientEmail: 'test-client-email', - tempAsyncResultsBucket: 'test-temp-bucket', - moderationResultsBucket: 'test-moderation-results-bucket', - }, - }, - SlackConfigService, QualificationService, { provide: NetworkConfigService, @@ -136,10 +115,6 @@ describe('CronJobService', () => { }, }, { provide: JobRepository, useValue: createMock() }, - { - provide: ContentModerationRequestRepository, - useValue: createMock(), - }, { provide: PaymentRepository, useValue: createMock(), @@ -148,10 +123,6 @@ describe('CronJobService', () => { { provide: PaymentService, useValue: createMock() }, { provide: WhitelistService, useValue: createMock() }, { provide: ConfigService, useValue: mockConfigService }, - { - provide: RoutingProtocolService, - useValue: createMock(), - }, { provide: WebhookRepository, useValue: createMock(), @@ -170,9 +141,6 @@ describe('CronJobService', () => { service = module.get(CronJobService); // paymentService = module.get(PaymentService); - contentModerationService = module.get( - GCVContentModerationService, - ); jobService = module.get(JobService); jobRepository = module.get(JobRepository); paymentService = module.get(PaymentService); @@ -758,112 +726,6 @@ describe('CronJobService', () => { }); }); - describe('moderateContentCronJob', () => { - let contentModerationMock: any; - let cronJobEntityMock: Partial; - let jobEntity1: Partial, jobEntity2: Partial; - - beforeEach(() => { - cronJobEntityMock = { - cronJobType: CronJobType.ContentModeration, - startedAt: new Date(), - }; - - jobEntity1 = { - id: 1, - status: JobStatus.PAID, - }; - - jobEntity2 = { - id: 2, - status: JobStatus.PAID, - }; - - jest - .spyOn(jobRepository, 'findByStatus') - .mockResolvedValue([jobEntity1 as any, jobEntity2 as any]); - - contentModerationMock = jest.spyOn( - contentModerationService, - 'moderateJob', - ); - contentModerationMock.mockResolvedValue(true); - - jest.spyOn(service, 'isCronJobRunning').mockResolvedValue(false); - - jest.spyOn(repository, 'findOneByType').mockResolvedValue(null); - jest - .spyOn(repository, 'createUnique') - .mockResolvedValue(cronJobEntityMock as any); - }); - - afterEach(() => { - jest.restoreAllMocks(); - }); - - it('should not run if cron job is already running', async () => { - jest.spyOn(service, 'isCronJobRunning').mockResolvedValueOnce(true); - - const startCronJobMock = jest.spyOn(service, 'startCronJob'); - - await service.moderateContentCronJob(); - - expect(startCronJobMock).not.toHaveBeenCalled(); - }); - - it('should create a cron job entity to lock the process', async () => { - jest - .spyOn(service, 'startCronJob') - .mockResolvedValueOnce(cronJobEntityMock as any); - - await service.moderateContentCronJob(); - - expect(service.startCronJob).toHaveBeenCalledWith( - CronJobType.ContentModeration, - ); - }); - - it('should process all jobs with status PAID', async () => { - await service.moderateContentCronJob(); - - expect(contentModerationMock).toHaveBeenCalledTimes(2); - expect(contentModerationMock).toHaveBeenCalledWith(jobEntity1); - expect(contentModerationMock).toHaveBeenCalledWith(jobEntity2); - }); - - it('should handle failed moderation attempts', async () => { - const error = new Error('Moderation failed'); - contentModerationMock.mockRejectedValueOnce(error); - - const handleFailureMock = jest.spyOn( - jobService, - 'handleProcessJobFailure', - ); - - await service.moderateContentCronJob(); - - expect(handleFailureMock).toHaveBeenCalledTimes(1); - expect(handleFailureMock).toHaveBeenCalledWith( - jobEntity1, - expect.stringContaining(ErrorContentModeration.ResultsParsingFailed), - ); - expect(handleFailureMock).not.toHaveBeenCalledWith( - jobEntity2, - expect.anything(), - ); - }); - - it('should complete the cron job entity to unlock', async () => { - jest - .spyOn(service, 'completeCronJob') - .mockResolvedValueOnce(cronJobEntityMock as any); - - await service.moderateContentCronJob(); - - expect(service.completeCronJob).toHaveBeenCalledWith(cronJobEntityMock); - }); - }); - describe('syncJobStatuses Cron Job', () => { let cronJobEntityMock: Partial; let jobEntityMock: Partial; diff --git a/packages/apps/job-launcher/server/src/modules/cron-job/cron-job.service.ts b/packages/apps/job-launcher/server/src/modules/cron-job/cron-job.service.ts index f41f759733..8034f103b1 100644 --- a/packages/apps/job-launcher/server/src/modules/cron-job/cron-job.service.ts +++ b/packages/apps/job-launcher/server/src/modules/cron-job/cron-job.service.ts @@ -1,6 +1,5 @@ import { Injectable } from '@nestjs/common'; import { - ErrorContentModeration, ErrorCronJob, ErrorEscrow, ErrorJob, @@ -19,7 +18,6 @@ import { } from '../../common/enums/webhook'; import { ConflictError, NotFoundError } from '../../common/errors'; import logger from '../../logger'; -import { GCVContentModerationService } from '../content-moderation/gcv-content-moderation.service'; import { JobEntity } from '../job/job.entity'; import { JobRepository } from '../job/job.repository'; import { JobService } from '../job/job.service'; @@ -39,7 +37,6 @@ export class CronJobService { private readonly cronJobRepository: CronJobRepository, private readonly jobService: JobService, private readonly jobRepository: JobRepository, - private readonly contentModerationService: GCVContentModerationService, private readonly webhookService: WebhookService, private readonly web3Service: Web3Service, private readonly paymentService: PaymentService, @@ -82,45 +79,6 @@ export class CronJobService { return this.cronJobRepository.updateOne(cronJobEntity); } - @Cron('*/2 * * * *') - public async moderateContentCronJob() { - if (await this.isCronJobRunning(CronJobType.ContentModeration)) { - return; - } - - const cronJobEntity = await this.startCronJob( - CronJobType.ContentModeration, - ); - - try { - const jobs = await this.jobRepository.findByStatus([ - JobStatus.PAID, - JobStatus.UNDER_MODERATION, - ]); - - await Promise.all( - jobs.map(async (jobEntity) => { - try { - await this.contentModerationService.moderateJob(jobEntity); - } catch (error) { - this.logger.error('Error parse job moderation results job', { - jobId: jobEntity.id, - error, - }); - await this.jobService.handleProcessJobFailure( - jobEntity, - ErrorContentModeration.ResultsParsingFailed, - ); - } - }), - ); - } catch (error) { - this.logger.error('Error in moderateContentCronJob', error); - } - - await this.completeCronJob(cronJobEntity); - } - @Cron('*/2 * * * *') public async createEscrowCronJob() { const isCronJobRunning = await this.isCronJobRunning( @@ -135,9 +93,7 @@ export class CronJobService { const cronJob = await this.startCronJob(CronJobType.CreateEscrow); try { - const jobEntities = await this.jobRepository.findByStatus( - JobStatus.MODERATION_PASSED, - ); + const jobEntities = await this.jobRepository.findByStatus(JobStatus.PAID); for (const jobEntity of jobEntities) { try { await this.jobService.createEscrow(jobEntity); diff --git a/packages/apps/job-launcher/server/src/modules/job/fixtures.ts b/packages/apps/job-launcher/server/src/modules/job/fixtures.ts index cf8f0ddf88..a4a0823533 100644 --- a/packages/apps/job-launcher/server/src/modules/job/fixtures.ts +++ b/packages/apps/job-launcher/server/src/modules/job/fixtures.ts @@ -1,16 +1,9 @@ import { faker } from '@faker-js/faker'; import { ChainId } from '@human-protocol/sdk'; -import { - getMockedProvider, - getMockedRegion, -} from '../../../test/fixtures/storage'; -import { - CvatJobType, - EscrowFundToken, - FortuneJobType, -} from '../../common/enums/job'; +import { EscrowFundToken, FortuneJobType } from '../../common/enums/job'; import { PaymentCurrency } from '../../common/enums/payment'; -import { JobCvatDto, JobFortuneDto } from './job.dto'; +import { createMockFortuneManifest } from '../manifest/fixtures'; +import { JobManifestDto } from './job.dto'; import { JobEntity } from './job.entity'; import { JobStatus } from '../../common/enums/job'; @@ -22,11 +15,10 @@ const escrowFundTokens = ( Object.values(EscrowFundToken) as EscrowFundToken[] ).filter((c) => c !== EscrowFundToken.HMT); -export const createFortuneJobDto = (overrides = {}): JobFortuneDto => ({ +export const createJobManifestDto = (overrides = {}): JobManifestDto => ({ chainId: ChainId.POLYGON_AMOY, - submissionsRequired: faker.number.int({ min: 1, max: 10 }), - requesterTitle: faker.lorem.words(3), - requesterDescription: faker.lorem.sentence(), + requestType: FortuneJobType.FORTUNE, + manifest: createMockFortuneManifest(), paymentAmount: faker.number.float({ min: 1, max: 100, fractionDigits: 6 }), paymentCurrency: faker.helpers.arrayElement(paymentCurrencies), escrowFundToken: faker.helpers.arrayElement(escrowFundTokens), @@ -36,36 +28,6 @@ export const createFortuneJobDto = (overrides = {}): JobFortuneDto => ({ ...overrides, }); -export const createCvatJobDto = (overrides = {}): JobCvatDto => ({ - chainId: ChainId.POLYGON_AMOY, - data: { - dataset: { - provider: getMockedProvider(), - region: getMockedRegion(), - bucketName: faker.lorem.word(), - path: faker.system.filePath(), - }, - }, - labels: [{ name: faker.lorem.word(), nodes: [faker.string.uuid()] }], - requesterDescription: faker.lorem.sentence(), - userGuide: faker.internet.url(), - minQuality: faker.number.float({ min: 0.1, max: 1 }), - groundTruth: { - provider: getMockedProvider(), - region: getMockedRegion(), - bucketName: faker.lorem.word(), - path: faker.system.filePath(), - }, - type: faker.helpers.arrayElement(Object.values(CvatJobType)), - paymentCurrency: faker.helpers.arrayElement(paymentCurrencies), - paymentAmount: faker.number.int({ min: 1, max: 1000 }), - escrowFundToken: faker.helpers.arrayElement(escrowFundTokens), - exchangeOracle: faker.finance.ethereumAddress(), - recordingOracle: faker.finance.ethereumAddress(), - reputationOracle: faker.finance.ethereumAddress(), - ...overrides, -}); - export const createJobEntity = ( overrides: Partial = {}, ): JobEntity => { @@ -93,7 +55,6 @@ export const createJobEntity = ( entity.status = faker.helpers.arrayElement(Object.values(JobStatus)); entity.userId = faker.number.int(); entity.payments = []; - entity.contentModerationRequests = []; entity.retriesCount = faker.number.int({ min: 0, max: 4 }); entity.waitUntil = faker.date.future(); Object.assign(entity, overrides); diff --git a/packages/apps/job-launcher/server/src/modules/job/job.controller.spec.ts b/packages/apps/job-launcher/server/src/modules/job/job.controller.spec.ts index bc2cb7a1df..5778fa133d 100644 --- a/packages/apps/job-launcher/server/src/modules/job/job.controller.spec.ts +++ b/packages/apps/job-launcher/server/src/modules/job/job.controller.spec.ts @@ -1,33 +1,30 @@ -import { Test, TestingModule } from '@nestjs/testing'; -import { JobController } from './job.controller'; -import { JobService } from './job.service'; +import { faker } from '@faker-js/faker'; +import { ChainId } from '@human-protocol/sdk'; import { BadRequestException, ConflictException, ExecutionContext, UnauthorizedException, } from '@nestjs/common'; +import { Test, TestingModule } from '@nestjs/testing'; import { MUTEX_TIMEOUT } from '../../common/constants'; -import { MutexManagerService } from '../mutex/mutex-manager.service'; -import { RequestWithUser } from '../../common/types'; -import { JwtAuthGuard } from '../../common/guards'; -import { JobFortuneDto, JobQuickLaunchDto } from './job.dto'; import { - // CvatJobType, + CvatJobType, EscrowFundToken, FortuneJobType, JobRequestType, } from '../../common/enums/job'; -import { - MOCK_FILE_HASH, - MOCK_FILE_URL, - MOCK_REQUESTER_DESCRIPTION, - MOCK_REQUESTER_TITLE, -} from '../../../test/constants'; -// import { AWSRegions, StorageProviders } from '../../common/enums/storage'; -import { Web3ConfigService } from '../../common/config/web3-config.service'; -import { ConfigService } from '@nestjs/config'; import { PaymentCurrency } from '../../common/enums/payment'; +import { JwtAuthGuard } from '../../common/guards'; +import { RequestWithUser } from '../../common/types'; +import { + createMockCvatManifest, + createMockFortuneManifest, +} from '../manifest/fixtures'; +import { MutexManagerService } from '../mutex/mutex-manager.service'; +import { JobController } from './job.controller'; +import { JobManifestDto, JobQuickLaunchDto } from './job.dto'; +import { JobService } from './job.service'; describe('JobController', () => { let jobController: JobController; @@ -56,8 +53,6 @@ describe('JobController', () => { provide: MutexManagerService, useValue: mockMutexManagerService, }, - Web3ConfigService, - ConfigService, ], }) .overrideGuard(JwtAuthGuard) @@ -84,11 +79,12 @@ describe('JobController', () => { describe('quickLaunch', () => { it('should create a job and return job ID', async () => { const jobDto: JobQuickLaunchDto = { + chainId: ChainId.POLYGON_AMOY, requestType: 'type_a' as JobRequestType, - manifestUrl: MOCK_FILE_URL, - manifestHash: MOCK_FILE_HASH, + manifestUrl: faker.internet.url(), + manifestHash: faker.string.uuid(), paymentCurrency: PaymentCurrency.USD, - paymentAmount: 500, + paymentAmount: faker.number.int({ min: 100, max: 1000 }), escrowFundToken: EscrowFundToken.HMT, }; @@ -119,11 +115,14 @@ describe('JobController', () => { it('should throw a conflict error if mutex manager fails', async () => { const jobDto: JobQuickLaunchDto = { + chainId: ChainId.POLYGON_AMOY, requestType: 'type_a' as JobRequestType, - manifestUrl: MOCK_FILE_URL, - manifestHash: MOCK_FILE_HASH, - paymentCurrency: PaymentCurrency.USD, - paymentAmount: 500, + manifestUrl: faker.internet.url(), + manifestHash: faker.string.uuid(), + paymentCurrency: faker.helpers.arrayElement( + Object.values(PaymentCurrency), + ), + paymentAmount: faker.number.int({ min: 100, max: 1000 }), escrowFundToken: EscrowFundToken.HMT, }; @@ -142,9 +141,13 @@ describe('JobController', () => { requestType: '', // Invalid input manifestUrl: '', manifestHash: '', - paymentCurrency: PaymentCurrency.USD, - paymentAmount: 500, - escrowFundToken: EscrowFundToken.HMT, + paymentCurrency: faker.helpers.arrayElement( + Object.values(PaymentCurrency), + ), + paymentAmount: faker.number.int({ min: 100, max: 1000 }), + escrowFundToken: faker.helpers.arrayElement( + Object.values(EscrowFundToken), + ), }; mockMutexManagerService.runExclusive.mockRejectedValueOnce( @@ -159,12 +162,17 @@ describe('JobController', () => { it('should return unauthorized error if user is not authenticated', async () => { const jobDto: JobQuickLaunchDto = { + chainId: ChainId.POLYGON_AMOY, requestType: 'type_a' as JobRequestType, - manifestUrl: MOCK_FILE_URL, - manifestHash: MOCK_FILE_HASH, - paymentCurrency: PaymentCurrency.USD, - paymentAmount: 500, - escrowFundToken: EscrowFundToken.HMT, + manifestUrl: faker.internet.url(), + manifestHash: faker.string.uuid(), + paymentCurrency: faker.helpers.arrayElement( + Object.values(PaymentCurrency), + ), + paymentAmount: faker.number.int({ min: 100, max: 1000 }), + escrowFundToken: faker.helpers.arrayElement( + Object.values(EscrowFundToken), + ), }; mockMutexManagerService.runExclusive.mockRejectedValueOnce( @@ -183,26 +191,31 @@ describe('JobController', () => { }); }); - describe('createFortuneJob', () => { - const jobFortuneDto: JobFortuneDto = { - requesterTitle: MOCK_REQUESTER_TITLE, - requesterDescription: MOCK_REQUESTER_DESCRIPTION, - submissionsRequired: 10, - paymentCurrency: PaymentCurrency.HMT, - paymentAmount: 500, - escrowFundToken: EscrowFundToken.HMT, + describe('createJob', () => { + const jobManifestDto: JobManifestDto = { + chainId: ChainId.POLYGON_AMOY, + requestType: FortuneJobType.FORTUNE, + manifest: createMockFortuneManifest({ + requesterTitle: faker.string.sample(), + requesterDescription: faker.string.sample(), + submissionsRequired: faker.number.int({ min: 1, max: 10 }), + }), + paymentCurrency: faker.helpers.arrayElement( + Object.values(PaymentCurrency), + ), + paymentAmount: faker.number.int({ min: 100, max: 1000 }), + escrowFundToken: faker.helpers.arrayElement( + Object.values(EscrowFundToken), + ), }; - it('should create a fortune job successfully', async () => { + it('should create a job successfully', async () => { mockJobService.createJob.mockResolvedValue(1); mockMutexManagerService.runExclusive.mockImplementation( async (_lock, _timeout, fn) => await fn(), ); - const result = await jobController.createFortuneJob( - jobFortuneDto, - mockRequest, - ); + const result = await jobController.createJob(jobManifestDto, mockRequest); expect(result).toBe(1); expect(mockMutexManagerService.runExclusive).toHaveBeenCalledWith( @@ -212,8 +225,43 @@ describe('JobController', () => { ); expect(mockJobService.createJob).toHaveBeenCalledWith( mockRequest.user, - FortuneJobType.FORTUNE, - jobFortuneDto, + jobManifestDto.requestType, + jobManifestDto, + ); + }); + + it('should create a CVAT job successfully', async () => { + const cvatManifest = createMockCvatManifest(); + cvatManifest.annotation.type = CvatJobType.IMAGE_BOXES; + + const cvatJobManifestDto: JobManifestDto = { + chainId: ChainId.POLYGON_AMOY, + requestType: CvatJobType.IMAGE_BOXES, + manifest: cvatManifest, + paymentCurrency: faker.helpers.arrayElement( + Object.values(PaymentCurrency), + ), + paymentAmount: faker.number.int({ min: 100, max: 1000 }), + escrowFundToken: faker.helpers.arrayElement( + Object.values(EscrowFundToken), + ), + }; + + mockJobService.createJob.mockResolvedValue(2); + mockMutexManagerService.runExclusive.mockImplementation( + async (_lock, _timeout, fn) => await fn(), + ); + + const result = await jobController.createJob( + cvatJobManifestDto, + mockRequest, + ); + + expect(result).toBe(2); + expect(mockJobService.createJob).toHaveBeenCalledWith( + mockRequest.user, + cvatJobManifestDto.requestType, + cvatJobManifestDto, ); }); @@ -223,7 +271,7 @@ describe('JobController', () => { ); await expect( - jobController.createFortuneJob(jobFortuneDto, mockRequest), + jobController.createJob(jobManifestDto, mockRequest), ).rejects.toThrow(UnauthorizedException); expect(mockMutexManagerService.runExclusive).toHaveBeenCalledWith( @@ -240,7 +288,7 @@ describe('JobController', () => { ); await expect( - jobController.createFortuneJob(jobFortuneDto, mockRequest), + jobController.createJob(jobManifestDto, mockRequest), ).rejects.toThrow(ConflictException); expect(mockMutexManagerService.runExclusive).toHaveBeenCalledWith( @@ -257,7 +305,7 @@ describe('JobController', () => { ); await expect( - jobController.createFortuneJob(jobFortuneDto, mockRequest), + jobController.createJob(jobManifestDto, mockRequest), ).rejects.toThrow(BadRequestException); expect(mockMutexManagerService.runExclusive).toHaveBeenCalledWith( @@ -268,109 +316,4 @@ describe('JobController', () => { expect(mockJobService.createJob).not.toHaveBeenCalled(); }); }); - - //disabled CVAT jobs - // describe('createCvatJob', () => { - // const jobCvatDto: JobCvatDto = { - // requesterDescription: 'Sample description', - // data: { - // dataset: { - // provider: 'AWS' as StorageProviders, - // region: 'us-east-1' as AWSRegions, - // bucketName: 'sample-bucket', - // path: 'path/to/dataset', - // }, - // }, - // labels: [ - // { - // name: 'Label 1', - // nodes: ['node1', 'node2'], - // }, - // ], - // minQuality: 90, - // groundTruth: { - // provider: 'AWS' as StorageProviders, - // region: 'us-west-1' as AWSRegions, - // bucketName: 'ground-truth-bucket', - // path: 'path/to/groundtruth', - // }, - // userGuide: 'https://example.com/user-guide', - // type: CvatJobType.IMAGE_BOXES, - // paymentCurrency: PaymentCurrency.USDC, - // paymentAmount: 500, - // escrowFundToken: EscrowFundToken.USDC, - // }; - - // it('should create a CVAT job successfully', async () => { - // mockJobService.createJob.mockResolvedValue(1); - // mockMutexManagerService.runExclusive.mockImplementation( - // async (_lock, _timeout, fn) => await fn(), - // ); - - // const result = await jobController.createCvatJob(jobCvatDto, mockRequest); - - // expect(result).toBe(1); - // expect(mockMutexManagerService.runExclusive).toHaveBeenCalledWith( - // `user${mockRequest.user.id}`, - // expect.any(Number), - // expect.any(Function), - // ); - // expect(mockJobService.createJob).toHaveBeenCalledWith( - // mockRequest.user, - // CvatJobType.IMAGE_BOXES, - // jobCvatDto, - // ); - // }); - - // it('should throw UnauthorizedException if user is not authorized', async () => { - // mockMutexManagerService.runExclusive.mockRejectedValueOnce( - // new UnauthorizedException(), - // ); - - // await expect( - // jobController.createCvatJob(jobCvatDto, mockRequest), - // ).rejects.toThrow(UnauthorizedException); - - // expect(mockMutexManagerService.runExclusive).toHaveBeenCalledWith( - // `user${mockRequest.user.id}`, - // expect.any(Number), - // expect.any(Function), - // ); - // expect(mockJobService.createJob).not.toHaveBeenCalled(); - // }); - - // it('should throw ConflictException if there is a conflict', async () => { - // mockMutexManagerService.runExclusive.mockRejectedValueOnce( - // new ConflictException(), - // ); - - // await expect( - // jobController.createCvatJob(jobCvatDto, mockRequest), - // ).rejects.toThrow(ConflictException); - - // expect(mockMutexManagerService.runExclusive).toHaveBeenCalledWith( - // `user${mockRequest.user.id}`, - // expect.any(Number), - // expect.any(Function), - // ); - // expect(mockJobService.createJob).not.toHaveBeenCalled(); - // }); - - // it('should throw BadRequestException for invalid input', async () => { - // mockMutexManagerService.runExclusive.mockRejectedValueOnce( - // new BadRequestException(), - // ); - - // await expect( - // jobController.createCvatJob(jobCvatDto, mockRequest), - // ).rejects.toThrow(BadRequestException); - - // expect(mockMutexManagerService.runExclusive).toHaveBeenCalledWith( - // `user${mockRequest.user.id}`, - // expect.any(Number), - // expect.any(Function), - // ); - // expect(mockJobService.createJob).not.toHaveBeenCalled(); - // }); - // }); }); diff --git a/packages/apps/job-launcher/server/src/modules/job/job.controller.ts b/packages/apps/job-launcher/server/src/modules/job/job.controller.ts index bc7ae37fb9..abb5dbdf0f 100644 --- a/packages/apps/job-launcher/server/src/modules/job/job.controller.ts +++ b/packages/apps/job-launcher/server/src/modules/job/job.controller.ts @@ -18,12 +18,8 @@ import { ApiResponse, ApiTags, } from '@nestjs/swagger'; -import { Web3ConfigService } from '../../common/config/web3-config.service'; import { MUTEX_TIMEOUT } from '../../common/constants'; import { ApiKey } from '../../common/decorators'; -import { FortuneJobType } from '../../common/enums/job'; -import { Web3Env } from '../../common/enums/web3'; -import { ForbiddenError } from '../../common/errors'; import { JwtAuthGuard } from '../../common/guards'; import { PageDto } from '../../common/pagination/pagination.dto'; import { RequestWithUser } from '../../common/types'; @@ -32,11 +28,10 @@ import { FortuneFinalResultDto, GetJobsDto, JobCancelDto, - JobCvatDto, JobDetailsDto, - JobFortuneDto, JobIdDto, JobListDto, + JobManifestDto, JobQuickLaunchDto, } from './job.dto'; import { JobService } from './job.service'; @@ -50,7 +45,6 @@ export class JobController { constructor( private readonly jobService: JobService, private readonly mutexManagerService: MutexManagerService, - private readonly web3ConfigService: Web3ConfigService, ) {} @ApiOperation({ @@ -95,13 +89,13 @@ export class JobController { } @ApiOperation({ - summary: 'Create a fortune job', - description: 'Endpoint to create a new fortune job.', + summary: 'Create a job', + description: 'Endpoint to create a new job using a manifest JSON body.', }) - @ApiBody({ type: JobFortuneDto }) + @ApiBody({ type: JobManifestDto }) @ApiResponse({ status: 201, - description: 'ID of the created fortune job.', + description: 'ID of the created job.', type: Number, }) @ApiResponse({ @@ -116,65 +110,24 @@ export class JobController { status: 409, description: 'Conflict. Conflict with the current state of the server.', }) - @Post('/fortune') - public async createFortuneJob( - @Body() data: JobFortuneDto, + @Post() + public async createJob( + @Body() data: JobManifestDto, @Request() req: RequestWithUser, ): Promise { - if (this.web3ConfigService.env === Web3Env.MAINNET) { - throw new ForbiddenError('Disabled'); - } - return await this.mutexManagerService.runExclusive( `user${req.user.id}`, MUTEX_TIMEOUT, async () => { return await this.jobService.createJob( req.user, - FortuneJobType.FORTUNE, + data.requestType, data, ); }, ); } - @ApiOperation({ - summary: 'Create a CVAT job', - description: 'Endpoint to create a new CVAT job.', - }) - @ApiBody({ type: JobCvatDto }) - @ApiResponse({ - status: 201, - description: 'ID of the created CVAT job.', - type: Number, - }) - @ApiResponse({ - status: 400, - description: 'Bad Request. Invalid input parameters.', - }) - @ApiResponse({ - status: 401, - description: 'Unauthorized. Missing or invalid credentials.', - }) - @ApiResponse({ - status: 409, - description: 'Conflict. Conflict with the current state of the server.', - }) - @Post('/cvat') - public async createCvatJob( - @Body() data: JobCvatDto, - @Request() req: RequestWithUser, - ): Promise { - throw new ForbiddenError('Disabled'); - return await this.mutexManagerService.runExclusive( - `user${req.user.id}`, - MUTEX_TIMEOUT, - async () => { - return await this.jobService.createJob(req.user, data.type, data); - }, - ); - } - @ApiOperation({ summary: 'Get a list of jobs', description: diff --git a/packages/apps/job-launcher/server/src/modules/job/job.dto.ts b/packages/apps/job-launcher/server/src/modules/job/job.dto.ts index 729fcb3ccb..dc5822aa8d 100644 --- a/packages/apps/job-launcher/server/src/modules/job/job.dto.ts +++ b/packages/apps/job-launcher/server/src/modules/job/job.dto.ts @@ -2,25 +2,22 @@ import { ChainId } from '@human-protocol/sdk'; import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger'; import { Transform, Type } from 'class-transformer'; import { - ArrayMinSize, IsArray, IsEthereumAddress, IsIn, IsNotEmpty, IsNumber, IsNumberString, - IsObject, IsOptional, IsPositive, IsString, - IsUrl, - Max, Min, + IsUrl, ValidateNested, + IsObject, } from 'class-validator'; import { IsEnumCaseInsensitive } from '../../common/decorators'; import { - CvatJobType, EscrowFundToken, JobRequestType, JobSortField, @@ -33,13 +30,20 @@ import { AWSRegions, StorageProviders } from '../../common/enums/storage'; import { PageOptionsDto } from '../../common/pagination/pagination.dto'; import { IsValidTokenDecimals } from '../../common/validators/token-decimals'; import { IsValidToken } from '../../common/validators/tokens'; -import { Label, ManifestDetails } from '../manifest/manifest.dto'; +import { ManifestDetails, ManifestDto } from '../manifest/manifest.dto'; export class JobDto { - @ApiProperty({ enum: ChainId, required: false, name: 'chain_id' }) + @ApiProperty({ enum: ChainId, name: 'chain_id' }) @IsEnumCaseInsensitive(ChainId) - @IsOptional() - public chainId?: ChainId; + public chainId: ChainId; + + @ApiProperty({ + description: 'Request type', + name: 'request_type', + enum: JobType, + }) + @IsEnumCaseInsensitive(JobType) + public requestType: JobRequestType; @ApiPropertyOptional() @IsArray() @@ -89,14 +93,6 @@ export class JobDto { } export class JobQuickLaunchDto extends JobDto { - @ApiProperty({ - description: 'Request type', - name: 'request_type', - enum: JobType, - }) - @IsEnumCaseInsensitive(JobType) - public requestType: JobRequestType; - @ApiProperty({ name: 'manifest_url' }) @IsUrl() @IsNotEmpty() @@ -105,24 +101,14 @@ export class JobQuickLaunchDto extends JobDto { @ApiProperty({ name: 'manifest_hash' }) @IsString() @IsOptional() - public manifestHash: string; + public manifestHash?: string; } -export class JobFortuneDto extends JobDto { - @ApiProperty({ name: 'requester_title' }) - @IsString() - @IsNotEmpty() - public requesterTitle: string; - - @ApiProperty({ name: 'requester_description' }) - @IsString() +export class JobManifestDto extends JobDto { + @ApiProperty({ type: Object }) + @IsObject() @IsNotEmpty() - public requesterDescription: string; - - @ApiProperty({ name: 'submissions_required' }) - @IsNumber() - @IsPositive() - public submissionsRequired: number; + public manifest: ManifestDto; } export class StorageDataDto { @@ -145,68 +131,6 @@ export class StorageDataDto { public path?: string; } -export class CvatDataDto { - @ApiProperty() - @IsObject() - @ValidateNested() - @Type(() => StorageDataDto) - public dataset: StorageDataDto; - - @ApiPropertyOptional() - @IsObject() - @IsOptional() - @ValidateNested() - @Type(() => StorageDataDto) - public points?: StorageDataDto; - - @ApiPropertyOptional() - @IsObject() - @IsOptional() - @ValidateNested() - @Type(() => StorageDataDto) - public boxes?: StorageDataDto; -} - -export class JobCvatDto extends JobDto { - @ApiProperty({ name: 'requester_description' }) - @IsString() - @IsNotEmpty() - public requesterDescription: string; - - @ApiProperty() - @IsObject() - @ValidateNested() - @Type(() => CvatDataDto) - public data: CvatDataDto; - - @ApiProperty({ type: [Label] }) - @IsArray() - @ArrayMinSize(1) - @ValidateNested({ each: true }) - @Type(() => Label) - public labels: Label[]; - - @ApiProperty({ name: 'min_quality' }) - @IsNumber() - @IsPositive() - @Max(1) - public minQuality: number; - - @ApiProperty({ name: 'ground_truth' }) - @IsObject() - @ValidateNested() - @Type(() => StorageDataDto) - public groundTruth: StorageDataDto; - - @ApiProperty({ name: 'user_guide' }) - @IsUrl() - public userGuide: string; - - @ApiProperty({ enum: CvatJobType }) - @IsEnumCaseInsensitive(CvatJobType) - public type: CvatJobType; -} - export class JobCancelDto { @ApiProperty() @IsNumberString() @@ -364,4 +288,4 @@ export class GetJobsDto extends PageOptionsDto { status?: JobStatusFilter; } -export type CreateJob = JobQuickLaunchDto | JobFortuneDto | JobCvatDto; +export type CreateJob = JobQuickLaunchDto | JobManifestDto; diff --git a/packages/apps/job-launcher/server/src/modules/job/job.entity.ts b/packages/apps/job-launcher/server/src/modules/job/job.entity.ts index 3887a7670b..56f5cbc38c 100644 --- a/packages/apps/job-launcher/server/src/modules/job/job.entity.ts +++ b/packages/apps/job-launcher/server/src/modules/job/job.entity.ts @@ -6,7 +6,6 @@ import { JobRequestType, JobStatus, JobType } from '../../common/enums/job'; import { BaseEntity } from '../../database/base.entity'; import { UserEntity } from '../user/user.entity'; import { PaymentEntity } from '../payment/payment.entity'; -import { ContentModerationRequestEntity } from '../content-moderation/content-moderation-request.entity'; @Entity({ schema: NS, name: 'jobs' }) @Index(['chainId', 'escrowAddress'], { unique: true }) @@ -65,13 +64,6 @@ export class JobEntity extends BaseEntity implements IJob { @OneToMany(() => PaymentEntity, (payment) => payment.job) public payments: PaymentEntity[]; - @OneToMany( - () => ContentModerationRequestEntity, - (contentModerationRequest) => contentModerationRequest.job, - { cascade: ['insert'] }, - ) - public contentModerationRequests: ContentModerationRequestEntity[]; - @Column({ type: 'int', default: 0 }) public retriesCount: number; diff --git a/packages/apps/job-launcher/server/src/modules/job/job.interface.ts b/packages/apps/job-launcher/server/src/modules/job/job.interface.ts index b4ad55106a..c1aa8b2b75 100644 --- a/packages/apps/job-launcher/server/src/modules/job/job.interface.ts +++ b/packages/apps/job-launcher/server/src/modules/job/job.interface.ts @@ -1,86 +1,5 @@ -import { CvatJobType, JobRequestType } from '../../common/enums/job'; -import { - CvatDataDto, - JobCvatDto, - JobFortuneDto, - StorageDataDto, -} from './job.dto'; import { JobEntity } from './job.entity'; -export interface RequestAction { - createManifest: ( - dto: JobFortuneDto | JobCvatDto, - requestType: JobRequestType, - fundAmount: number, - decimals: number, - ) => Promise; -} - -export interface ManifestAction { - getElementsCount: (urls: GenerateUrls) => Promise; - generateUrls: ( - data: CvatDataDto, - groundTruth: StorageDataDto, - ) => GenerateUrls; -} - -export interface EscrowAction { - getTrustedHandlers: () => string[]; -} - -export interface OracleAction { - getOracleAddresses: () => OracleAddresses; -} - -export interface OracleAddresses { - exchangeOracle: string; - recordingOracle: string; - reputationOracle: string; -} - -export interface CvatCalculateJobBounty { - requestType: CvatJobType; - fundAmount: number; - decimals: number; - urls: GenerateUrls; - nodesTotal?: number; -} - -export interface GenerateUrls { - dataUrl: URL; - gtUrl: URL; - pointsUrl?: URL; - boxesUrl?: URL; -} - -export interface CvatImageData { - id: number; - width: number; - height: number; - file_name: string; - license: number; - flickr_url: string; - coco_url: string; - date_captured: number; -} - -export interface CvatAnnotationData { - id: number; - image_id: number; - category_id: number; - segmentation: number[]; - area: number; - bbox: [number, number, number, number]; - iscrowd: number; - attributes: { - scale: number; - x: number; - y: number; - }; - keypoints: [number, number, number]; - num_keypoints: number; -} - export interface ListResult { entities: JobEntity[]; itemCount: number; diff --git a/packages/apps/job-launcher/server/src/modules/job/job.module.ts b/packages/apps/job-launcher/server/src/modules/job/job.module.ts index 531a5e8beb..9ebe2c27ca 100644 --- a/packages/apps/job-launcher/server/src/modules/job/job.module.ts +++ b/packages/apps/job-launcher/server/src/modules/job/job.module.ts @@ -15,7 +15,6 @@ import { WebhookRepository } from '../webhook/webhook.repository'; import { MutexManagerService } from '../mutex/mutex-manager.service'; import { QualificationModule } from '../qualification/qualification.module'; import { WhitelistModule } from '../whitelist/whitelist.module'; -import { RoutingProtocolModule } from '../routing-protocol/routing-protocol.module'; import { RateModule } from '../rate/rate.module'; import { ManifestModule } from '../manifest/manifest.module'; @@ -29,7 +28,6 @@ import { ManifestModule } from '../manifest/manifest.module'; StorageModule, QualificationModule, WhitelistModule, - RoutingProtocolModule, RateModule, ManifestModule, ], diff --git a/packages/apps/job-launcher/server/src/modules/job/job.repository.ts b/packages/apps/job-launcher/server/src/modules/job/job.repository.ts index 0dadd98f28..9e68fcbb47 100644 --- a/packages/apps/job-launcher/server/src/modules/job/job.repository.ts +++ b/packages/apps/job-launcher/server/src/modules/job/job.repository.ts @@ -81,7 +81,6 @@ export class JobRepository extends BaseRepository { waitUntil: SortDirection.ASC, }, ...(take && { take }), - relations: ['contentModerationRequests'], }); } @@ -108,12 +107,7 @@ export class JobRepository extends BaseRepository { switch (data.status) { case JobStatusFilter.PENDING: - statusFilter = [ - JobStatus.PAID, - JobStatus.UNDER_MODERATION, - JobStatus.MODERATION_PASSED, - JobStatus.POSSIBLE_ABUSE_IN_REVIEW, - ]; + statusFilter = [JobStatus.PAID]; break; case JobStatusFilter.CANCELED: statusFilter = [ diff --git a/packages/apps/job-launcher/server/src/modules/job/job.service.spec.ts b/packages/apps/job-launcher/server/src/modules/job/job.service.spec.ts index 27178d5ef5..1ac1487f52 100644 --- a/packages/apps/job-launcher/server/src/modules/job/job.service.spec.ts +++ b/packages/apps/job-launcher/server/src/modules/job/job.service.spec.ts @@ -10,6 +10,7 @@ import { IEscrow, KVStoreUtils, NETWORKS, + Role, } from '@human-protocol/sdk'; import { Test } from '@nestjs/testing'; import { ethers, ZeroAddress } from 'ethers'; @@ -48,22 +49,17 @@ import { PaymentRepository } from '../payment/payment.repository'; import { PaymentService } from '../payment/payment.service'; import { QualificationService } from '../qualification/qualification.service'; import { RateService } from '../rate/rate.service'; -import { RoutingProtocolService } from '../routing-protocol/routing-protocol.service'; import { StorageService } from '../storage/storage.service'; import { createUser } from '../user/fixtures'; import { Web3Service } from '../web3/web3.service'; import { WebhookRepository } from '../webhook/webhook.repository'; import { WhitelistEntity } from '../whitelist/whitelist.entity'; import { WhitelistService } from '../whitelist/whitelist.service'; -import { - createCvatJobDto, - createFortuneJobDto, - createJobEntity, -} from './fixtures'; +import { createJobEntity, createJobManifestDto } from './fixtures'; import { FortuneFinalResultDto, GetJobsDto, - JobFortuneDto, + JobManifestDto, JobQuickLaunchDto, } from './job.dto'; import { JobRepository } from './job.repository'; @@ -78,11 +74,14 @@ const mockPaymentRepository = createMock(); const mockStorageService = createMock(); const mockPaymentService = createMock(); const mockRateService = createMock(); -const mockRoutingProtocolService = createMock(); const mockManifestService = createMock(); const mockWhitelistService = createMock(); const mockWeb3ConfigService = { txTimeoutMs: faker.number.int({ min: 30000, max: 120000 }), + reputationOracleAddress: faker.finance.ethereumAddress(), + cvatExchangeOracleAddress: faker.finance.ethereumAddress(), + cvatRecordingOracleAddress: faker.finance.ethereumAddress(), + hCaptchaOracleAddress: faker.finance.ethereumAddress(), }; const mockedEscrowClient = jest.mocked(EscrowClient); @@ -123,10 +122,6 @@ describe('JobService', () => { { provide: PaymentService, useValue: mockPaymentService }, { provide: StorageService, useValue: mockStorageService }, { provide: WhitelistService, useValue: mockWhitelistService }, - { - provide: RoutingProtocolService, - useValue: mockRoutingProtocolService, - }, { provide: ManifestService, useValue: mockManifestService, @@ -149,22 +144,16 @@ describe('JobService', () => { describe('Fortune', () => { it('should create a Fortune job successfully paid and funded with the same currency', async () => { - const fortuneJobDto: JobFortuneDto = createFortuneJobDto({ + const jobManifestDto: JobManifestDto = createJobManifestDto({ paymentCurrency: PaymentCurrency.USDC, escrowFundToken: EscrowFundToken.USDC, + manifest: createMockFortuneManifest(), }); const fundTokenDecimals = getTokenDecimals( - fortuneJobDto.chainId!, - fortuneJobDto.escrowFundToken, + jobManifestDto.chainId!, + jobManifestDto.escrowFundToken, ); - const mockManifest = createMockFortuneManifest({ - submissionsRequired: fortuneJobDto.submissionsRequired, - requesterTitle: fortuneJobDto.requesterTitle, - requesterDescription: fortuneJobDto.requesterDescription, - fundAmount: fortuneJobDto.paymentAmount, - }); - mockManifestService.createManifest.mockResolvedValueOnce(mockManifest); const mockUrl = faker.internet.url(); const mockHash = faker.string.uuid(); mockManifestService.uploadManifest.mockResolvedValueOnce({ @@ -181,51 +170,40 @@ describe('JobService', () => { const result = await jobService.createJob( userMock, FortuneJobType.FORTUNE, - fortuneJobDto, + jobManifestDto, ); const paymentCurrencyFee = Number( max( div(mockServerConfigService.minimumFeeUsd, tokenToUsdRate), - mul(div(1, 100), fortuneJobDto.paymentAmount), + mul(div(1, 100), jobManifestDto.paymentAmount), ).toFixed(18), ); - expect(result).toBe(jobEntityMock.id); expect(mockWeb3Service.validateChainId).toHaveBeenCalledWith( - fortuneJobDto.chainId, - ); - expect(mockRoutingProtocolService.selectOracles).not.toHaveBeenCalled(); - expect(mockRoutingProtocolService.validateOracles).toHaveBeenCalledWith( - fortuneJobDto.chainId, - FortuneJobType.FORTUNE, - fortuneJobDto.reputationOracle, - fortuneJobDto.exchangeOracle, - fortuneJobDto.recordingOracle, + jobManifestDto.chainId, ); - expect(mockManifestService.createManifest).toHaveBeenCalledWith( - fortuneJobDto, + expect(mockManifestService.validateManifest).toHaveBeenCalledWith( FortuneJobType.FORTUNE, - fortuneJobDto.paymentAmount, - fundTokenDecimals, + jobManifestDto.manifest, ); expect(mockManifestService.uploadManifest).toHaveBeenCalledWith( - fortuneJobDto.chainId, - mockManifest, + jobManifestDto.chainId, + jobManifestDto.manifest, [ - fortuneJobDto.exchangeOracle, - fortuneJobDto.reputationOracle, - fortuneJobDto.recordingOracle, + jobManifestDto.exchangeOracle, + jobManifestDto.reputationOracle, + jobManifestDto.recordingOracle, ], ); expect(mockPaymentService.createWithdrawalPayment).toHaveBeenCalledWith( userMock.id, expect.any(Number), - fortuneJobDto.paymentCurrency, + jobManifestDto.paymentCurrency, tokenToUsdRate, ); expect(mockJobRepository.updateOne).toHaveBeenCalledWith({ - chainId: fortuneJobDto.chainId, + chainId: jobManifestDto.chainId, userId: userMock.id, manifestUrl: mockUrl, manifestHash: mockHash, @@ -236,35 +214,29 @@ describe('JobService', () => { usdToTokenRate, ).toFixed(fundTokenDecimals), ), - fundAmount: fortuneJobDto.paymentAmount, - status: JobStatus.MODERATION_PASSED, + fundAmount: jobManifestDto.paymentAmount, + status: JobStatus.PAID, waitUntil: expect.any(Date), - token: fortuneJobDto.escrowFundToken, - exchangeOracle: fortuneJobDto.exchangeOracle, - recordingOracle: fortuneJobDto.recordingOracle, - reputationOracle: fortuneJobDto.reputationOracle, + token: jobManifestDto.escrowFundToken, + exchangeOracle: jobManifestDto.exchangeOracle, + recordingOracle: jobManifestDto.recordingOracle, + reputationOracle: jobManifestDto.reputationOracle, payments: expect.any(Array), }); }); it('should create a Fortune job successfully paid and funded with different currencies', async () => { - const fortuneJobDto: JobFortuneDto = createFortuneJobDto({ + const jobManifestDto: JobManifestDto = createJobManifestDto({ paymentCurrency: PaymentCurrency.USD, escrowFundToken: EscrowFundToken.USDC, + manifest: createMockFortuneManifest(), }); const fundTokenDecimals = getTokenDecimals( - fortuneJobDto.chainId!, - fortuneJobDto.escrowFundToken, + jobManifestDto.chainId!, + jobManifestDto.escrowFundToken, ); - const mockManifest = createMockFortuneManifest({ - submissionsRequired: fortuneJobDto.submissionsRequired, - requesterTitle: fortuneJobDto.requesterTitle, - requesterDescription: fortuneJobDto.requesterDescription, - fundAmount: fortuneJobDto.paymentAmount, - }); - mockManifestService.createManifest.mockResolvedValueOnce(mockManifest); const mockUrl = faker.internet.url(); const mockHash = faker.string.uuid(); mockManifestService.uploadManifest.mockResolvedValueOnce({ @@ -281,52 +253,47 @@ describe('JobService', () => { const result = await jobService.createJob( userMock, FortuneJobType.FORTUNE, - fortuneJobDto, + jobManifestDto, ); const paymentCurrencyFee = Number( max( div(mockServerConfigService.minimumFeeUsd, tokenToUsdRate), - mul(div(1, 100), fortuneJobDto.paymentAmount), + mul(div(1, 100), jobManifestDto.paymentAmount), ).toFixed(18), ); - + const expectedFundAmount = Number( + mul( + mul(jobManifestDto.paymentAmount, tokenToUsdRate), + usdToTokenRate, + ).toFixed(6), + ); expect(result).toBe(jobEntityMock.id); expect(mockWeb3Service.validateChainId).toHaveBeenCalledWith( - fortuneJobDto.chainId, + jobManifestDto.chainId, ); - expect(mockRoutingProtocolService.selectOracles).not.toHaveBeenCalled(); - expect(mockRoutingProtocolService.validateOracles).toHaveBeenCalledWith( - fortuneJobDto.chainId, + expect(mockManifestService.validateManifest).toHaveBeenCalledWith( FortuneJobType.FORTUNE, - fortuneJobDto.reputationOracle, - fortuneJobDto.exchangeOracle, - fortuneJobDto.recordingOracle, - ); - expect(mockManifestService.createManifest).toHaveBeenCalledWith( - fortuneJobDto, - FortuneJobType.FORTUNE, - Number(fortuneJobDto.paymentAmount.toFixed(6)), - fundTokenDecimals, + jobManifestDto.manifest, ); expect(mockManifestService.uploadManifest).toHaveBeenCalledWith( - fortuneJobDto.chainId, - mockManifest, + jobManifestDto.chainId, + jobManifestDto.manifest, [ - fortuneJobDto.exchangeOracle, - fortuneJobDto.reputationOracle, - fortuneJobDto.recordingOracle, + jobManifestDto.exchangeOracle, + jobManifestDto.reputationOracle, + jobManifestDto.recordingOracle, ], ); expect(mockPaymentService.createWithdrawalPayment).toHaveBeenCalledWith( userMock.id, expect.any(Number), - fortuneJobDto.paymentCurrency, + jobManifestDto.paymentCurrency, tokenToUsdRate, ); expect(mockJobRepository.updateOne).toHaveBeenCalledWith({ - chainId: fortuneJobDto.chainId, + chainId: jobManifestDto.chainId, userId: userMock.id, manifestUrl: mockUrl, manifestHash: mockHash, @@ -337,43 +304,32 @@ describe('JobService', () => { usdToTokenRate, ).toFixed(fundTokenDecimals), ), - fundAmount: Number( - mul( - mul(fortuneJobDto.paymentAmount, tokenToUsdRate), - usdToTokenRate, - ).toFixed(6), - ), - status: JobStatus.MODERATION_PASSED, + fundAmount: expectedFundAmount, + status: JobStatus.PAID, waitUntil: expect.any(Date), - token: fortuneJobDto.escrowFundToken, - exchangeOracle: fortuneJobDto.exchangeOracle, - recordingOracle: fortuneJobDto.recordingOracle, - reputationOracle: fortuneJobDto.reputationOracle, + token: jobManifestDto.escrowFundToken, + exchangeOracle: jobManifestDto.exchangeOracle, + recordingOracle: jobManifestDto.recordingOracle, + reputationOracle: jobManifestDto.reputationOracle, payments: expect.any(Array), }); }); it('should select the right oracles when no oracle addresses provided', async () => { - const fortuneJobDto: JobFortuneDto = createFortuneJobDto({ + const jobManifestDto: JobManifestDto = createJobManifestDto({ paymentCurrency: EscrowFundToken.USDC, escrowFundToken: EscrowFundToken.USDC, exchangeOracle: null, recordingOracle: null, reputationOracle: null, + manifest: createMockFortuneManifest(), }); const fundTokenDecimals = getTokenDecimals( - fortuneJobDto.chainId!, - fortuneJobDto.escrowFundToken, + jobManifestDto.chainId!, + jobManifestDto.escrowFundToken, ); - const mockManifest = createMockFortuneManifest({ - submissionsRequired: fortuneJobDto.submissionsRequired, - requesterTitle: fortuneJobDto.requesterTitle, - requesterDescription: fortuneJobDto.requesterDescription, - fundAmount: fortuneJobDto.paymentAmount, - }); - mockManifestService.createManifest.mockResolvedValueOnce(mockManifest); const mockUrl = faker.internet.url(); const mockHash = faker.string.uuid(); mockManifestService.uploadManifest.mockResolvedValueOnce({ @@ -388,48 +344,50 @@ describe('JobService', () => { const mockOracles = { recordingOracle: faker.finance.ethereumAddress(), exchangeOracle: faker.finance.ethereumAddress(), - reputationOracle: faker.finance.ethereumAddress(), + reputationOracle: mockWeb3ConfigService.reputationOracleAddress, }; - mockRoutingProtocolService.selectOracles.mockResolvedValueOnce({ - recordingOracle: mockOracles.recordingOracle, - exchangeOracle: mockOracles.exchangeOracle, - reputationOracle: mockOracles.reputationOracle, - }); + mockWeb3Service.findAvailableOracles.mockResolvedValueOnce([ + { + address: mockOracles.exchangeOracle, + role: Role.ExchangeOracle, + url: null, + }, + { + address: mockOracles.recordingOracle, + role: Role.RecordingOracle, + url: null, + }, + ]); mockedKVStoreUtils.get.mockResolvedValueOnce('1'); const result = await jobService.createJob( userMock, FortuneJobType.FORTUNE, - fortuneJobDto, + jobManifestDto, ); const paymentCurrencyFee = Number( max( div(mockServerConfigService.minimumFeeUsd, tokenToUsdRate), - mul(div(1, 100), fortuneJobDto.paymentAmount), + mul(div(1, 100), jobManifestDto.paymentAmount), ).toFixed(18), ); - expect(result).toBe(jobEntityMock.id); expect(mockWeb3Service.validateChainId).toHaveBeenCalledWith( - fortuneJobDto.chainId, + jobManifestDto.chainId, ); - expect(mockRoutingProtocolService.selectOracles).toHaveBeenCalledWith( - fortuneJobDto.chainId, + expect(mockWeb3Service.findAvailableOracles).toHaveBeenCalledWith( + jobManifestDto.chainId, FortuneJobType.FORTUNE, + mockOracles.reputationOracle, ); - expect( - mockRoutingProtocolService.validateOracles, - ).not.toHaveBeenCalled(); - expect(mockManifestService.createManifest).toHaveBeenCalledWith( - fortuneJobDto, + expect(mockManifestService.validateManifest).toHaveBeenCalledWith( FortuneJobType.FORTUNE, - fortuneJobDto.paymentAmount, - fundTokenDecimals, + jobManifestDto.manifest, ); expect(mockManifestService.uploadManifest).toHaveBeenCalledWith( - fortuneJobDto.chainId, - mockManifest, + jobManifestDto.chainId, + jobManifestDto.manifest, [ mockOracles.exchangeOracle, mockOracles.reputationOracle, @@ -439,11 +397,11 @@ describe('JobService', () => { expect(mockPaymentService.createWithdrawalPayment).toHaveBeenCalledWith( userMock.id, expect.any(Number), - fortuneJobDto.paymentCurrency, + jobManifestDto.paymentCurrency, tokenToUsdRate, ); expect(mockJobRepository.updateOne).toHaveBeenCalledWith({ - chainId: fortuneJobDto.chainId, + chainId: jobManifestDto.chainId, userId: userMock.id, manifestUrl: mockUrl, manifestHash: mockHash, @@ -454,10 +412,10 @@ describe('JobService', () => { usdToTokenRate, ).toFixed(fundTokenDecimals), ), - fundAmount: fortuneJobDto.paymentAmount, - status: JobStatus.MODERATION_PASSED, + fundAmount: jobManifestDto.paymentAmount, + status: JobStatus.PAID, waitUntil: expect.any(Date), - token: fortuneJobDto.escrowFundToken, + token: jobManifestDto.escrowFundToken, exchangeOracle: mockOracles.exchangeOracle, recordingOracle: mockOracles.recordingOracle, reputationOracle: mockOracles.reputationOracle, @@ -467,12 +425,12 @@ describe('JobService', () => { it('should throw if user is not whitelisted and has no payment method', async () => { mockWhitelistService.isUserWhitelisted.mockResolvedValueOnce(false); - const fortuneJobDto: JobFortuneDto = createFortuneJobDto(); + const jobManifestDto: JobManifestDto = createJobManifestDto(); await expect( jobService.createJob( createUser({ paymentProviderId: null }), FortuneJobType.FORTUNE, - fortuneJobDto, + jobManifestDto, ), ).rejects.toThrow(new ValidationError(ErrorJob.NotActiveCard)); }); @@ -482,7 +440,7 @@ describe('JobService', () => { mockWeb3Service.validateChainId.mockImplementationOnce(() => { throw randomError; }); - const dto = createFortuneJobDto(); + const dto = createJobManifestDto(); await expect( jobService.createJob(createUser(), FortuneJobType.FORTUNE, dto), ).rejects.toThrow(randomError); @@ -490,15 +448,20 @@ describe('JobService', () => { }); describe('CVAT', () => { - it('should create a CVAT job', async () => { - const cvatJobDto = createCvatJobDto(); + it('should create a CVAT job successfully with a manifest JSON body', async () => { + const cvatManifest = createMockCvatManifest(); + cvatManifest.annotation.type = CvatJobType.IMAGE_BOXES; + + const jobManifestDto: JobManifestDto = createJobManifestDto({ + requestType: CvatJobType.IMAGE_BOXES, + manifest: cvatManifest, + paymentCurrency: PaymentCurrency.USDC, + escrowFundToken: EscrowFundToken.USDC, + }); const fundTokenDecimals = getTokenDecimals( - cvatJobDto.chainId!, - cvatJobDto.escrowFundToken, + jobManifestDto.chainId!, + jobManifestDto.escrowFundToken, ); - - const mockManifest = createMockCvatManifest(); - mockManifestService.createManifest.mockResolvedValueOnce(mockManifest); const mockUrl = faker.internet.url(); const mockHash = faker.string.uuid(); mockManifestService.uploadManifest.mockResolvedValueOnce({ @@ -510,60 +473,58 @@ describe('JobService', () => { mockRateService.getRate .mockResolvedValueOnce(tokenToUsdRate) .mockResolvedValueOnce(usdToTokenRate); + mockedKVStoreUtils.get.mockResolvedValueOnce('1'); - await jobService.createJob(userMock, cvatJobDto.type, cvatJobDto); + const result = await jobService.createJob( + userMock, + CvatJobType.IMAGE_BOXES, + jobManifestDto, + ); - expect(mockWeb3Service.validateChainId).toHaveBeenCalledWith( - cvatJobDto.chainId, + const paymentCurrencyFee = Number( + max( + div(mockServerConfigService.minimumFeeUsd, tokenToUsdRate), + mul(div(1, 100), jobManifestDto.paymentAmount), + ).toFixed(fundTokenDecimals), ); - expect(mockRoutingProtocolService.selectOracles).not.toHaveBeenCalled(); - expect(mockRoutingProtocolService.validateOracles).toHaveBeenCalledWith( - cvatJobDto.chainId, - cvatJobDto.type, - cvatJobDto.reputationOracle, - cvatJobDto.exchangeOracle, - cvatJobDto.recordingOracle, + + expect(result).toBe(jobEntityMock.id); + expect(mockWeb3Service.validateChainId).toHaveBeenCalledWith( + jobManifestDto.chainId, ); - expect(mockManifestService.createManifest).toHaveBeenCalledWith( - cvatJobDto, - cvatJobDto.type, - cvatJobDto.paymentAmount, - fundTokenDecimals, + expect(mockManifestService.validateManifest).toHaveBeenCalledWith( + CvatJobType.IMAGE_BOXES, + jobManifestDto.manifest, ); expect(mockManifestService.uploadManifest).toHaveBeenCalledWith( - cvatJobDto.chainId, - mockManifest, + jobManifestDto.chainId, + jobManifestDto.manifest, [ - cvatJobDto.exchangeOracle, - cvatJobDto.reputationOracle, - cvatJobDto.recordingOracle, + jobManifestDto.exchangeOracle, + jobManifestDto.reputationOracle, + jobManifestDto.recordingOracle, ], ); expect(mockPaymentService.createWithdrawalPayment).toHaveBeenCalledWith( userMock.id, expect.any(Number), - cvatJobDto.paymentCurrency, + jobManifestDto.paymentCurrency, tokenToUsdRate, ); expect(mockJobRepository.updateOne).toHaveBeenCalledWith({ - chainId: cvatJobDto.chainId, + chainId: jobManifestDto.chainId, userId: userMock.id, manifestUrl: mockUrl, manifestHash: mockHash, - requestType: cvatJobDto.type, - fee: expect.any(Number), - fundAmount: Number( - mul( - mul(cvatJobDto.paymentAmount, tokenToUsdRate), - usdToTokenRate, - ).toFixed(6), - ), - status: JobStatus.MODERATION_PASSED, + requestType: CvatJobType.IMAGE_BOXES, + fee: paymentCurrencyFee, + fundAmount: jobManifestDto.paymentAmount, + status: JobStatus.PAID, waitUntil: expect.any(Date), - token: cvatJobDto.escrowFundToken, - exchangeOracle: cvatJobDto.exchangeOracle, - recordingOracle: cvatJobDto.recordingOracle, - reputationOracle: cvatJobDto.reputationOracle, + token: jobManifestDto.escrowFundToken, + exchangeOracle: jobManifestDto.exchangeOracle, + recordingOracle: jobManifestDto.recordingOracle, + reputationOracle: jobManifestDto.reputationOracle, payments: expect.any(Array), }); }); @@ -602,15 +563,8 @@ describe('JobService', () => { expect(mockWeb3Service.validateChainId).toHaveBeenCalledWith( jobQuickLaunchDto.chainId, ); - expect(mockRoutingProtocolService.selectOracles).not.toHaveBeenCalled(); - expect(mockRoutingProtocolService.validateOracles).toHaveBeenCalledWith( - jobQuickLaunchDto.chainId, - HCaptchaJobType.HCAPTCHA, - jobQuickLaunchDto.reputationOracle, - jobQuickLaunchDto.exchangeOracle, - jobQuickLaunchDto.recordingOracle, - ); - expect(mockManifestService.createManifest).not.toHaveBeenCalled(); + expect(mockManifestService.downloadManifest).not.toHaveBeenCalled(); + expect(mockManifestService.validateManifest).not.toHaveBeenCalled(); expect(mockManifestService.uploadManifest).not.toHaveBeenCalled(); expect(mockPaymentService.createWithdrawalPayment).toHaveBeenCalledWith( userMock.id, @@ -631,7 +585,7 @@ describe('JobService', () => { usdToTokenRate, ).toFixed(6), ), - status: JobStatus.MODERATION_PASSED, + status: JobStatus.PAID, waitUntil: expect.any(Date), token: jobQuickLaunchDto.escrowFundToken, exchangeOracle: jobQuickLaunchDto.exchangeOracle, @@ -646,7 +600,7 @@ describe('JobService', () => { describe('createEscrow', () => { it('should create an escrow and update job entity', async () => { const jobEntity = createJobEntity({ - status: JobStatus.MODERATION_PASSED, + status: JobStatus.PAID, token: EscrowFundToken.HMT, escrowAddress: null, }); @@ -734,7 +688,7 @@ describe('JobService', () => { it('should throw if escrow address is not returned', async () => { const jobEntity = createJobEntity({ - status: JobStatus.MODERATION_PASSED, + status: JobStatus.PAID, token: EscrowFundToken.HMT, escrowAddress: null, }); @@ -1338,9 +1292,7 @@ describe('JobService', () => { 18, ); - const manifestMock = createMockFortuneManifest({ - fundAmount: jobEntity.fundAmount, - }); + const manifestMock = createMockFortuneManifest(); const getEscrowData = { token: faker.finance.ethereumAddress(), @@ -1396,9 +1348,7 @@ describe('JobService', () => { it('should return job details without escrow address successfully', async () => { const jobEntity = createJobEntity({ escrowAddress: null }); - const manifestMock = createMockFortuneManifest({ - fundAmount: jobEntity.fundAmount, - }); + const manifestMock = createMockFortuneManifest(); mockJobRepository.findOneByIdAndUserId.mockResolvedValueOnce(jobEntity); mockManifestService.downloadManifest.mockResolvedValueOnce(manifestMock); diff --git a/packages/apps/job-launcher/server/src/modules/job/job.service.ts b/packages/apps/job-launcher/server/src/modules/job/job.service.ts index 0cc8735a72..2c967136b3 100644 --- a/packages/apps/job-launcher/server/src/modules/job/job.service.ts +++ b/packages/apps/job-launcher/server/src/modules/job/job.service.ts @@ -6,6 +6,7 @@ import { KVStoreKeys, KVStoreUtils, NETWORKS, + Role, } from '@human-protocol/sdk'; import { Inject, Injectable } from '@nestjs/common'; import { ModuleRef } from '@nestjs/core'; @@ -61,7 +62,6 @@ import { ManifestService } from '../manifest/manifest.service'; import { PaymentService } from '../payment/payment.service'; import { QualificationService } from '../qualification/qualification.service'; import { RateService } from '../rate/rate.service'; -import { RoutingProtocolService } from '../routing-protocol/routing-protocol.service'; import { StorageService } from '../storage/storage.service'; import { UserEntity } from '../user/user.entity'; import { Web3Service } from '../web3/web3.service'; @@ -75,7 +75,6 @@ import { GetJobsDto, JobDetailsDto, JobListDto, - JobQuickLaunchDto, } from './job.dto'; import { JobEntity } from './job.entity'; import { JobRepository } from './job.repository'; @@ -93,7 +92,6 @@ export class JobService { private readonly webhookRepository: WebhookRepository, private readonly paymentService: PaymentService, private readonly serverConfigService: ServerConfigService, - private readonly routingProtocolService: RoutingProtocolService, private readonly storageService: StorageService, private readonly rateService: RateService, private readonly whitelistService: WhitelistService, @@ -123,10 +121,8 @@ export class JobService { throw new ValidationError(ErrorPayment.HMTTokenDisabled); } - let { chainId, reputationOracle, exchangeOracle, recordingOracle } = dto; - - // Select network - chainId = chainId || this.routingProtocolService.selectNetwork(); + const { chainId } = dto; + let { reputationOracle, exchangeOracle, recordingOracle } = dto; this.web3Service.validateChainId(chainId); // Check if not whitelisted user has an active payment method @@ -196,25 +192,11 @@ export class JobService { ).toFixed(fundTokenDecimals), ); - // Select oracles if (!reputationOracle || !exchangeOracle || !recordingOracle) { - const selectedOracles = await this.routingProtocolService.selectOracles( - chainId, - requestType, - ); - - exchangeOracle = exchangeOracle || selectedOracles.exchangeOracle; - recordingOracle = recordingOracle || selectedOracles.recordingOracle; - reputationOracle = reputationOracle || selectedOracles.reputationOracle; - } else { - // Validate if all oracles are provided - await this.routingProtocolService.validateOracles( - chainId, - requestType, - reputationOracle, - exchangeOracle, - recordingOracle, - ); + const defaultOracles = await this.getDefaultOracles(chainId, requestType); + reputationOracle = reputationOracle ?? defaultOracles.reputationOracle; + exchangeOracle = exchangeOracle ?? defaultOracles.exchangeOracle; + recordingOracle = recordingOracle ?? defaultOracles.recordingOracle; } if (dto.qualifications) { @@ -234,7 +216,7 @@ export class JobService { let jobEntity = new JobEntity(); - if (dto instanceof JobQuickLaunchDto) { + if ('manifestUrl' in dto) { if (!dto.manifestHash) { const { filename } = parseUrl(dto.manifestUrl); @@ -248,22 +230,19 @@ export class JobService { } jobEntity.manifestUrl = dto.manifestUrl; - } else { - const manifestOrigin = await this.manifestService.createManifest( - dto, - requestType, - fundTokenAmount, - fundTokenDecimals, - ); + } else if ('manifest' in dto) { + await this.manifestService.validateManifest(requestType, dto.manifest); const { url, hash } = await this.manifestService.uploadManifest( chainId, - manifestOrigin, + dto.manifest, [exchangeOracle, reputationOracle, recordingOracle], ); jobEntity.manifestUrl = url; jobEntity.manifestHash = hash; + } else { + throw new ValidationError(ErrorJob.InvalidRequestType); } const paymentEntity = await this.paymentService.createWithdrawalPayment( @@ -285,22 +264,56 @@ export class JobService { jobEntity.token = dto.escrowFundToken; jobEntity.waitUntil = new Date(); - if ( - user.whitelist || - ( - [FortuneJobType.FORTUNE, HCaptchaJobType.HCAPTCHA] as JobRequestType[] - ).includes(requestType) - ) { - jobEntity.status = JobStatus.MODERATION_PASSED; - } else { - jobEntity.status = JobStatus.PAID; - } + jobEntity.status = JobStatus.PAID; jobEntity = await this.jobRepository.updateOne(jobEntity); return jobEntity.id; } + private async getDefaultOracles( + chainId: ChainId, + requestType: JobRequestType, + ): Promise<{ + reputationOracle: string; + exchangeOracle: string; + recordingOracle: string; + }> { + if (requestType === HCaptchaJobType.HCAPTCHA) { + const oracleAddress = this.web3ConfigService.hCaptchaOracleAddress; + return { + reputationOracle: oracleAddress, + exchangeOracle: oracleAddress, + recordingOracle: oracleAddress, + }; + } + + if (Object.values(CvatJobType).includes(requestType as CvatJobType)) { + return { + reputationOracle: this.web3ConfigService.reputationOracleAddress, + exchangeOracle: this.web3ConfigService.cvatExchangeOracleAddress, + recordingOracle: this.web3ConfigService.cvatRecordingOracleAddress, + }; + } + + const reputationOracle = this.web3ConfigService.reputationOracleAddress; + const availableOracles = await this.web3Service.findAvailableOracles( + chainId, + requestType, + reputationOracle, + ); + + return { + reputationOracle, + exchangeOracle: + availableOracles.find((oracle) => oracle.role === Role.ExchangeOracle) + ?.address || '', + recordingOracle: + availableOracles.find((oracle) => oracle.role === Role.RecordingOracle) + ?.address || '', + }; + } + public async createEscrow(jobEntity: JobEntity): Promise { const signer = this.web3Service.getSigner(jobEntity.chainId); diff --git a/packages/apps/job-launcher/server/src/modules/manifest/fixtures.ts b/packages/apps/job-launcher/server/src/modules/manifest/fixtures.ts index ba2b12bb13..0338e929dd 100644 --- a/packages/apps/job-launcher/server/src/modules/manifest/fixtures.ts +++ b/packages/apps/job-launcher/server/src/modules/manifest/fixtures.ts @@ -1,53 +1,6 @@ import { faker } from '@faker-js/faker'; -import { ChainId } from '@human-protocol/sdk'; -import { CvatConfigService } from '../../common/config/cvat-config.service'; -import { CvatJobType, EscrowFundToken } from '../../common/enums/job'; -import { PaymentCurrency } from '../../common/enums/payment'; -import { JobCvatDto } from '../job/job.dto'; -import { - getMockedProvider, - getMockedRegion, -} from '../../../test/fixtures/storage'; +import { CvatJobType, FortuneJobType } from '../../common/enums/job'; import { CvatManifestDto, FortuneManifestDto } from './manifest.dto'; -import { FortuneJobType } from '../../common/enums/job'; - -export const mockCvatConfigService: Omit = { - jobSize: faker.number.int({ min: 1, max: 1000 }), - maxTime: faker.number.int({ min: 1, max: 1000 }), - valSize: faker.number.int({ min: 1, max: 1000 }), - skeletonsJobSizeMultiplier: faker.number.int({ min: 1, max: 1000 }), -}; - -export function createJobCvatDto( - overrides: Partial = {}, -): JobCvatDto { - return { - data: { - dataset: { - provider: getMockedProvider(), - region: getMockedRegion(), - bucketName: faker.lorem.word(), - path: faker.system.filePath(), - }, - }, - labels: [{ name: faker.lorem.word(), nodes: [faker.string.uuid()] }], - requesterDescription: faker.lorem.sentence(), - userGuide: faker.internet.url(), - minQuality: faker.number.float({ min: 0.1, max: 1 }), - groundTruth: { - provider: getMockedProvider(), - region: getMockedRegion(), - bucketName: faker.lorem.word(), - path: faker.system.filePath(), - }, - type: CvatJobType.IMAGE_BOXES, - chainId: faker.helpers.arrayElement(Object.values(ChainId)) as ChainId, - paymentCurrency: faker.helpers.arrayElement(Object.values(PaymentCurrency)), - paymentAmount: faker.number.int({ min: 1, max: 1000 }), - escrowFundToken: faker.helpers.arrayElement(Object.values(EscrowFundToken)), - ...overrides, - }; -} export function createMockFortuneManifest( overrides: Partial = {}, @@ -56,7 +9,6 @@ export function createMockFortuneManifest( submissionsRequired: faker.number.int({ min: 1, max: 100 }), requesterTitle: faker.lorem.sentence(), requesterDescription: faker.lorem.sentence(), - fundAmount: faker.number.int({ min: 1, max: 100000 }), requestType: FortuneJobType.FORTUNE, ...overrides, }; diff --git a/packages/apps/job-launcher/server/src/modules/manifest/manifest.dto.ts b/packages/apps/job-launcher/server/src/modules/manifest/manifest.dto.ts index bfdd368bd2..222d1f8b57 100644 --- a/packages/apps/job-launcher/server/src/modules/manifest/manifest.dto.ts +++ b/packages/apps/job-launcher/server/src/modules/manifest/manifest.dto.ts @@ -35,11 +35,6 @@ export class FortuneManifestDto { @IsString() public requesterDescription: string; - @ApiProperty({ name: 'fund_amount' }) - @IsNumber() - @IsPositive() - public fundAmount: number; - @ApiProperty({ enum: FortuneJobType, name: 'request_type' }) @IsEnumCaseInsensitive(FortuneJobType) public requestType: FortuneJobType; diff --git a/packages/apps/job-launcher/server/src/modules/manifest/manifest.module.ts b/packages/apps/job-launcher/server/src/modules/manifest/manifest.module.ts index ab52fd4050..48d4742160 100644 --- a/packages/apps/job-launcher/server/src/modules/manifest/manifest.module.ts +++ b/packages/apps/job-launcher/server/src/modules/manifest/manifest.module.ts @@ -3,19 +3,9 @@ import { ManifestService } from './manifest.service'; import { StorageModule } from '../storage/storage.module'; import { Web3Module } from '../web3/web3.module'; import { EncryptionModule } from '../encryption/encryption.module'; -import { RoutingProtocolModule } from '../routing-protocol/routing-protocol.module'; -import { RateModule } from '../rate/rate.module'; -import { QualificationModule } from '../qualification/qualification.module'; @Module({ - imports: [ - StorageModule, - Web3Module, - EncryptionModule, - RoutingProtocolModule, - RateModule, - QualificationModule, - ], + imports: [StorageModule, Web3Module, EncryptionModule], providers: [ManifestService], exports: [ManifestService], }) diff --git a/packages/apps/job-launcher/server/src/modules/manifest/manifest.service.spec.ts b/packages/apps/job-launcher/server/src/modules/manifest/manifest.service.spec.ts index 2f9363f42e..413e893a0a 100644 --- a/packages/apps/job-launcher/server/src/modules/manifest/manifest.service.spec.ts +++ b/packages/apps/job-launcher/server/src/modules/manifest/manifest.service.spec.ts @@ -1,31 +1,21 @@ -jest.mock('../../common/utils/storage', () => ({ - ...jest.requireActual('../../common/utils/storage'), - listObjectsInBucket: jest.fn(), -})); - import { faker } from '@faker-js/faker'; import { createMock } from '@golevelup/ts-jest'; import { Encryption } from '@human-protocol/sdk'; import { Test } from '@nestjs/testing'; -import { CvatConfigService } from '../../common/config/cvat-config.service'; import { PGPConfigService } from '../../common/config/pgp-config.service'; import { ErrorJob } from '../../common/constants/errors'; -import { CvatJobType, FortuneJobType } from '../../common/enums/job'; import { - ConflictError, - ServerError, - ValidationError, -} from '../../common/errors'; -import { generateBucketUrl } from '../../common/utils/storage'; + CvatJobType, + FortuneJobType, + HCaptchaJobType, + JobCaptchaRequestType, +} from '../../common/enums/job'; +import { ServerError, ValidationError } from '../../common/errors'; import { StorageService } from '../storage/storage.service'; import { Web3Service } from '../web3/web3.service'; -import { createJobCvatDto, mockCvatConfigService } from './fixtures'; -import { FortuneManifestDto } from './manifest.dto'; +import { createMockCvatManifest, createMockFortuneManifest } from './fixtures'; +import { ManifestDto } from './manifest.dto'; import { ManifestService } from './manifest.service'; -import { - getMockedProvider, - getMockedRegion, -} from '../../../test/fixtures/storage'; describe('ManifestService', () => { let manifestService: ManifestService; @@ -40,10 +30,6 @@ describe('ManifestService', () => { ManifestService, { provide: Web3Service, useValue: createMock() }, { provide: StorageService, useValue: mockStorageService }, - { - provide: CvatConfigService, - useValue: mockCvatConfigService, - }, { provide: PGPConfigService, useValue: { encrypt: false } }, { provide: Encryption, useValue: createMock() }, ], @@ -56,214 +42,107 @@ describe('ManifestService', () => { jest.clearAllMocks(); }); - describe('createManifest', () => { - describe('createCvatManifest', () => { - const tokenFundAmount = faker.number.int({ min: 1, max: 1000 }); - const tokenFundDecimals = faker.number.int({ min: 1, max: 18 }); - let jobBounty: string; - - beforeAll(() => { - jobBounty = faker.number.int({ min: 1, max: 1000 }).toString(); - manifestService['calculateCvatJobBounty'] = jest - .fn() - .mockResolvedValue(jobBounty); - }); - - it('should create a valid CVAT manifest for image boxes job type', async () => { - const dto = createJobCvatDto({ type: CvatJobType.IMAGE_BOXES }); - const requestType = CvatJobType.IMAGE_BOXES; - - const result = await manifestService.createManifest( - dto, - requestType, - tokenFundAmount, - tokenFundDecimals, - ); - - expect(result).toEqual({ - data: { - data_url: generateBucketUrl(dto.data.dataset, requestType).href, - }, - annotation: { - labels: dto.labels, - description: dto.requesterDescription, - user_guide: dto.userGuide, - type: requestType, - job_size: mockCvatConfigService.jobSize, - }, - validation: { - min_quality: dto.minQuality, - val_size: mockCvatConfigService.valSize, - gt_url: generateBucketUrl(dto.groundTruth, requestType).href, - }, - job_bounty: jobBounty, - }); - }); - - it('should create a valid CVAT manifest for image polygons job type', async () => { - const dto = createJobCvatDto({ type: CvatJobType.IMAGE_POLYGONS }); - const requestType = CvatJobType.IMAGE_POLYGONS; - - const result = await manifestService.createManifest( - dto, - requestType, - tokenFundAmount, - tokenFundDecimals, - ); - - expect(result).toEqual({ - data: { - data_url: generateBucketUrl(dto.data.dataset, requestType).href, - }, - annotation: { - labels: dto.labels, - description: dto.requesterDescription, - user_guide: dto.userGuide, - type: requestType, - job_size: mockCvatConfigService.jobSize, - }, - validation: { - min_quality: dto.minQuality, - val_size: mockCvatConfigService.valSize, - gt_url: generateBucketUrl(dto.groundTruth, requestType).href, - }, - job_bounty: jobBounty, - }); - }); - - it('should create a valid CVAT manifest for image boxes from points job type', async () => { - const dto = createJobCvatDto({ - data: { - dataset: { - provider: getMockedProvider(), - region: getMockedRegion(), - bucketName: faker.lorem.word(), - path: faker.system.filePath(), - }, - points: { - provider: getMockedProvider(), - region: getMockedRegion(), - bucketName: faker.lorem.word(), - path: faker.system.filePath(), - }, - }, - type: CvatJobType.IMAGE_BOXES_FROM_POINTS, - }); - const requestType = CvatJobType.IMAGE_BOXES_FROM_POINTS; - - const result = await manifestService.createManifest( - dto, - requestType, - tokenFundAmount, - tokenFundDecimals, - ); + describe('validateManifest', () => { + it('should validate a fortune manifest successfully', async () => { + await expect( + manifestService.validateManifest( + FortuneJobType.FORTUNE, + createMockFortuneManifest(), + ), + ).resolves.toBeUndefined(); + }); - expect(result).toEqual({ - data: { - data_url: generateBucketUrl(dto.data.dataset, requestType).href, - points_url: generateBucketUrl(dto.data.points!, requestType).href, - }, - annotation: { - labels: dto.labels, - description: dto.requesterDescription, - user_guide: dto.userGuide, - type: requestType, - job_size: mockCvatConfigService.jobSize, - }, - validation: { - min_quality: dto.minQuality, - val_size: mockCvatConfigService.valSize, - gt_url: generateBucketUrl(dto.groundTruth, requestType).href, - }, - job_bounty: jobBounty, - }); - }); + it('should validate a cvat manifest successfully', async () => { + const manifest = createMockCvatManifest(); + manifest.annotation.type = CvatJobType.IMAGE_BOXES; - it('should create a valid CVAT manifest for image skeletons from boxes job type', async () => { - const dto = createJobCvatDto({ - data: { - dataset: { - provider: getMockedProvider(), - region: getMockedRegion(), - bucketName: faker.lorem.word(), - path: faker.system.filePath(), - }, - boxes: { - provider: getMockedProvider(), - region: getMockedRegion(), - bucketName: faker.lorem.word(), - path: faker.system.filePath(), - }, - }, - type: CvatJobType.IMAGE_SKELETONS_FROM_BOXES, - }); - const requestType = CvatJobType.IMAGE_SKELETONS_FROM_BOXES; + await expect( + manifestService.validateManifest(CvatJobType.IMAGE_BOXES, manifest), + ).resolves.toBeUndefined(); + }); - const result = await manifestService.createManifest( - dto, - requestType, - tokenFundAmount, - tokenFundDecimals, - ); + it('should validate an hcaptcha manifest successfully', async () => { + const manifest = { + job_mode: faker.lorem.word(), + request_type: JobCaptchaRequestType.IMAGE_LABEL_BINARY, + request_config: {}, + requester_accuracy_target: faker.number.float({ + min: 0.5, + max: 1, + fractionDigits: 2, + }), + requester_max_repeats: faker.number.int({ min: 2, max: 10 }), + requester_min_repeats: faker.number.int({ min: 1, max: 1 }), + requester_question: { en: faker.lorem.sentence() }, + taskdata_uri: faker.internet.url(), + job_total_tasks: faker.number.int({ min: 1, max: 100 }), + task_bid_price: faker.number.int({ min: 1, max: 10 }), + public_results: faker.datatype.boolean(), + oracle_stake: faker.number.int({ min: 1, max: 10 }), + repo_uri: faker.internet.url(), + ro_uri: faker.internet.url(), + restricted_audience: {}, + requester_restricted_answer_set: {}, + }; - expect(result).toEqual({ - data: { - data_url: generateBucketUrl(dto.data.dataset, requestType).href, - boxes_url: generateBucketUrl(dto.data.boxes!, requestType).href, - }, - annotation: { - labels: dto.labels, - description: dto.requesterDescription, - user_guide: dto.userGuide, - type: requestType, - job_size: mockCvatConfigService.jobSize, - }, - validation: { - min_quality: dto.minQuality, - val_size: mockCvatConfigService.valSize, - gt_url: generateBucketUrl(dto.groundTruth, requestType).href, - }, - job_bounty: jobBounty, - }); - }); + await expect( + manifestService.validateManifest(HCaptchaJobType.HCAPTCHA, manifest), + ).resolves.toBeUndefined(); + }); - it('should throw an error if data does not exist for image boxes from points job type', async () => { - const requestType = CvatJobType.IMAGE_BOXES_FROM_POINTS; + it('should throw when a required fortune property is missing', async () => { + const manifest = createMockFortuneManifest(); + delete (manifest as Partial).requesterDescription; - const dto = createJobCvatDto({ type: requestType }); + await expect( + manifestService.validateManifest(FortuneJobType.FORTUNE, manifest), + ).rejects.toThrow(new ValidationError(ErrorJob.ManifestValidationFailed)); + }); - await expect( - manifestService.createManifest( - dto, - requestType, - tokenFundAmount, - tokenFundDecimals, - ), - ).rejects.toThrow(new ConflictError(ErrorJob.DataNotExist)); - }); + it('should throw when a required cvat property is missing', async () => { + const manifest = createMockCvatManifest(); + delete (manifest.validation as Partial<(typeof manifest)['validation']>) + .gt_url; - it('should throw an error if data does not exist for image skeletons from boxes job type', async () => { - const requestType = CvatJobType.IMAGE_SKELETONS_FROM_BOXES; + await expect( + manifestService.validateManifest(CvatJobType.IMAGE_BOXES, manifest), + ).rejects.toThrow(new ValidationError(ErrorJob.ManifestValidationFailed)); + }); - const dto = createJobCvatDto({ type: requestType }); + it('should throw when a required hcaptcha property is missing', async () => { + const manifest = { + job_mode: faker.lorem.word(), + request_type: JobCaptchaRequestType.IMAGE_LABEL_BINARY, + request_config: {}, + requester_accuracy_target: faker.number.float({ + min: 0.5, + max: 1, + fractionDigits: 2, + }), + requester_max_repeats: faker.number.int({ min: 2, max: 10 }), + requester_min_repeats: faker.number.int({ min: 1, max: 1 }), + requester_question: { en: faker.lorem.sentence() }, + job_total_tasks: faker.number.int({ min: 1, max: 100 }), + task_bid_price: faker.number.int({ min: 1, max: 10 }), + public_results: faker.datatype.boolean(), + oracle_stake: faker.number.int({ min: 1, max: 10 }), + repo_uri: faker.internet.url(), + ro_uri: faker.internet.url(), + restricted_audience: {}, + requester_restricted_answer_set: {}, + }; - await expect( - manifestService.createManifest( - dto, - requestType, - tokenFundAmount, - tokenFundDecimals, - ), - ).rejects.toThrow(new ConflictError(ErrorJob.DataNotExist)); - }); + await expect( + manifestService.validateManifest( + HCaptchaJobType.HCAPTCHA, + manifest as unknown as ManifestDto, + ), + ).rejects.toThrow(new ValidationError(ErrorJob.ManifestValidationFailed)); }); }); describe('uploadManifest', () => { it('should upload a manifest successfully', async () => { - const mockChainId = faker.number.int(); - const mockData = { key: faker.lorem.word() }; - const mockOracleAddresses: string[] = []; const mockManifestData = { url: faker.internet.url(), hash: faker.string.uuid(), @@ -274,33 +153,24 @@ describe('ManifestService', () => { ); const result = await manifestService.uploadManifest( - mockChainId, - mockData, - mockOracleAddresses, + faker.number.int(), + { key: faker.lorem.word() }, + [], ); - expect(result).toEqual( - expect.objectContaining({ - url: mockManifestData.url, - hash: mockManifestData.hash, - }), - ); + expect(result).toEqual(mockManifestData); }); it('should throw an error if upload fails', async () => { - const mockChainId = faker.number.int(); - const mockData = { key: faker.lorem.word() }; - const mockOracleAddresses: string[] = []; - - mockStorageService.uploadJsonLikeData.mockRejectedValue( + mockStorageService.uploadJsonLikeData.mockRejectedValueOnce( new ServerError('File not uploaded'), ); await expect( manifestService.uploadManifest( - mockChainId, - mockData, - mockOracleAddresses, + faker.number.int(), + { key: faker.lorem.word() }, + [], ), ).rejects.toThrow(ServerError); }); @@ -308,42 +178,34 @@ describe('ManifestService', () => { describe('downloadManifest', () => { it('should download and validate a manifest successfully', async () => { - const mockManifestUrl = faker.internet.url(); - const mockRequestType = FortuneJobType.FORTUNE; - const mockManifest: FortuneManifestDto = { - submissionsRequired: faker.number.int({ min: 1, max: 100 }), - requesterTitle: faker.lorem.words(3), - requesterDescription: faker.lorem.sentence(), - fundAmount: faker.number.float({ min: 1, max: 1000 }), - requestType: FortuneJobType.FORTUNE, - qualifications: [faker.lorem.word(), faker.lorem.word()], - }; + const mockManifest: ManifestDto = createMockFortuneManifest(); + mockStorageService.downloadJsonLikeData.mockResolvedValueOnce( mockManifest, ); + const result = await manifestService.downloadManifest( - mockManifestUrl, - mockRequestType, + faker.internet.url(), + FortuneJobType.FORTUNE, ); + expect(result).toEqual(mockManifest); }); - it('should throw an error if validation fails', async () => { - const mockManifestUrl = faker.internet.url(); - const mockRequestType = CvatJobType.IMAGE_BOXES; - const mockManifest: FortuneManifestDto = { - submissionsRequired: faker.number.int({ min: 1, max: 100 }), - requesterTitle: faker.lorem.words(3), - requesterDescription: faker.lorem.sentence(), - fundAmount: faker.number.float({ min: 1, max: 1000 }), - requestType: FortuneJobType.FORTUNE, - qualifications: [faker.lorem.word(), faker.lorem.word()], - }; + it('should throw if downloaded manifest is invalid', async () => { + const mockManifest = createMockFortuneManifest(); + delete (mockManifest as Partial) + .requesterDescription; + mockStorageService.downloadJsonLikeData.mockResolvedValueOnce( mockManifest, ); + await expect( - manifestService.downloadManifest(mockManifestUrl, mockRequestType), + manifestService.downloadManifest( + faker.internet.url(), + FortuneJobType.FORTUNE, + ), ).rejects.toThrow(new ValidationError(ErrorJob.ManifestValidationFailed)); }); }); diff --git a/packages/apps/job-launcher/server/src/modules/manifest/manifest.service.ts b/packages/apps/job-launcher/server/src/modules/manifest/manifest.service.ts index 99b9911652..4283fcd3ca 100644 --- a/packages/apps/job-launcher/server/src/modules/manifest/manifest.service.ts +++ b/packages/apps/job-launcher/server/src/modules/manifest/manifest.service.ts @@ -4,28 +4,15 @@ import { Injectable, } from '@nestjs/common'; import { validate } from 'class-validator'; -import { ethers } from 'ethers'; -import { CvatConfigService } from '../../common/config/cvat-config.service'; +import { plainToInstance } from 'class-transformer'; import { PGPConfigService } from '../../common/config/pgp-config.service'; import { ErrorJob } from '../../common/constants/errors'; import { - CvatJobType, FortuneJobType, HCaptchaJobType, JobRequestType, } from '../../common/enums/job'; -import { ConflictError, ValidationError } from '../../common/errors'; -import { - generateBucketUrl, - listObjectsInBucket, -} from '../../common/utils/storage'; -import { CreateJob, JobCvatDto } from '../job/job.dto'; -import { - CvatAnnotationData, - CvatCalculateJobBounty, - CvatImageData, - GenerateUrls, -} from '../job/job.interface'; +import { ValidationError } from '../../common/errors'; import { StorageService } from '../storage/storage.service'; import { Web3Service } from '../web3/web3.service'; import { @@ -39,244 +26,11 @@ import { export class ManifestService { constructor( private readonly web3Service: Web3Service, - private readonly cvatConfigService: CvatConfigService, private readonly pgpConfigService: PGPConfigService, private readonly storageService: StorageService, private readonly encryption: Encryption, ) {} - async createManifest( - dto: CreateJob, - requestType: JobRequestType, - fundAmount: number, - decimals: number, - ): Promise { - switch (requestType) { - case FortuneJobType.FORTUNE: - return { - ...dto, - requestType, - fundAmount, - }; - - case CvatJobType.IMAGE_POLYGONS: - case CvatJobType.IMAGE_BOXES: - case CvatJobType.IMAGE_POINTS: - case CvatJobType.IMAGE_BOXES_FROM_POINTS: - case CvatJobType.IMAGE_SKELETONS_FROM_BOXES: - return this.createCvatManifest( - dto as JobCvatDto, - requestType, - fundAmount, - decimals, - ); - - default: - throw new ValidationError(ErrorJob.InvalidRequestType); - } - } - - private async getCvatElementsCount( - urls: GenerateUrls, - requestType: CvatJobType, - ): Promise { - let gt: any, gtEntries: number; - switch (requestType) { - case CvatJobType.IMAGE_POLYGONS: - case CvatJobType.IMAGE_BOXES: - case CvatJobType.IMAGE_POINTS: { - const data = await listObjectsInBucket(urls.dataUrl); - if (!data || data.length === 0 || !data[0]) - throw new ValidationError(ErrorJob.DatasetValidationFailed); - gt = (await this.storageService.downloadJsonLikeData( - `${urls.gtUrl.protocol}//${urls.gtUrl.host}${urls.gtUrl.pathname}`, - )) as any; - if (!gt || !gt.images || gt.images.length === 0) - throw new ValidationError(ErrorJob.GroundThuthValidationFailed); - - await this.checkImageConsistency(gt.images, data); - - return data.length - gt.images.length; - } - - case CvatJobType.IMAGE_BOXES_FROM_POINTS: { - const points = (await this.storageService.downloadJsonLikeData( - urls.pointsUrl!.href, - )) as any; - gt = (await this.storageService.downloadJsonLikeData( - urls.gtUrl.href, - )) as any; - - if (!gt || !gt.images || gt.images.length === 0) { - throw new ValidationError(ErrorJob.GroundThuthValidationFailed); - } - - gtEntries = 0; - gt.images.forEach((gtImage: CvatImageData) => { - const { id } = points.images.find( - (dataImage: CvatImageData) => - dataImage.file_name === gtImage.file_name, - ); - - if (id) { - const matchingAnnotations = points.annotations.filter( - (dataAnnotation: CvatAnnotationData) => - dataAnnotation.image_id === id, - ); - gtEntries += matchingAnnotations.length; - } - }); - - return points.annotations.length - gtEntries; - } - - case CvatJobType.IMAGE_SKELETONS_FROM_BOXES: { - const boxes = (await this.storageService.downloadJsonLikeData( - urls.boxesUrl!.href, - )) as any; - gt = (await this.storageService.downloadJsonLikeData( - urls.gtUrl.href, - )) as any; - - if (!gt || !gt.images || gt.images.length === 0) { - throw new ValidationError(ErrorJob.GroundThuthValidationFailed); - } - - gtEntries = 0; - gt.images.forEach((gtImage: CvatImageData) => { - const { id } = boxes.images.find( - (dataImage: CvatImageData) => - dataImage.file_name === gtImage.file_name, - ); - - if (id) { - const matchingAnnotations = boxes.annotations.filter( - (dataAnnotation: CvatAnnotationData) => - dataAnnotation.image_id === id, - ); - gtEntries += matchingAnnotations.length; - } - }); - - return boxes.annotations.length - gtEntries; - } - - default: - throw new ValidationError(ErrorJob.InvalidRequestType); - } - } - - private async checkImageConsistency( - gtImages: any[], - dataFiles: string[], - ): Promise { - const gtFileNames = gtImages.map((image: any) => image.file_name); - const baseFileNames = dataFiles.map((fileName) => - fileName.split('/').pop(), - ); - const missingFileNames = gtFileNames.filter( - (fileName: any) => !baseFileNames.includes(fileName), - ); - - if (missingFileNames.length !== 0) { - throw new ConflictError(ErrorJob.ImageConsistency); - } - } - - private async calculateCvatJobBounty( - params: CvatCalculateJobBounty, - ): Promise { - const { requestType, fundAmount, urls, nodesTotal } = params; - - const elementsCount = await this.getCvatElementsCount(urls, requestType); - - let jobSize = Number(this.cvatConfigService.jobSize); - - if (requestType === CvatJobType.IMAGE_SKELETONS_FROM_BOXES) { - const jobSizeMultiplier = Number( - this.cvatConfigService.skeletonsJobSizeMultiplier, - ); - jobSize *= jobSizeMultiplier; - } - - let totalJobs: number; - - // For each skeleton node CVAT creates a separate project thus increasing the number of jobs - if (requestType === CvatJobType.IMAGE_SKELETONS_FROM_BOXES && nodesTotal) { - totalJobs = Math.ceil(elementsCount / jobSize) * nodesTotal; - } else { - totalJobs = Math.ceil(elementsCount / jobSize); - } - - const jobBounty = - ethers.parseUnits(fundAmount.toString(), params.decimals) / - BigInt(totalJobs); - - return ethers.formatUnits(jobBounty, params.decimals); - } - - private async createCvatManifest( - dto: JobCvatDto, - requestType: CvatJobType, - tokenFundAmount: number, - decimals: number, - ): Promise { - if ( - (requestType === CvatJobType.IMAGE_SKELETONS_FROM_BOXES && - !dto.data.boxes) || - (requestType === CvatJobType.IMAGE_BOXES_FROM_POINTS && !dto.data.points) - ) { - throw new ConflictError(ErrorJob.DataNotExist); - } - - const urls = { - dataUrl: generateBucketUrl(dto.data.dataset, requestType), - gtUrl: generateBucketUrl(dto.groundTruth, requestType), - boxesUrl: dto.data.boxes - ? generateBucketUrl(dto.data.boxes, requestType) - : undefined, - pointsUrl: dto.data.points - ? generateBucketUrl(dto.data.points, requestType) - : undefined, - }; - - const jobBounty = await this.calculateCvatJobBounty({ - requestType, - fundAmount: tokenFundAmount, - decimals, - urls, - nodesTotal: dto.labels[0]?.nodes?.length, - }); - - return { - data: { - data_url: urls.dataUrl.href, - ...(urls.pointsUrl && { - points_url: urls.pointsUrl?.href, - }), - ...(urls.boxesUrl && { - boxes_url: urls.boxesUrl?.href, - }), - }, - annotation: { - labels: dto.labels, - description: dto.requesterDescription, - user_guide: dto.userGuide, - type: requestType as CvatJobType, - job_size: this.cvatConfigService.jobSize, - ...(dto.qualifications && { - qualifications: dto.qualifications, - }), - }, - validation: { - min_quality: dto.minQuality, - val_size: this.cvatConfigService.valSize, - gt_url: urls.gtUrl.href, - }, - job_bounty: jobBounty, - }; - } - async uploadManifest( chainId: ChainId, data: any, @@ -294,34 +48,31 @@ export class ManifestService { const publicKey = await KVStoreUtils.getPublicKey(chainId, address); if (publicKey) publicKeys.push(publicKey); } - const encryptedManifest = await this.encryption.signAndEncrypt( + manifestFile = await this.encryption.signAndEncrypt( JSON.stringify(data), publicKeys, ); - manifestFile = encryptedManifest; } return this.storageService.uploadJsonLikeData(manifestFile); } - private async validateManifest( + public async validateManifest( requestType: JobRequestType, - manifest: FortuneManifestDto | CvatManifestDto | HCaptchaManifestDto, + manifest: ManifestDto, ): Promise { let dtoCheck; if (requestType === FortuneJobType.FORTUNE) { - dtoCheck = new FortuneManifestDto(); + dtoCheck = plainToInstance(FortuneManifestDto, manifest); } else if (requestType === HCaptchaJobType.HCAPTCHA) { - return; - dtoCheck = new HCaptchaManifestDto(); + dtoCheck = plainToInstance(HCaptchaManifestDto, manifest); } else { - dtoCheck = new CvatManifestDto(); + dtoCheck = plainToInstance(CvatManifestDto, manifest); } - Object.assign(dtoCheck, manifest); - const validationErrors: ClassValidationError[] = await validate(dtoCheck); + if (validationErrors.length > 0) { throw new ValidationError(ErrorJob.ManifestValidationFailed); } @@ -330,7 +81,7 @@ export class ManifestService { async downloadManifest( manifestUrl: string, requestType: JobRequestType, - ): Promise { + ): Promise { const manifest = (await this.storageService.downloadJsonLikeData( manifestUrl, )) as ManifestDto; diff --git a/packages/apps/job-launcher/server/src/modules/routing-protocol/routing-protocol.interface.ts b/packages/apps/job-launcher/server/src/modules/routing-protocol/routing-protocol.interface.ts deleted file mode 100644 index 5ccc44a7ee..0000000000 --- a/packages/apps/job-launcher/server/src/modules/routing-protocol/routing-protocol.interface.ts +++ /dev/null @@ -1,29 +0,0 @@ -export interface OracleOrder { - [chainId: number]: { - [reputationOracle: string]: { - [oracleType: string]: { - [jobType: string]: string[]; - }; - }; - }; -} - -export interface OracleIndex { - [chainId: number]: { - [reputationOracle: string]: { - [oracleType: string]: { - [jobType: string]: number; - }; - }; - }; -} - -export interface OracleHash { - [chainId: number]: { - [reputationOracle: string]: { - [oracleType: string]: { - [jobType: string]: string; - }; - }; - }; -} diff --git a/packages/apps/job-launcher/server/src/modules/routing-protocol/routing-protocol.module.ts b/packages/apps/job-launcher/server/src/modules/routing-protocol/routing-protocol.module.ts deleted file mode 100644 index 1ae2b644ba..0000000000 --- a/packages/apps/job-launcher/server/src/modules/routing-protocol/routing-protocol.module.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { Module } from '@nestjs/common'; -import { RoutingProtocolService } from './routing-protocol.service'; -import { Web3Module } from '../web3/web3.module'; -import { ConfigModule } from '@nestjs/config'; - -@Module({ - imports: [Web3Module, ConfigModule], - providers: [RoutingProtocolService], - exports: [RoutingProtocolService], -}) -export class RoutingProtocolModule {} diff --git a/packages/apps/job-launcher/server/src/modules/routing-protocol/routing-protocol.service.spec.ts b/packages/apps/job-launcher/server/src/modules/routing-protocol/routing-protocol.service.spec.ts deleted file mode 100644 index 01d3e64cbd..0000000000 --- a/packages/apps/job-launcher/server/src/modules/routing-protocol/routing-protocol.service.spec.ts +++ /dev/null @@ -1,498 +0,0 @@ -jest.mock('../../common/utils', () => ({ - ...jest.requireActual('../../common/utils'), - hashString: jest.fn(), -})); - -jest.mock('@human-protocol/sdk', () => ({ - ...jest.requireActual('@human-protocol/sdk'), - EscrowClient: { - build: jest.fn().mockImplementation(() => ({})), - }, -})); - -import { ChainId, Role } from '@human-protocol/sdk'; -import { ConfigService } from '@nestjs/config'; -import { Test } from '@nestjs/testing'; -import { MOCK_REPUTATION_ORACLE_1, mockConfig } from '../../../test/constants'; -import { NetworkConfigService } from '../../common/config/network-config.service'; -import { Web3ConfigService } from '../../common/config/web3-config.service'; -import { ErrorRoutingProtocol } from '../../common/constants/errors'; -import { FortuneJobType } from '../../common/enums/job'; -import { ServerError } from '../../common/errors'; -import { hashString } from '../../common/utils'; -import { Web3Service } from '../web3/web3.service'; -import { RoutingProtocolService } from './routing-protocol.service'; - -describe('RoutingProtocolService', () => { - let web3Service: Web3Service; - let routingProtocolService: RoutingProtocolService; - - beforeEach(async () => { - const moduleRef = await Test.createTestingModule({ - providers: [ - { - provide: ConfigService, - useValue: { - get: jest.fn((key: string) => mockConfig[key]), - getOrThrow: jest.fn((key: string) => { - if (!mockConfig[key]) { - throw new Error(`Configuration key "${key}" does not exist`); - } - return mockConfig[key]; - }), - }, - }, - Web3ConfigService, - NetworkConfigService, - { - provide: Web3Service, - useValue: { - findAvailableOracles: jest.fn(), - }, - }, - RoutingProtocolService, - ], - }).compile(); - - web3Service = moduleRef.get(Web3Service); - routingProtocolService = moduleRef.get(RoutingProtocolService); - }); - - describe('constructor', () => { - it('should initialize chains and reputation oracles from config', () => { - const chains = routingProtocolService['chains']; - const reputationOracles = routingProtocolService['reputationOracles']; - - expect(chains).toHaveLength(routingProtocolService['chains'].length); - expect(reputationOracles).toEqual( - mockConfig['REPUTATION_ORACLES'] - .split(',') - .map((address: string) => address.trim()), - ); - }); - - it('should shuffle chains and reputation oracles', () => { - const chainPriorityOrder = routingProtocolService['chainPriorityOrder']; - const reputationOraclePriorityOrder = - routingProtocolService['reputationOraclePriorityOrder']; - - expect(chainPriorityOrder).toHaveLength( - routingProtocolService['chains'].length, - ); - expect(reputationOraclePriorityOrder).toHaveLength( - routingProtocolService['reputationOracles'].length, - ); - }); - }); - - describe('selectNetwork', () => { - it('should select a network in a random order', () => { - const chainIds = []; - for (let i = 0; i < routingProtocolService['chains'].length; i++) { - chainIds.push(routingProtocolService.selectNetwork()); - } - expect(chainIds).toHaveLength(routingProtocolService['chains'].length); - }); - - it('should cycle back to the first network after cycling through all', () => { - const firstCycle = routingProtocolService.selectNetwork(); - const chainLength = routingProtocolService['chains'].length; - - for (let i = 1; i < chainLength; i++) { - routingProtocolService.selectNetwork(); - } - - const secondCycle = routingProtocolService.selectNetwork(); - expect(firstCycle).toBe(secondCycle); - }); - }); - - describe('selectReputationOracle', () => { - it('should select a reputation oracle in shuffled order', () => { - const selectedOracles = []; - const oracleLength = routingProtocolService['reputationOracles'].length; - - for (let i = 0; i < oracleLength; i++) { - selectedOracles.push(routingProtocolService.selectReputationOracle()); - } - - expect(selectedOracles).toHaveLength(oracleLength); - expect(new Set(selectedOracles).size).toBe(oracleLength); // Ensure all oracles are unique - }); - - it('should cycle back to the first reputation oracle after cycling through all', () => { - const firstCycle = routingProtocolService.selectReputationOracle(); - const oracleLength = routingProtocolService['reputationOracles'].length; - - for (let i = 1; i < oracleLength; i++) { - routingProtocolService.selectReputationOracle(); - } - - const secondCycle = routingProtocolService.selectReputationOracle(); - expect(firstCycle).toBe(secondCycle); - }); - }); - - describe('selectOracleFromAvailable', () => { - it('should return empty string if no oracles of the specified type are available', () => { - const result = routingProtocolService.selectOracleFromAvailable( - [], - Role.ExchangeOracle, - ChainId.POLYGON_AMOY, - MOCK_REPUTATION_ORACLE_1, - 'jobType', - ); - expect(result).toBe(''); - }); - - it('should select the first available oracle of specified type', async () => { - const availableOracles = [ - { role: Role.ExchangeOracle, address: '0xExchangeOracle1', url: null }, - { role: Role.ExchangeOracle, address: '0xExchangeOracle2', url: null }, - ]; - - const result = routingProtocolService.selectOracleFromAvailable( - availableOracles, - Role.ExchangeOracle, - ChainId.POLYGON_AMOY, - MOCK_REPUTATION_ORACLE_1, - 'jobType', - ); - expect(result).toEqual(expect.stringContaining('0xExchangeOracle')); // 0xExchangeOraclex; - }); - - it('should shuffle oracles and return the first oracle from the shuffled list', () => { - const availableOracles = [ - { role: Role.ExchangeOracle, address: '0xExchangeOracle1', url: null }, - { role: Role.ExchangeOracle, address: '0xExchangeOracle2', url: null }, - ]; - - jest - .spyOn(routingProtocolService, 'shuffleArray') - .mockReturnValue(['0xExchangeOracle2', '0xExchangeOracle1']); - - const result = routingProtocolService.selectOracleFromAvailable( - availableOracles, - Role.ExchangeOracle, - ChainId.POLYGON_AMOY, - MOCK_REPUTATION_ORACLE_1, - 'jobType', - ); - expect(result).toBe('0xExchangeOracle2'); - }); - - it('should update oracle order and select from the newly shuffled list if jobType changes', () => { - const availableOracles = [ - { role: Role.ExchangeOracle, address: '0xExchangeOracle1', url: null }, - { role: Role.ExchangeOracle, address: '0xExchangeOracle2', url: null }, - ]; - - routingProtocolService.oracleOrder = { - [ChainId.POLYGON_AMOY]: { - [MOCK_REPUTATION_ORACLE_1]: { - [Role.ExchangeOracle]: { - oldJobType: ['0xExchangeOracle1'], - }, - }, - }, - }; - - const result = routingProtocolService.selectOracleFromAvailable( - availableOracles, - Role.ExchangeOracle, - ChainId.POLYGON_AMOY, - MOCK_REPUTATION_ORACLE_1, - 'newJobType', - ); - - // The jobType changed, so the order should have been shuffled - expect( - routingProtocolService.oracleOrder[ChainId.POLYGON_AMOY][ - MOCK_REPUTATION_ORACLE_1 - ][Role.ExchangeOracle]['newJobType'], - ).toEqual( - expect.arrayContaining(['0xExchangeOracle1', '0xExchangeOracle2']), - ); - expect(result).toBeDefined(); - }); - - it('should not shuffle if the oracle hash has not changed for the same jobType', () => { - const availableOracles = [ - { role: Role.ExchangeOracle, address: '0xExchangeOracle1', url: null }, - ]; - - const latestOraclesHash = 'hash123'; - (hashString as jest.Mock).mockReturnValue(latestOraclesHash); - - routingProtocolService.oracleOrder = { - [ChainId.POLYGON_AMOY]: { - [MOCK_REPUTATION_ORACLE_1]: { - [Role.ExchangeOracle]: { jobType: ['0xExchangeOracle1'] }, - }, - }, - }; - - routingProtocolService.oracleHashes = { - [ChainId.POLYGON_AMOY]: { - [MOCK_REPUTATION_ORACLE_1]: { - [Role.ExchangeOracle]: { jobType: latestOraclesHash }, - }, - }, - }; - - jest.spyOn(routingProtocolService, 'shuffleArray'); - - const result = routingProtocolService.selectOracleFromAvailable( - availableOracles, - Role.ExchangeOracle, - ChainId.POLYGON_AMOY, - MOCK_REPUTATION_ORACLE_1, - 'jobType', - ); - - // Shuffle should not be called if the oracle hash is unchanged - expect(routingProtocolService.shuffleArray).not.toHaveBeenCalled(); - expect(result).toBe('0xExchangeOracle1'); - }); - - it('should update the oracle order and hash if the list of available oracles changes', () => { - const availableOracles = [ - { role: Role.ExchangeOracle, address: '0xExchangeOracle1', url: null }, - { role: Role.ExchangeOracle, address: '0xExchangeOracle2', url: null }, - ]; - - const previousHash = 'oldHash'; - routingProtocolService.oracleHashes = { - [ChainId.POLYGON_AMOY]: { - [MOCK_REPUTATION_ORACLE_1]: { - [Role.ExchangeOracle]: { jobType: previousHash }, - }, - }, - }; - - jest - .spyOn(routingProtocolService, 'shuffleArray') - .mockReturnValue(availableOracles.map((oracle) => oracle.address)); - const latestOraclesHash = 'newHash'; - (hashString as jest.Mock).mockReturnValue(latestOraclesHash); - - const result = routingProtocolService.selectOracleFromAvailable( - availableOracles, - Role.ExchangeOracle, - ChainId.POLYGON_AMOY, - MOCK_REPUTATION_ORACLE_1, - 'jobType', - ); - - expect( - routingProtocolService.oracleHashes[ChainId.POLYGON_AMOY][ - MOCK_REPUTATION_ORACLE_1 - ][Role.ExchangeOracle]['jobType'], - ).toBe(latestOraclesHash); - expect(result).toBe('0xExchangeOracle1'); - }); - - it('should select the oracle from available ones and rotate index', async () => { - const availableOracles = [ - { role: Role.ExchangeOracle, address: '0xExchangeOracle1', url: null }, - { role: Role.ExchangeOracle, address: '0xExchangeOracle2', url: null }, - { - role: Role.RecordingOracle, - address: '0xRecordingOracle1', - url: null, - }, - { - role: Role.RecordingOracle, - address: '0xRecordingOracle2', - url: null, - }, - ]; - - const result1 = routingProtocolService.selectOracleFromAvailable( - availableOracles, - Role.ExchangeOracle, - ChainId.POLYGON_AMOY, - MOCK_REPUTATION_ORACLE_1, - 'jobType', - ); - - const result2 = routingProtocolService.selectOracleFromAvailable( - availableOracles, - Role.RecordingOracle, - ChainId.POLYGON_AMOY, - MOCK_REPUTATION_ORACLE_1, - 'jobType', - ); - - const result3 = routingProtocolService.selectOracleFromAvailable( - availableOracles, - Role.ExchangeOracle, - ChainId.POLYGON_AMOY, - MOCK_REPUTATION_ORACLE_1, - 'jobType', - ); - - const result4 = routingProtocolService.selectOracleFromAvailable( - availableOracles, - Role.RecordingOracle, - ChainId.POLYGON_AMOY, - MOCK_REPUTATION_ORACLE_1, - 'jobType', - ); - - expect(result1).toEqual(expect.stringContaining('0xExchangeOracle')); // 0xExchangeOraclex; - expect(result2).toEqual(expect.stringContaining('0xRecordingOracle')); // 0xRecordingOraclex - expect(result3).toEqual(expect.stringContaining('0xExchangeOracle')); // 0xExchangeOraclex; - expect(result4).toEqual(expect.stringContaining('0xRecordingOracle')); // 0xRecordingOraclex - }); - }); - - describe('selectOracles', () => { - it('should select reputation oracle and find available oracles', async () => { - const mockAvailableOracles = [ - { role: Role.ExchangeOracle, address: '0xExchangeOracle1', url: null }, - { - role: Role.RecordingOracle, - address: '0xRecordingOracle1', - url: null, - }, - ]; - - web3Service.findAvailableOracles = jest - .fn() - .mockResolvedValue(mockAvailableOracles); - - const result = await routingProtocolService.selectOracles( - ChainId.POLYGON_AMOY, - FortuneJobType.FORTUNE, - ); - expect(result.reputationOracle).toBeDefined(); - expect(result.exchangeOracle).toBe('0xExchangeOracle1'); - expect(result.recordingOracle).toBe('0xRecordingOracle1'); - }); - - it('should return null for exchange and recording oracles if none available', async () => { - web3Service.findAvailableOracles = jest.fn().mockResolvedValue([]); - - const result = await routingProtocolService.selectOracles( - ChainId.POLYGON_AMOY, - FortuneJobType.FORTUNE, - ); - expect(result.exchangeOracle).toBe(''); - expect(result.recordingOracle).toBe(''); - }); - }); - - describe('validateOracles', () => { - it('should validate oracles successfully', async () => { - const chainId = ChainId.POLYGON_AMOY; - const reputationOracle = '0xReputationOracle'; - const exchangeOracle = '0xExchangeOracle'; - const recordingOracle = '0xRecordingOracle'; - - jest - .spyOn( - routingProtocolService.web3ConfigService, - 'reputationOracles', - 'get', - ) - .mockReturnValue(`${reputationOracle},otherOracle`); - jest.spyOn(web3Service, 'findAvailableOracles').mockResolvedValue([ - { address: exchangeOracle, role: Role.ExchangeOracle, url: null }, - { address: recordingOracle, role: Role.RecordingOracle, url: null }, - ]); - - await expect( - routingProtocolService.validateOracles( - chainId, - FortuneJobType.FORTUNE, - reputationOracle, - exchangeOracle, - recordingOracle, - ), - ).resolves.not.toThrow(); - }); - - it('should throw error if reputation oracle not found', async () => { - const chainId = ChainId.POLYGON_AMOY; - const invalidReputationOracle = 'invalidOracle'; - - jest - .spyOn( - routingProtocolService.web3ConfigService, - 'reputationOracles', - 'get', - ) - .mockReturnValue('validReputationOracle,otherOracle'); - - await expect( - routingProtocolService.validateOracles( - chainId, - FortuneJobType.FORTUNE, - invalidReputationOracle, - ), - ).rejects.toThrow( - new ServerError(ErrorRoutingProtocol.ReputationOracleNotFound), - ); - }); - - it('should throw error if exchange oracle not found', async () => { - const chainId = ChainId.POLYGON_AMOY; - const reputationOracle = '0xReputationOracle'; - - jest - .spyOn( - routingProtocolService.web3ConfigService, - 'reputationOracles', - 'get', - ) - .mockReturnValue(reputationOracle); - jest - .spyOn(web3Service, 'findAvailableOracles') - .mockResolvedValue([ - { address: 'anotherOracle', role: Role.ExchangeOracle, url: null }, - ]); - - await expect( - routingProtocolService.validateOracles( - chainId, - FortuneJobType.FORTUNE, - reputationOracle, - 'invalidExchangeOracle', - ), - ).rejects.toThrow( - new ServerError(ErrorRoutingProtocol.ExchangeOracleNotFound), - ); - }); - - it('should throw error if recording oracle not found', async () => { - const chainId = ChainId.POLYGON_AMOY; - const reputationOracle = '0xReputationOracle'; - - jest - .spyOn( - routingProtocolService.web3ConfigService, - 'reputationOracles', - 'get', - ) - .mockReturnValue(reputationOracle); - jest - .spyOn(web3Service, 'findAvailableOracles') - .mockResolvedValue([ - { address: 'anotherOracle', role: Role.RecordingOracle, url: null }, - ]); - - await expect( - routingProtocolService.validateOracles( - chainId, - FortuneJobType.FORTUNE, - reputationOracle, - undefined, - 'invalidRecordingOracle', - ), - ).rejects.toThrow( - new ServerError(ErrorRoutingProtocol.RecordingOracleNotFound), - ); - }); - }); -}); diff --git a/packages/apps/job-launcher/server/src/modules/routing-protocol/routing-protocol.service.ts b/packages/apps/job-launcher/server/src/modules/routing-protocol/routing-protocol.service.ts deleted file mode 100644 index b08e1f6819..0000000000 --- a/packages/apps/job-launcher/server/src/modules/routing-protocol/routing-protocol.service.ts +++ /dev/null @@ -1,259 +0,0 @@ -import { ChainId, Role } from '@human-protocol/sdk'; -import { Injectable } from '@nestjs/common'; -import { NetworkConfigService } from '../../common/config/network-config.service'; -import { Web3ConfigService } from '../../common/config/web3-config.service'; -import { ErrorRoutingProtocol } from '../../common/constants/errors'; -import { - CvatJobType, - HCaptchaJobType, - JobRequestType, -} from '../../common/enums/job'; -import { ServerError } from '../../common/errors'; -import { hashString } from '../../common/utils'; -import { Web3Service } from '../web3/web3.service'; -import { - OracleHash, - OracleIndex, - OracleOrder, -} from './routing-protocol.interface'; -import { OracleDataDto } from '../web3/web3.dto'; - -type OracleValue = { - [reputationOracle: string]: { - [oracleType: string]: { [jobType: string]: T }; - }; -}; - -@Injectable() -export class RoutingProtocolService { - private readonly chains: ChainId[]; - private readonly reputationOracles: string[]; - private readonly chainPriorityOrder: number[]; - private readonly reputationOraclePriorityOrder: number[]; - private chainCurrentIndex = 0; - private reputationOracleIndex = 0; - private oracleIndexes: OracleIndex = {}; - public oracleHashes: OracleHash = {}; - public oracleOrder: OracleOrder = {}; - - constructor( - public readonly web3Service: Web3Service, - public readonly web3ConfigService: Web3ConfigService, - private readonly networkConfigService: NetworkConfigService, - ) { - this.chains = this.networkConfigService.networks.map( - (network) => network.chainId, - ); - this.reputationOracles = this.web3ConfigService.reputationOracles - .split(',') - .map((address) => address.trim()); - - this.chainPriorityOrder = this.shuffleArray(this.chains); - this.reputationOraclePriorityOrder = this.shuffleArray( - this.reputationOracles.map((_, i) => i), - ); - - this.oracleOrder = this.createOracleStructure( - this.chains, - this.reputationOracles, - ); - this.oracleIndexes = this.createOracleStructure( - this.chains, - this.reputationOracles, - ); - this.oracleHashes = this.createOracleStructure( - this.chains, - this.reputationOracles, - ); - } - - private createOracleStructure( - chains: ChainId[], - reputationOracles: string[], - ): { [chainId: string]: OracleValue } { - return chains.reduce( - (acc: { [chainId: string]: OracleValue }, chainId) => { - acc[chainId] = reputationOracles.reduce( - (oracleAcc: OracleValue, reputationOracle) => { - oracleAcc[reputationOracle] = { - [Role.ExchangeOracle]: {}, - [Role.RecordingOracle]: {}, - }; - return oracleAcc; - }, - {} as OracleValue, - ); - return acc; - }, - {} as { [chainId: string]: OracleValue }, - ); - } - - public shuffleArray(array: T[]): T[] { - return array.sort(() => Math.random() - 0.5); - } - - public selectNetwork(): ChainId { - const chainId = - this.chains[this.chainPriorityOrder[this.chainCurrentIndex]]; - this.chainCurrentIndex = (this.chainCurrentIndex + 1) % this.chains.length; - return chainId; - } - - public selectReputationOracle(): string { - const reputationOracle = - this.reputationOracles[ - this.reputationOraclePriorityOrder[this.reputationOracleIndex] - ]; - - this.reputationOracleIndex = - (this.reputationOracleIndex + 1) % this.reputationOracles.length; - return reputationOracle; - } - - public selectOracleFromAvailable( - availableOracles: OracleDataDto[], - oracleType: string, - chainId: ChainId, - reputationOracle: string, - jobType: string, - ): string { - const oraclesOfType = availableOracles - .filter((oracle) => oracle.role === oracleType) - .map((oracle) => oracle.address); - - if (!oraclesOfType.length) return ''; - - const latestOraclesHash = hashString( - JSON.stringify(availableOracles, (_, value) => - typeof value === 'bigint' ? value.toString() : value, - ), - ); - - if ( - !this.oracleOrder[chainId][reputationOracle][oracleType][jobType] || - this.oracleHashes[chainId][reputationOracle][oracleType][jobType] !== - latestOraclesHash - ) { - this.oracleHashes[chainId][reputationOracle][oracleType][jobType] = - latestOraclesHash; - - const shuffledOracles = this.shuffleArray(oraclesOfType); - this.oracleOrder[chainId][reputationOracle][oracleType][jobType] = - shuffledOracles; - this.oracleIndexes[chainId][reputationOracle][oracleType][jobType] = 0; - } - - const orderedOracles = - this.oracleOrder[chainId][reputationOracle][oracleType][jobType]; - const currentIndex = - this.oracleIndexes[chainId][reputationOracle][oracleType][jobType] || 0; - const selectedOracle = orderedOracles[currentIndex]; - - this.oracleIndexes[chainId][reputationOracle][oracleType][jobType] = - (currentIndex + 1) % orderedOracles.length; - return selectedOracle; - } - - public async selectOracles( - chainId: ChainId, - jobType: JobRequestType, - ): Promise<{ - reputationOracle: string; - exchangeOracle: string; - recordingOracle: string; - }> { - if (jobType === HCaptchaJobType.HCAPTCHA) { - return { - reputationOracle: this.web3ConfigService.hCaptchaOracleAddress, - exchangeOracle: this.web3ConfigService.hCaptchaOracleAddress, - recordingOracle: this.web3ConfigService.hCaptchaOracleAddress, - }; - } else if (Object.values(CvatJobType).includes(jobType as CvatJobType)) { - return { - reputationOracle: this.web3ConfigService.reputationOracleAddress, - exchangeOracle: this.web3ConfigService.cvatExchangeOracleAddress, - recordingOracle: this.web3ConfigService.cvatRecordingOracleAddress, - }; - } - - const reputationOracle = this.selectReputationOracle(); - const availableOracles = await this.web3Service.findAvailableOracles( - chainId, - jobType, - reputationOracle, - ); - - const exchangeOracle = this.selectOracleFromAvailable( - availableOracles, - Role.ExchangeOracle, - chainId, - reputationOracle, - jobType, - ); - const recordingOracle = this.selectOracleFromAvailable( - availableOracles, - Role.RecordingOracle, - chainId, - reputationOracle, - jobType, - ); - - return { reputationOracle, exchangeOracle, recordingOracle }; - } - - public async validateOracles( - chainId: ChainId, - jobType: JobRequestType, - reputationOracle: string, - exchangeOracle?: string | null, - recordingOracle?: string | null, - ) { - const reputationOracles = this.web3ConfigService.reputationOracles - .split(',') - .map((address) => address.trim()); - - if (!reputationOracles.includes(reputationOracle)) { - throw new ServerError(ErrorRoutingProtocol.ReputationOracleNotFound); - } - - const availableOracles = await this.web3Service.findAvailableOracles( - chainId, - jobType, - reputationOracle, - ); - - if ( - exchangeOracle && - !this.isOracleAvailable( - availableOracles, - exchangeOracle, - Role.ExchangeOracle, - ) - ) { - throw new ServerError(ErrorRoutingProtocol.ExchangeOracleNotFound); - } - - if ( - recordingOracle && - !this.isOracleAvailable( - availableOracles, - recordingOracle, - Role.RecordingOracle, - ) - ) { - throw new ServerError(ErrorRoutingProtocol.RecordingOracleNotFound); - } - } - - private isOracleAvailable( - availableOracles: any[], - oracle: string, - role: string, - ): boolean { - return availableOracles.some( - (o) => - o.address.toLowerCase() === oracle.toLowerCase() && o.role === role, - ); - } -} diff --git a/packages/apps/job-launcher/server/src/modules/web3/web3.service.spec.ts b/packages/apps/job-launcher/server/src/modules/web3/web3.service.spec.ts index dcaaadd94b..d0b7bc5688 100644 --- a/packages/apps/job-launcher/server/src/modules/web3/web3.service.spec.ts +++ b/packages/apps/job-launcher/server/src/modules/web3/web3.service.spec.ts @@ -15,7 +15,6 @@ import { MOCK_ADDRESS, MOCK_EXCHANGE_ORACLE_URL, MOCK_RECORDING_ORACLE_URL, - MOCK_REPUTATION_ORACLES, mockConfig, } from './../../../test/constants'; import { RateService } from '../rate/rate.service'; @@ -359,12 +358,6 @@ describe('Web3Service', () => { }); describe('getReputationOraclesByJobType', () => { - beforeEach(async () => { - jest - .spyOn(web3Service.web3ConfigService, 'reputationOracles', 'get') - .mockReturnValue(MOCK_REPUTATION_ORACLES); - }); - afterEach(() => { jest.clearAllMocks(); }); @@ -494,20 +487,6 @@ describe('Web3Service', () => { expect(result).toEqual([]); expect(OperatorUtils.getOperator).toHaveBeenCalledTimes(1); }); - - it('should return an empty array if no reputation oracles are configured', async () => { - jest - .spyOn(web3Service.web3ConfigService, 'reputationOracles', 'get') - .mockReturnValue(''); - - const result = await web3Service.getReputationOraclesByJobType( - ChainId.POLYGON_AMOY, - 'Points', - ); - - expect(result).toEqual([]); - expect(OperatorUtils.getOperator).toHaveBeenCalledTimes(1); - }); }); describe('ensureEscrowAllowance', () => { diff --git a/packages/apps/job-launcher/server/src/modules/webhook/webhook.controller.spec.ts b/packages/apps/job-launcher/server/src/modules/webhook/webhook.controller.spec.ts index 22305582c9..cec113cd8e 100644 --- a/packages/apps/job-launcher/server/src/modules/webhook/webhook.controller.spec.ts +++ b/packages/apps/job-launcher/server/src/modules/webhook/webhook.controller.spec.ts @@ -8,10 +8,6 @@ import { ConfigService } from '@nestjs/config'; import { Test, TestingModule } from '@nestjs/testing'; import { MOCK_ADDRESS, - MOCK_CVAT_JOB_SIZE, - MOCK_CVAT_MAX_TIME, - MOCK_CVAT_SKELETONS_JOB_SIZE_MULTIPLIER, - MOCK_CVAT_VAL_SIZE, MOCK_EXPIRES_IN, MOCK_HCAPTCHA_SITE_KEY, MOCK_MAX_RETRY_COUNT, @@ -70,11 +66,6 @@ describe('WebhookController', () => { HCAPTCHA_REPUTATION_ORACLE_URI: MOCK_REPUTATION_ORACLE_URL, HCAPTCHA_SECRET: MOCK_SECRET, JWT_ACCESS_TOKEN_EXPIRES_IN: MOCK_EXPIRES_IN, - CVAT_JOB_SIZE: MOCK_CVAT_JOB_SIZE, - CVAT_MAX_TIME: MOCK_CVAT_MAX_TIME, - CVAT_VAL_SIZE: MOCK_CVAT_VAL_SIZE, - CVAT_SKELETONS_JOB_SIZE_MULTIPLIER: - MOCK_CVAT_SKELETONS_JOB_SIZE_MULTIPLIER, }; const module: TestingModule = await Test.createTestingModule({ diff --git a/packages/apps/job-launcher/server/test/constants.ts b/packages/apps/job-launcher/server/test/constants.ts index d28777eef2..8870601340 100644 --- a/packages/apps/job-launcher/server/test/constants.ts +++ b/packages/apps/job-launcher/server/test/constants.ts @@ -1,51 +1,20 @@ import { FortuneJobType } from '../src/common/enums/job'; -import { AWSRegions, StorageProviders } from '../src/common/enums/storage'; import { Web3Env } from '../src/common/enums/web3'; -import { CvatDataDto, StorageDataDto } from '../src/modules/job/job.dto'; -import { - FortuneManifestDto, - Label, -} from '../src/modules/manifest/manifest.dto'; +import { FortuneManifestDto } from '../src/modules/manifest/manifest.dto'; -export const MOCK_REQUESTER_TITLE = 'Mock job title'; -export const MOCK_REQUESTER_DESCRIPTION = 'Mock job description'; -export const MOCK_SUBMISSION_REQUIRED = 5; -export const MOCK_CHAIN_ID = 1; export const MOCK_ADDRESS = '0xCf88b3f1992458C2f5a229573c768D0E9F70C44e'; export const MOCK_FILE_URL = 'http://mockedFileUrl.test/bucket/file.json'; export const MOCK_FILE_HASH = 'mockedFileHash'; -export const MOCK_FILE_KEY = 'manifest.json'; -export const MOCK_BUCKET_FILES = [ - 'file0', - 'file1', - 'file2', - 'file3', - 'file4', - 'file5', -]; export const MOCK_PRIVATE_KEY = 'd334daf65a631f40549cc7de126d5a0016f32a2d00c49f94563f9737f7135e55'; -export const MOCK_GAS_PRICE_MULTIPLIER = 1; -export const MOCK_REPUTATION_ORACLES = - '0x0000000000000000000000000000000000000001,0x0000000000000000000000000000000000000002,0x0000000000000000000000000000000000000003'; -export const MOCK_REPUTATION_ORACLE_1 = - '0x0000000000000000000000000000000000000001'; export const MOCK_WEB3_RPC_URL = 'http://localhost:8545'; -export const MOCK_WEB3_NODE_HOST = 'localhost'; -export const MOCK_BUCKET_NAME = 'bucket-name'; export const MOCK_EXCHANGE_ORACLE_ADDRESS = '0xCf88b3f1992458C2f5a229573c768D0E9F70C44e'; -export const MOCK_RECORDING_ORACLE_ADDRESS = - '0xCf88b3f1992458C2f5a229573c768D0E9F70C44e'; -export const MOCK_REPUTATION_ORACLE_ADDRESS = - '0x2E04d5D6cE3fF2261D0Cb04d41Fb4Cd67362A473'; export const MOCK_EXCHANGE_ORACLE_WEBHOOK_URL = 'http://localhost:3000'; export const MOCK_REPUTATION_ORACLE_URL = 'http://reporacle:3000'; export const MOCK_RECORDING_ORACLE_URL = 'http://recoracle:3000'; export const MOCK_EXCHANGE_ORACLE_URL = 'http://exoracle:3000'; export const MOCK_SECRET = 'secrete'; -export const MOCK_JOB_LAUNCHER_FEE = 5; -export const MOCK_ORACLE_FEE = 5; export const MOCK_TRANSACTION_HASH = '0xd28e4c40571530afcb25ea1890e77b2d18c35f06049980ca4fb71829f64d89dc'; export const MOCK_SIGNATURE = @@ -53,22 +22,15 @@ export const MOCK_SIGNATURE = export const MOCK_EMAIL = 'test@example.com'; export const MOCK_PASSWORD = 'password123'; export const MOCK_HASHED_PASSWORD = 'hashedPassword'; -export const MOCK_CUSTOMER_ID = 'customer123'; export const MOCK_PAYMENT_ID = 'payment123'; export const MOCK_ACCESS_TOKEN = 'access_token'; export const MOCK_REFRESH_TOKEN = 'refresh_token'; -export const MOCK_ACCESS_TOKEN_HASHED = 'access_token_hashed'; -export const MOCK_REFRESH_TOKEN_HASHED = 'refresh_token_hashed'; export const MOCK_EXPIRES_IN = 1000000000000000; -export const MOCK_USER_ID = 1; -export const MOCK_JOB_ID = 1; export const MOCK_SENDGRID_API_KEY = 'SG.xxxxxxxxxxxxxxxxxxxxxx.xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'; export const MOCK_PAYMENT_PROVIDER_SECRET_KEY = 'xxxxxxxxxxxxxxxxxxxxxx'; -export const MOCK_COINGECKO_API_KEY = 'xxxxxxxxxxxxxxxxxxxxxx'; export const MOCK_PAYMENT_PROVIDER_API_VERSION = '2022-11-15'; -export const MOCK_PAYMENT_PROVIDER_APP_NAME = 'Name'; export const MOCK_PAYMENT_PROVIDER_APP_INFO_URL = 'https://test-app-url.com'; export const MOCK_SENDGRID_FROM_EMAIL = 'info@hmt.ai'; export const MOCK_SENDGRID_FROM_NAME = 'John Doe'; @@ -82,7 +44,6 @@ export const MOCK_MANIFEST: FortuneManifestDto = { submissionsRequired: 2, requesterTitle: 'Fortune', requesterDescription: 'Some desc', - fundAmount: 10, requestType: FortuneJobType.FORTUNE, }; @@ -117,124 +78,12 @@ Fx3dwWk9YaZ4lQD+MHnMYu48TwdE4ZKNcNUaOmWLBbZTgedqqHGLXbiyZAg= =IMAe -----END PGP PUBLIC KEY BLOCK-----`; export const MOCK_PGP_PASSPHRASE = ''; -export const MOCK_HCAPTCHA_ORACLE_ADDRESS = - '0xa62a1c18571b869e43eeabd217e233e7f0275af3'; -export const MOCK_CVAT_JOB_SIZE = '1'; -export const MOCK_CVAT_MAX_TIME = '300'; -export const MOCK_CVAT_VAL_SIZE = '2'; -export const MOCK_CVAT_SKELETONS_JOB_SIZE_MULTIPLIER = '6'; export const MOCK_HCAPTCHA_SITE_KEY = '1234'; -export const MOCK_HCAPTCHA_IMAGE_URL = - 'http://mockedFileUrl.test/bucket/img_1.jpg'; -export const MOCK_HCAPTCHA_IMAGE_LABEL = 'cat'; -export const MOCK_HCAPTCHA_REPO_URI = 'http://recoracle:3000'; -export const MOCK_HCAPTCHA_RO_URI = 'http://recoracle:3000'; export const MOCK_MAX_RETRY_COUNT = 5; -export const MOCK_STORAGE_DATA: StorageDataDto = { - provider: StorageProviders.AWS, - region: AWSRegions.EU_CENTRAL_1, - bucketName: 'bucket', - path: 'folder/test', -}; -export const MOCK_CVAT_DATA_DATASET: CvatDataDto = { - dataset: MOCK_STORAGE_DATA, -}; - -export const MOCK_CVAT_DATA_POINTS: CvatDataDto = { - dataset: MOCK_STORAGE_DATA, - points: MOCK_STORAGE_DATA, -}; - -export const MOCK_CVAT_DATA_BOXES: CvatDataDto = { - dataset: MOCK_STORAGE_DATA, - boxes: MOCK_STORAGE_DATA, -}; - -export const MOCK_CVAT_LABELS: Label[] = [ - { - name: 'label1', - }, - { - name: 'label2', - }, -]; - -export const MOCK_CVAT_LABELS_WITH_NODES: Label[] = [ - { - name: 'label1', - nodes: ['node 1', 'node 2', 'node 3', 'node 4'], - }, - { - name: 'label2', - nodes: ['node 1', 'node 2', 'node 3', 'node 4'], - }, -]; - -export const MOCK_BUCKET_FILE = - 'https://bucket.s3.eu-central-1.amazonaws.com/folder/test'; - -export const MOCK_CVAT_DATA = { - images: [ - { - id: 1, - file_name: '1.jpg', - }, - { - id: 2, - file_name: '2.jpg', - }, - { - id: 3, - file_name: '3.jpg', - }, - { - id: 4, - file_name: '4.jpg', - }, - { - id: 5, - file_name: '5.jpg', - }, - ], - annotations: [ - { - image_id: 1, - }, - { - image_id: 2, - }, - { - image_id: 3, - }, - { - image_id: 4, - }, - { - image_id: 5, - }, - ], -}; - -export const MOCK_CVAT_GT = { - images: [ - { - file_name: '1.jpg', - }, - { - file_name: '2.jpg', - }, - { - file_name: '3.jpg', - }, - ], -}; - -export const MOCK_MINIMUM_FEE_USD = 0.01; export const MOCK_RATE_CACHE_TIME = 30; export const MOCK_FE_URL = 'http://localhost:3001'; export const mockConfig: any = { - MINIMUM_FEE_USD: MOCK_MINIMUM_FEE_USD, RATE_CACHE_TIME: MOCK_RATE_CACHE_TIME, S3_ACCESS_KEY: MOCK_S3_ACCESS_KEY, S3_SECRET_KEY: MOCK_S3_SECRET_KEY, @@ -249,27 +98,18 @@ export const mockConfig: any = { WEB3_PRIVATE_KEY: MOCK_PRIVATE_KEY, PAYMENT_PROVIDER_SECRET_KEY: MOCK_PAYMENT_PROVIDER_SECRET_KEY, PAYMENT_PROVIDER_API_VERSION: MOCK_PAYMENT_PROVIDER_API_VERSION, - PAYMENT_PROVIDER_APP_NAME: MOCK_PAYMENT_PROVIDER_APP_NAME, PAYMENT_PROVIDER_APP_INFO_URL: MOCK_PAYMENT_PROVIDER_APP_INFO_URL, - CVAT_EXCHANGE_ORACLE_ADDRESS: MOCK_ADDRESS, - CVAT_RECORDING_ORACLE_ADDRESS: MOCK_ADDRESS, HCAPTCHA_SITE_KEY: MOCK_HCAPTCHA_SITE_KEY, HCAPTCHA_RECORDING_ORACLE_URI: MOCK_RECORDING_ORACLE_URL, HCAPTCHA_REPUTATION_ORACLE_URI: MOCK_REPUTATION_ORACLE_URL, HCAPTCHA_ORACLE_ADDRESS: MOCK_ADDRESS, HCAPTCHA_SECRET: MOCK_SECRET, JWT_ACCESS_TOKEN_EXPIRES_IN: MOCK_EXPIRES_IN, - CVAT_JOB_SIZE: MOCK_CVAT_JOB_SIZE, - CVAT_MAX_TIME: MOCK_CVAT_MAX_TIME, - CVAT_VAL_SIZE: MOCK_CVAT_VAL_SIZE, - CVAT_SKELETONS_JOB_SIZE_MULTIPLIER: MOCK_CVAT_SKELETONS_JOB_SIZE_MULTIPLIER, MAX_RETRY_COUNT: MOCK_MAX_RETRY_COUNT, RPC_URL_POLYGON_AMOY: MOCK_WEB3_RPC_URL, SENDGRID_API_KEY: MOCK_SENDGRID_API_KEY, SENDGRID_FROM_EMAIL: MOCK_SENDGRID_FROM_EMAIL, SENDGRID_FROM_NAME: MOCK_SENDGRID_FROM_NAME, - REPUTATION_ORACLES: MOCK_REPUTATION_ORACLES, WEB3_ENV: Web3Env.TESTNET, - COINGECKO_API_KEY: MOCK_COINGECKO_API_KEY, FE_URL: MOCK_FE_URL, }; diff --git a/packages/apps/reputation-oracle/server/src/common/types/manifest.ts b/packages/apps/reputation-oracle/server/src/common/types/manifest.ts index d52ffca23c..47616fafea 100644 --- a/packages/apps/reputation-oracle/server/src/common/types/manifest.ts +++ b/packages/apps/reputation-oracle/server/src/common/types/manifest.ts @@ -2,7 +2,6 @@ import { CvatJobType, FortuneJobType, MarketingJobType } from '@/common/enums'; export type FortuneManifest = { submissionsRequired: number; - fundAmount: number; requestType: FortuneJobType; }; diff --git a/packages/apps/reputation-oracle/server/src/modules/escrow-completion/fixtures/fortune.ts b/packages/apps/reputation-oracle/server/src/modules/escrow-completion/fixtures/fortune.ts index 90527b6fa0..7978fdce20 100644 --- a/packages/apps/reputation-oracle/server/src/modules/escrow-completion/fixtures/fortune.ts +++ b/packages/apps/reputation-oracle/server/src/modules/escrow-completion/fixtures/fortune.ts @@ -6,7 +6,6 @@ import { FortuneFinalResult, FortuneManifest } from '@/common/types'; export function generateFortuneManifest(): FortuneManifest { return { requestType: FortuneJobType.FORTUNE, - fundAmount: Number(faker.finance.amount()), submissionsRequired: faker.number.int({ min: 2, max: 5 }), }; } From ede7be5a00371c014a04372ba2cb48601ae31395 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 13 May 2026 12:31:55 +0200 Subject: [PATCH 2/6] chore(deps): bump serve from 14.2.5 to 14.2.6 (#3895) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- packages/apps/faucet/client/package.json | 2 +- .../exchange-oracle/client/package.json | 2 +- packages/apps/human-app/frontend/package.json | 2 +- .../apps/job-launcher/client/package.json | 2 +- packages/apps/staking/package.json | 2 +- yarn.lock | 63 +++++++++++-------- 6 files changed, 41 insertions(+), 32 deletions(-) diff --git a/packages/apps/faucet/client/package.json b/packages/apps/faucet/client/package.json index 9c0606702e..60db7f4122 100644 --- a/packages/apps/faucet/client/package.json +++ b/packages/apps/faucet/client/package.json @@ -27,7 +27,7 @@ "react-dom": "^19.2.4", "react-loading-skeleton": "^3.3.1", "react-router-dom": "^7.13.0", - "serve": "^14.2.4", + "serve": "^14.2.6", "viem": "2.x" }, "devDependencies": { diff --git a/packages/apps/fortune/exchange-oracle/client/package.json b/packages/apps/fortune/exchange-oracle/client/package.json index 133c683064..b55a02ad79 100644 --- a/packages/apps/fortune/exchange-oracle/client/package.json +++ b/packages/apps/fortune/exchange-oracle/client/package.json @@ -40,7 +40,7 @@ "react": "^19.2.4", "react-dom": "^19.2.4", "react-router-dom": "^7.13.0", - "serve": "^14.2.4", + "serve": "^14.2.6", "viem": "2.x", "wagmi": "^2.14.6" }, diff --git a/packages/apps/human-app/frontend/package.json b/packages/apps/human-app/frontend/package.json index 32e15a8ab3..46f6984998 100644 --- a/packages/apps/human-app/frontend/package.json +++ b/packages/apps/human-app/frontend/package.json @@ -51,7 +51,7 @@ "react-imask": "^7.4.0", "react-number-format": "^5.4.5", "react-router-dom": "^7.13.0", - "serve": "^14.2.4", + "serve": "^14.2.6", "viem": "^2.31.4", "vite-plugin-svgr": "^4.2.0", "wagmi": "^2.15.6", diff --git a/packages/apps/job-launcher/client/package.json b/packages/apps/job-launcher/client/package.json index 563219e225..d415708d3a 100644 --- a/packages/apps/job-launcher/client/package.json +++ b/packages/apps/job-launcher/client/package.json @@ -32,7 +32,7 @@ "react-redux": "^9.1.0", "react-router-dom": "^7.13.0", "recharts": "^2.7.2", - "serve": "^14.2.4", + "serve": "^14.2.6", "swr": "^2.2.4", "typescript": "^5.6.3", "viem": "2.x", diff --git a/packages/apps/staking/package.json b/packages/apps/staking/package.json index dd4da353ee..64de60f6c0 100644 --- a/packages/apps/staking/package.json +++ b/packages/apps/staking/package.json @@ -42,7 +42,7 @@ "react": "^19.2.4", "react-dom": "^19.2.4", "react-router-dom": "^7.13.0", - "serve": "^14.2.4", + "serve": "^14.2.6", "simplebar-react": "^3.3.2", "viem": "2.x", "wagmi": "^2.14.6" diff --git a/yarn.lock b/yarn.lock index a52a69eb8d..005f0b3277 100644 --- a/yarn.lock +++ b/yarn.lock @@ -230,7 +230,7 @@ __metadata: react-dom: "npm:^19.2.4" react-loading-skeleton: "npm:^3.3.1" react-router-dom: "npm:^7.13.0" - serve: "npm:^14.2.4" + serve: "npm:^14.2.6" typescript: "npm:^5.8.3" typescript-eslint: "npm:^8.57.0" viem: "npm:2.x" @@ -298,7 +298,7 @@ __metadata: react: "npm:^19.2.4" react-dom: "npm:^19.2.4" react-router-dom: "npm:^7.13.0" - serve: "npm:^14.2.4" + serve: "npm:^14.2.6" typescript: "npm:^5.6.3" typescript-eslint: "npm:^8.57.0" viem: "npm:2.x" @@ -471,7 +471,7 @@ __metadata: react-imask: "npm:^7.4.0" react-number-format: "npm:^5.4.5" react-router-dom: "npm:^7.13.0" - serve: "npm:^14.2.4" + serve: "npm:^14.2.6" typescript: "npm:^5.6.3" typescript-eslint: "npm:^8.57.0" viem: "npm:^2.31.4" @@ -595,7 +595,7 @@ __metadata: react-router-dom: "npm:^7.13.0" recharts: "npm:^2.7.2" resize-observer-polyfill: "npm:^1.5.1" - serve: "npm:^14.2.4" + serve: "npm:^14.2.6" swr: "npm:^2.2.4" typescript: "npm:^5.6.3" typescript-eslint: "npm:^8.57.0" @@ -788,7 +788,7 @@ __metadata: react-dom: "npm:^19.2.4" react-router-dom: "npm:^7.13.0" sass: "npm:^1.89.2" - serve: "npm:^14.2.4" + serve: "npm:^14.2.6" simplebar-react: "npm:^3.3.2" typescript: "npm:^5.6.3" typescript-eslint: "npm:^8.57.0" @@ -14391,27 +14391,27 @@ __metadata: languageName: node linkType: hard -"ajv@npm:8.12.0": - version: 8.12.0 - resolution: "ajv@npm:8.12.0" +"ajv@npm:8.17.1, ajv@npm:^8.0.0, ajv@npm:^8.0.1, ajv@npm:^8.9.0": + version: 8.17.1 + resolution: "ajv@npm:8.17.1" dependencies: - fast-deep-equal: "npm:^3.1.1" + fast-deep-equal: "npm:^3.1.3" + fast-uri: "npm:^3.0.1" json-schema-traverse: "npm:^1.0.0" require-from-string: "npm:^2.0.2" - uri-js: "npm:^4.2.2" - checksum: 10c0/ac4f72adf727ee425e049bc9d8b31d4a57e1c90da8d28bcd23d60781b12fcd6fc3d68db5df16994c57b78b94eed7988f5a6b482fd376dc5b084125e20a0a622e + checksum: 10c0/ec3ba10a573c6b60f94639ffc53526275917a2df6810e4ab5a6b959d87459f9ef3f00d5e7865b82677cb7d21590355b34da14d1d0b9c32d75f95a187e76fff35 languageName: node linkType: hard -"ajv@npm:8.17.1, ajv@npm:^8.0.0, ajv@npm:^8.0.1, ajv@npm:^8.9.0": - version: 8.17.1 - resolution: "ajv@npm:8.17.1" +"ajv@npm:8.18.0": + version: 8.18.0 + resolution: "ajv@npm:8.18.0" dependencies: fast-deep-equal: "npm:^3.1.3" fast-uri: "npm:^3.0.1" json-schema-traverse: "npm:^1.0.0" require-from-string: "npm:^2.0.2" - checksum: 10c0/ec3ba10a573c6b60f94639ffc53526275917a2df6810e4ab5a6b959d87459f9ef3f00d5e7865b82677cb7d21590355b34da14d1d0b9c32d75f95a187e76fff35 + checksum: 10c0/e7517c426173513a07391be951879932bdf3348feaebd2199f5b901c20f99d60db8cd1591502d4d551dc82f594e82a05c4fe1c70139b15b8937f7afeaed9532f languageName: node linkType: hard @@ -24430,7 +24430,7 @@ __metadata: languageName: node linkType: hard -"minimatch@npm:2 || 3, minimatch@npm:3.1.2, minimatch@npm:^3.0.2, minimatch@npm:^3.0.4, minimatch@npm:^3.0.5, minimatch@npm:^3.1.1, minimatch@npm:^3.1.2": +"minimatch@npm:2 || 3, minimatch@npm:^3.0.2, minimatch@npm:^3.0.4, minimatch@npm:^3.0.5, minimatch@npm:^3.1.1, minimatch@npm:^3.1.2": version: 3.1.2 resolution: "minimatch@npm:3.1.2" dependencies: @@ -24439,6 +24439,15 @@ __metadata: languageName: node linkType: hard +"minimatch@npm:3.1.5": + version: 3.1.5 + resolution: "minimatch@npm:3.1.5" + dependencies: + brace-expansion: "npm:^1.1.7" + checksum: 10c0/2ecbdc0d33f07bddb0315a8b5afbcb761307a8778b48f0b312418ccbced99f104a2d17d8aca7573433c70e8ccd1c56823a441897a45e384ea76ef401a26ace70 + languageName: node + linkType: hard + "minimatch@npm:^10.0.0, minimatch@npm:^10.1.1": version: 10.1.1 resolution: "minimatch@npm:10.1.1" @@ -28312,18 +28321,18 @@ __metadata: languageName: node linkType: hard -"serve-handler@npm:6.1.6": - version: 6.1.6 - resolution: "serve-handler@npm:6.1.6" +"serve-handler@npm:6.1.7": + version: 6.1.7 + resolution: "serve-handler@npm:6.1.7" dependencies: bytes: "npm:3.0.0" content-disposition: "npm:0.5.2" mime-types: "npm:2.1.18" - minimatch: "npm:3.1.2" + minimatch: "npm:3.1.5" path-is-inside: "npm:1.0.2" path-to-regexp: "npm:3.3.0" range-parser: "npm:1.2.0" - checksum: 10c0/1e1cb6bbc51ee32bc1505f2e0605bdc2e96605c522277c977b67f83be9d66bd1eec8604388714a4d728e036d86b629bc9aec02120ea030d3d2c3899d44696503 + checksum: 10c0/35afb68d81afd3c38d15792a5bc2451915b739bef2898a47ebd190db6a4e29846530ac00292b8008fe7297a819257c3948be2deaf4ffd32c96689e8947cf0ae9 languageName: node linkType: hard @@ -28339,12 +28348,12 @@ __metadata: languageName: node linkType: hard -"serve@npm:^14.2.4": - version: 14.2.5 - resolution: "serve@npm:14.2.5" +"serve@npm:^14.2.6": + version: 14.2.6 + resolution: "serve@npm:14.2.6" dependencies: "@zeit/schemas": "npm:2.36.0" - ajv: "npm:8.12.0" + ajv: "npm:8.18.0" arg: "npm:5.0.2" boxen: "npm:7.0.0" chalk: "npm:5.0.1" @@ -28352,11 +28361,11 @@ __metadata: clipboardy: "npm:3.0.0" compression: "npm:1.8.1" is-port-reachable: "npm:4.0.0" - serve-handler: "npm:6.1.6" + serve-handler: "npm:6.1.7" update-check: "npm:1.5.4" bin: serve: build/main.js - checksum: 10c0/7324a037beea0ee0211f2384e7af28ddf57c8297649e5dd0145ed5a48861cab6d680cbdce332ee9b517f745a31881e5c70074f0908d12c0a4b052cd65f4e9b7e + checksum: 10c0/7e1668e0d187719dbe4f3de967012ce2263c967f6135d9c630f803b0f173334e1442ab326fcc4c8e6cd4e293d8bd8c773aebab2746ecaa0fb1ab29a36079763b languageName: node linkType: hard From b61936693a2634c53299661d2a648111b2afa301 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 13 May 2026 12:39:42 +0200 Subject: [PATCH 3/6] chore(deps): bump actions/dependency-review-action from 4.9.0 to 5.0.0 (#3890) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci-dependency-review.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-dependency-review.yaml b/.github/workflows/ci-dependency-review.yaml index d5eb760ed3..f8af980edc 100644 --- a/.github/workflows/ci-dependency-review.yaml +++ b/.github/workflows/ci-dependency-review.yaml @@ -14,6 +14,6 @@ jobs: steps: - uses: actions/checkout@v6 - name: Dependency Review - uses: actions/dependency-review-action@v4.9.0 + uses: actions/dependency-review-action@v5.0.0 with: show-openssf-scorecard: false From 468009c06c88abeb97a0ac0b82caf1346ec9dfcc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 13 May 2026 13:07:20 +0200 Subject: [PATCH 4/6] chore(deps): bump @nestjs/schedule from 6.1.1 to 6.1.3 (#3892) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- packages/apps/dashboard/server/package.json | 2 +- .../exchange-oracle/server/package.json | 2 +- packages/apps/human-app/server/package.json | 2 +- packages/apps/job-launcher/server/package.json | 2 +- .../apps/reputation-oracle/server/package.json | 2 +- yarn.lock | 18 +++++++++--------- 6 files changed, 14 insertions(+), 14 deletions(-) diff --git a/packages/apps/dashboard/server/package.json b/packages/apps/dashboard/server/package.json index 7b5eda2d43..243b7449d2 100644 --- a/packages/apps/dashboard/server/package.json +++ b/packages/apps/dashboard/server/package.json @@ -32,7 +32,7 @@ "@nestjs/core": "^11.1.12", "@nestjs/mapped-types": "^2.1.0", "@nestjs/platform-express": "^11.1.12", - "@nestjs/schedule": "^6.1.1", + "@nestjs/schedule": "^6.1.3", "@nestjs/swagger": "^11.2.5", "axios": "^1.3.1", "cache-manager": "7.2.8", diff --git a/packages/apps/fortune/exchange-oracle/server/package.json b/packages/apps/fortune/exchange-oracle/server/package.json index 79e993faac..642c726963 100644 --- a/packages/apps/fortune/exchange-oracle/server/package.json +++ b/packages/apps/fortune/exchange-oracle/server/package.json @@ -38,7 +38,7 @@ "@nestjs/core": "^11.1.12", "@nestjs/passport": "^11.0.5", "@nestjs/platform-express": "^11.1.12", - "@nestjs/schedule": "^6.1.1", + "@nestjs/schedule": "^6.1.3", "@nestjs/swagger": "^11.2.5", "@nestjs/terminus": "^11.1.1", "@nestjs/typeorm": "^11.0.0", diff --git a/packages/apps/human-app/server/package.json b/packages/apps/human-app/server/package.json index eb0b876c9b..16170c9920 100644 --- a/packages/apps/human-app/server/package.json +++ b/packages/apps/human-app/server/package.json @@ -35,7 +35,7 @@ "@nestjs/core": "^11.1.12", "@nestjs/passport": "^11.0.5", "@nestjs/platform-express": "^11.1.12", - "@nestjs/schedule": "^6.1.1", + "@nestjs/schedule": "^6.1.3", "@nestjs/swagger": "^11.2.5", "@nestjs/terminus": "^11.1.1", "@types/passport-jwt": "^4.0.1", diff --git a/packages/apps/job-launcher/server/package.json b/packages/apps/job-launcher/server/package.json index 6c76a5d839..33ca7b21b7 100644 --- a/packages/apps/job-launcher/server/package.json +++ b/packages/apps/job-launcher/server/package.json @@ -41,7 +41,7 @@ "@nestjs/jwt": "^11.0.2", "@nestjs/passport": "^11.0.5", "@nestjs/platform-express": "^11.1.12", - "@nestjs/schedule": "^6.1.1", + "@nestjs/schedule": "^6.1.3", "@nestjs/swagger": "^11.2.5", "@nestjs/terminus": "^11.1.1", "@nestjs/throttler": "^6.5.0", diff --git a/packages/apps/reputation-oracle/server/package.json b/packages/apps/reputation-oracle/server/package.json index b265289a56..d7925bf0f4 100644 --- a/packages/apps/reputation-oracle/server/package.json +++ b/packages/apps/reputation-oracle/server/package.json @@ -40,7 +40,7 @@ "@nestjs/jwt": "^11.0.2", "@nestjs/passport": "^11.0.5", "@nestjs/platform-express": "^11.1.12", - "@nestjs/schedule": "^6.1.1", + "@nestjs/schedule": "^6.1.3", "@nestjs/swagger": "^11.2.5", "@nestjs/terminus": "^11.1.1", "@nestjs/typeorm": "^11.0.0", diff --git a/yarn.lock b/yarn.lock index 005f0b3277..7849653e6f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -169,7 +169,7 @@ __metadata: "@nestjs/core": "npm:^11.1.12" "@nestjs/mapped-types": "npm:^2.1.0" "@nestjs/platform-express": "npm:^11.1.12" - "@nestjs/schedule": "npm:^6.1.1" + "@nestjs/schedule": "npm:^6.1.3" "@nestjs/schematics": "npm:^11.0.9" "@nestjs/swagger": "npm:^11.2.5" "@nestjs/testing": "npm:^11.1.19" @@ -323,7 +323,7 @@ __metadata: "@nestjs/core": "npm:^11.1.12" "@nestjs/passport": "npm:^11.0.5" "@nestjs/platform-express": "npm:^11.1.12" - "@nestjs/schedule": "npm:^6.1.1" + "@nestjs/schedule": "npm:^6.1.3" "@nestjs/schematics": "npm:^11.0.9" "@nestjs/swagger": "npm:^11.2.5" "@nestjs/terminus": "npm:^11.1.1" @@ -504,7 +504,7 @@ __metadata: "@nestjs/core": "npm:^11.1.12" "@nestjs/passport": "npm:^11.0.5" "@nestjs/platform-express": "npm:^11.1.12" - "@nestjs/schedule": "npm:^6.1.1" + "@nestjs/schedule": "npm:^6.1.3" "@nestjs/schematics": "npm:^11.0.9" "@nestjs/swagger": "npm:^11.2.5" "@nestjs/terminus": "npm:^11.1.1" @@ -627,7 +627,7 @@ __metadata: "@nestjs/jwt": "npm:^11.0.2" "@nestjs/passport": "npm:^11.0.5" "@nestjs/platform-express": "npm:^11.1.12" - "@nestjs/schedule": "npm:^6.1.1" + "@nestjs/schedule": "npm:^6.1.3" "@nestjs/schematics": "npm:^11.0.9" "@nestjs/swagger": "npm:^11.2.5" "@nestjs/terminus": "npm:^11.1.1" @@ -701,7 +701,7 @@ __metadata: "@nestjs/jwt": "npm:^11.0.2" "@nestjs/passport": "npm:^11.0.5" "@nestjs/platform-express": "npm:^11.1.12" - "@nestjs/schedule": "npm:^6.1.1" + "@nestjs/schedule": "npm:^6.1.3" "@nestjs/schematics": "npm:^11.0.9" "@nestjs/swagger": "npm:^11.2.5" "@nestjs/terminus": "npm:^11.1.1" @@ -7699,15 +7699,15 @@ __metadata: languageName: node linkType: hard -"@nestjs/schedule@npm:^6.1.1": - version: 6.1.1 - resolution: "@nestjs/schedule@npm:6.1.1" +"@nestjs/schedule@npm:^6.1.3": + version: 6.1.3 + resolution: "@nestjs/schedule@npm:6.1.3" dependencies: cron: "npm:4.4.0" peerDependencies: "@nestjs/common": ^10.0.0 || ^11.0.0 "@nestjs/core": ^10.0.0 || ^11.0.0 - checksum: 10c0/0185efe4824bd353fea792954ea04d2d4cb99836c93c6d42d76fe9d57a6fd97eb7ba9d874be6f7f94b6f8bbca11badab16993f30e9b1f94b4c341639407578de + checksum: 10c0/de6da8d0246f486f7fdefaf6991fea18718f7a2d2eb698cf40094bfaa21e028107e37a27352af4571370c14cd757ca7402b5398af544438cc046d28575dee827 languageName: node linkType: hard From 506cd94b81205fd840a6e2ca6de4fc7777585547 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 13 May 2026 13:12:14 +0200 Subject: [PATCH 5/6] chore(deps): bump swr from 2.3.6 to 2.4.1 (#3891) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- packages/apps/job-launcher/client/package.json | 2 +- yarn.lock | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/apps/job-launcher/client/package.json b/packages/apps/job-launcher/client/package.json index d415708d3a..bbb56d5cb4 100644 --- a/packages/apps/job-launcher/client/package.json +++ b/packages/apps/job-launcher/client/package.json @@ -33,7 +33,7 @@ "react-router-dom": "^7.13.0", "recharts": "^2.7.2", "serve": "^14.2.6", - "swr": "^2.2.4", + "swr": "^2.4.1", "typescript": "^5.6.3", "viem": "2.x", "wagmi": "^2.14.6", diff --git a/yarn.lock b/yarn.lock index 7849653e6f..8e782c5a2a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -596,7 +596,7 @@ __metadata: recharts: "npm:^2.7.2" resize-observer-polyfill: "npm:^1.5.1" serve: "npm:^14.2.6" - swr: "npm:^2.2.4" + swr: "npm:^2.4.1" typescript: "npm:^5.6.3" typescript-eslint: "npm:^8.57.0" viem: "npm:2.x" @@ -29552,15 +29552,15 @@ __metadata: languageName: node linkType: hard -"swr@npm:^2.2.4": - version: 2.3.6 - resolution: "swr@npm:2.3.6" +"swr@npm:^2.4.1": + version: 2.4.1 + resolution: "swr@npm:2.4.1" dependencies: dequal: "npm:^2.0.3" - use-sync-external-store: "npm:^1.4.0" + use-sync-external-store: "npm:^1.6.0" peerDependencies: react: ^16.11.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 - checksum: 10c0/9534f350982e36a3ae0a13da8c0f7da7011fc979e77f306e60c4e5db0f9b84f17172c44f973441ba56bb684b69b0d9838ab40011a6b6b3e32d0cd7f3d5405f99 + checksum: 10c0/34d61fb4653ac8875ad24e7c6da37e210b0e90fce0815dc59f013b7554a0bd267e79aac0f8ae5fbf04992e2a1815ee3da581b0dab3ed6ac4c2ce0e82b351320f languageName: node linkType: hard From d4117d898dfaa6afacb435d055a2d7531bc8680c Mon Sep 17 00:00:00 2001 From: portuu3 <61605646+portuu3@users.noreply.github.com> Date: Wed, 13 May 2026 15:13:30 +0200 Subject: [PATCH 6/6] Enable reputation accounting for workers by job type (#3888) --- .../server/src/common/enums/manifest.ts | 6 ++ .../server/src/common/types/manifest.ts | 9 ++- .../1777986717078-reputationJobType.ts | 29 ++++++++++ .../escrow-completion.service.spec.ts | 38 +++++++++++++ .../escrow-completion.service.ts | 57 +++++++++++++++++++ .../src/modules/reputation/fixtures/index.ts | 2 + .../reputation/reputation.controller.ts | 2 + .../src/modules/reputation/reputation.dto.ts | 25 +++++++- .../modules/reputation/reputation.entity.ts | 7 ++- .../reputation/reputation.repository.ts | 12 +++- .../reputation/reputation.service.spec.ts | 17 ++++++ .../modules/reputation/reputation.service.ts | 16 ++++-- .../server/src/modules/reputation/types.ts | 4 ++ .../server/src/utils/database.ts | 7 +++ 14 files changed, 220 insertions(+), 11 deletions(-) create mode 100644 packages/apps/reputation-oracle/server/src/database/1777986717078-reputationJobType.ts create mode 100644 packages/apps/reputation-oracle/server/src/utils/database.ts diff --git a/packages/apps/reputation-oracle/server/src/common/enums/manifest.ts b/packages/apps/reputation-oracle/server/src/common/enums/manifest.ts index 9dc61669af..6ccdbb2421 100644 --- a/packages/apps/reputation-oracle/server/src/common/enums/manifest.ts +++ b/packages/apps/reputation-oracle/server/src/common/enums/manifest.ts @@ -13,3 +13,9 @@ export enum CvatJobType { IMAGE_SKELETONS_FROM_BOXES = 'image_skeletons_from_boxes', IMAGE_POLYGONS = 'image_polygons', } + +export const JobType = [ + ...Object.values(FortuneJobType), + ...Object.values(MarketingJobType), + ...Object.values(CvatJobType), +] as const; diff --git a/packages/apps/reputation-oracle/server/src/common/types/manifest.ts b/packages/apps/reputation-oracle/server/src/common/types/manifest.ts index 47616fafea..4aefc89961 100644 --- a/packages/apps/reputation-oracle/server/src/common/types/manifest.ts +++ b/packages/apps/reputation-oracle/server/src/common/types/manifest.ts @@ -1,4 +1,9 @@ -import { CvatJobType, FortuneJobType, MarketingJobType } from '@/common/enums'; +import { + CvatJobType, + FortuneJobType, + JobType, + MarketingJobType, +} from '@/common/enums'; export type FortuneManifest = { submissionsRequired: number; @@ -23,4 +28,4 @@ export type CvatManifest = { export type JobManifest = FortuneManifest | MarketingManifest | CvatManifest; -export type JobRequestType = FortuneJobType | MarketingJobType | CvatJobType; +export type JobRequestType = (typeof JobType)[number]; diff --git a/packages/apps/reputation-oracle/server/src/database/1777986717078-reputationJobType.ts b/packages/apps/reputation-oracle/server/src/database/1777986717078-reputationJobType.ts new file mode 100644 index 0000000000..376e812134 --- /dev/null +++ b/packages/apps/reputation-oracle/server/src/database/1777986717078-reputationJobType.ts @@ -0,0 +1,29 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class ReputationJobType1777986717078 implements MigrationInterface { + name = 'ReputationJobType1777986717078'; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `DROP INDEX "hmt"."IDX_5012dff596f037415a1370a0cb"`, + ); + await queryRunner.query( + `ALTER TABLE "hmt"."reputation" ADD "job_request_type" character varying NOT NULL DEFAULT 'image_skeletons_from_boxes'`, + ); + await queryRunner.query( + `CREATE UNIQUE INDEX "IDX_e2589f31dc15f8cadca6c560ff" ON "hmt"."reputation" ("chain_id", "address", "type", "job_request_type") `, + ); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `DROP INDEX "hmt"."IDX_e2589f31dc15f8cadca6c560ff"`, + ); + await queryRunner.query( + `ALTER TABLE "hmt"."reputation" DROP COLUMN "job_request_type"`, + ); + await queryRunner.query( + `CREATE UNIQUE INDEX "IDX_5012dff596f037415a1370a0cb" ON "hmt"."reputation" ("chain_id", "address", "type") `, + ); + } +} diff --git a/packages/apps/reputation-oracle/server/src/modules/escrow-completion/escrow-completion.service.spec.ts b/packages/apps/reputation-oracle/server/src/modules/escrow-completion/escrow-completion.service.spec.ts index e63eece4b1..916f276d6a 100644 --- a/packages/apps/reputation-oracle/server/src/modules/escrow-completion/escrow-completion.service.spec.ts +++ b/packages/apps/reputation-oracle/server/src/modules/escrow-completion/escrow-completion.service.spec.ts @@ -28,6 +28,7 @@ import _ from 'lodash'; import { CvatJobType, FortuneJobType, MarketingJobType } from '@/common/enums'; import { ServerConfigService, Web3ConfigService } from '@/config'; import { ReputationService } from '@/modules/reputation'; +import { ReputationEntityType } from '@/modules/reputation/constants'; import { StorageService } from '@/modules/storage'; import { WalletWithProvider, Web3Service } from '@/modules/web3'; import { @@ -708,6 +709,7 @@ describe('EscrowCompletionService', () => { let mockedSigner: SignerMock; const mockedCreateBulkPayoutTransaction = jest.fn(); let mockedRawTransaction: { nonce: number }; + let jobRequestType: FortuneJobType; beforeEach(() => { mockedSigner = createSignerMock(); @@ -725,6 +727,13 @@ describe('EscrowCompletionService', () => { mockedCreateBulkPayoutTransaction.mockResolvedValueOnce( mockedRawTransaction, ); + + const manifest = generateFortuneManifest(); + jobRequestType = manifest.requestType; + mockedEscrowUtils.getEscrow.mockResolvedValue({ + manifest: faker.internet.url(), + } as unknown as IEscrow); + mockStorageService.downloadJsonLikeData.mockResolvedValue(manifest); }); it('should succesfully process payouts batch', async () => { @@ -732,6 +741,16 @@ describe('EscrowCompletionService', () => { EscrowCompletionStatus.AWAITING_PAYOUTS, ); const payoutsBatch = generateEscrowPayoutsBatch(); + payoutsBatch.payouts = [ + { + address: faker.finance.ethereumAddress(), + amount: faker.number.bigInt({ min: 1n }).toString(), + }, + { + address: faker.finance.ethereumAddress(), + amount: faker.number.bigInt({ min: 1n }).toString(), + }, + ]; await service['processPayoutsBatch'](awaitingPayoutsRecord, { ...payoutsBatch, @@ -759,6 +778,21 @@ describe('EscrowCompletionService', () => { id: payoutsBatch.id, }), ); + + expect(mockReputationService.increaseReputation).toHaveBeenCalledTimes( + payoutsBatch.payouts.length, + ); + for (const payout of payoutsBatch.payouts) { + expect(mockReputationService.increaseReputation).toHaveBeenCalledWith( + { + chainId: awaitingPayoutsRecord.chainId, + address: payout.address, + type: ReputationEntityType.WORKER, + jobRequestType, + }, + 1, + ); + } }); it('should reset nonce if expired', async () => { @@ -859,6 +893,9 @@ describe('EscrowCompletionService', () => { launcherAddress = faker.finance.ethereumAddress(); exchangeOracleAddress = faker.finance.ethereumAddress(); recordingOracleAddress = faker.finance.ethereumAddress(); + mockStorageService.downloadJsonLikeData.mockResolvedValue( + generateFortuneManifest(), + ); }); describe('handle failures', () => { @@ -1075,6 +1112,7 @@ describe('EscrowCompletionService', () => { launcherAddress, exchangeOracleAddress, recordingOracleAddress, + FortuneJobType.FORTUNE, ); const expectedWebhookData = { diff --git a/packages/apps/reputation-oracle/server/src/modules/escrow-completion/escrow-completion.service.ts b/packages/apps/reputation-oracle/server/src/modules/escrow-completion/escrow-completion.service.ts index 46755dd3d1..cadc222b3a 100644 --- a/packages/apps/reputation-oracle/server/src/modules/escrow-completion/escrow-completion.service.ts +++ b/packages/apps/reputation-oracle/server/src/modules/escrow-completion/escrow-completion.service.ts @@ -20,6 +20,7 @@ import { ServerConfigService, Web3ConfigService } from '@/config'; import { isDuplicatedError } from '@/database'; import logger from '@/logger'; import { ReputationService } from '@/modules/reputation'; +import { ReputationEntityType } from '@/modules/reputation/constants'; import { StorageService } from '@/modules/storage'; import { Web3Service } from '@/modules/web3'; /** @@ -264,11 +265,14 @@ export class EscrowCompletionService { /** * This operation can fail and lost, so it's "at most once" */ + const jobRequestType = + await this.getJobRequestTypeFromEscrowData(escrowData); await this.reputationService.assessEscrowParties( chainId, escrowData.launcher, escrowData.exchangeOracle!, escrowData.recordingOracle!, + jobRequestType, ); } @@ -474,6 +478,17 @@ export class EscrowCompletionService { ); await this.escrowPayoutsBatchRepository.deleteOne(payoutsBatch); + const escrowData = await EscrowUtils.getEscrow( + escrowCompletionEntity.chainId, + escrowCompletionEntity.escrowAddress, + ); + const jobRequestType = + await this.getJobRequestTypeFromEscrowData(escrowData); + await this.increasePayoutRecipientsReputation( + escrowCompletionEntity.chainId, + Array.from(recipientToAmountMap.keys()), + jobRequestType, + ); } catch (error) { if (ethers.isError(error, 'NONCE_EXPIRED')) { payoutsBatch.txNonce = null; @@ -484,6 +499,48 @@ export class EscrowCompletionService { } } + private async increasePayoutRecipientsReputation( + chainId: ChainId, + recipients: string[], + jobRequestType: JobRequestType, + ): Promise { + for (const address of recipients) { + try { + await this.reputationService.increaseReputation( + { + chainId, + address, + type: ReputationEntityType.WORKER, + jobRequestType, + }, + 1, + ); + } catch (error) { + this.logger.error('Failed to increase payout recipient reputation', { + error, + address, + chainId, + jobRequestType, + }); + } + } + } + + private async getJobRequestTypeFromEscrowData( + escrowData: Awaited>, + ): Promise { + if (!escrowData) { + throw new Error('Escrow data is missing'); + } + + const manifest = + await this.storageService.downloadJsonLikeData( + escrowData.manifest as string, + ); + + return manifestUtils.getJobRequestType(manifest); + } + private getEscrowResultsProcessor( jobRequestType: JobRequestType, ): EscrowResultsProcessor { diff --git a/packages/apps/reputation-oracle/server/src/modules/reputation/fixtures/index.ts b/packages/apps/reputation-oracle/server/src/modules/reputation/fixtures/index.ts index a15853c4f1..071bfe2a70 100644 --- a/packages/apps/reputation-oracle/server/src/modules/reputation/fixtures/index.ts +++ b/packages/apps/reputation-oracle/server/src/modules/reputation/fixtures/index.ts @@ -1,5 +1,6 @@ import { faker } from '@faker-js/faker'; +import { CvatJobType } from '@/common/enums'; import { generateTestnetChainId } from '@/modules/web3/fixtures'; import { ReputationEntityType } from '../constants'; @@ -20,6 +21,7 @@ export function generateReputationEntity(score?: number): ReputationEntity { chainId: generateTestnetChainId(), address: faker.finance.ethereumAddress(), type: generateReputationEntityType(), + jobRequestType: CvatJobType.IMAGE_BOXES, reputationPoints: score || generateRandomScorePoints(), createdAt: faker.date.recent(), updatedAt: new Date(), diff --git a/packages/apps/reputation-oracle/server/src/modules/reputation/reputation.controller.ts b/packages/apps/reputation-oracle/server/src/modules/reputation/reputation.controller.ts index 839230de3d..62f349965d 100644 --- a/packages/apps/reputation-oracle/server/src/modules/reputation/reputation.controller.ts +++ b/packages/apps/reputation-oracle/server/src/modules/reputation/reputation.controller.ts @@ -47,6 +47,7 @@ export class ReputationController { const { chainId, address, + jobRequestTypes, roles, orderBy = GetReputationQueryOrderBy.CREATED_AT, orderDirection = SortDirection.DESC, @@ -58,6 +59,7 @@ export class ReputationController { { chainId, address, + jobRequestTypes, types: roles, }, { diff --git a/packages/apps/reputation-oracle/server/src/modules/reputation/reputation.dto.ts b/packages/apps/reputation-oracle/server/src/modules/reputation/reputation.dto.ts index 5132971dde..e02761885a 100644 --- a/packages/apps/reputation-oracle/server/src/modules/reputation/reputation.dto.ts +++ b/packages/apps/reputation-oracle/server/src/modules/reputation/reputation.dto.ts @@ -3,7 +3,8 @@ import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger'; import { Transform } from 'class-transformer'; import { IsEthereumAddress, IsOptional, Max, Min } from 'class-validator'; -import { SortDirection } from '@/common/enums'; +import { JobType, SortDirection } from '@/common/enums'; +import type { JobRequestType } from '@/common/types'; import { IsChainId, IsLowercasedEnum } from '@/common/validators'; import { @@ -47,6 +48,22 @@ export class GetReputationsQueryDto { @IsOptional() roles?: ReputationEntityType[]; + @ApiPropertyOptional({ + enum: [JobType], + name: 'job_request_types', + isArray: true, + }) + /** + * NOTE: Order of decorators here matters + * + * Query param is parsed as string if single value passed + * and as array if multiple + */ + @Transform(({ value }) => (Array.isArray(value) ? value : [value])) + @IsLowercasedEnum(JobType, { each: true }) + @IsOptional() + jobRequestTypes?: JobRequestType[]; + @ApiPropertyOptional({ name: 'order_by', enum: GetReputationQueryOrderBy, @@ -94,4 +111,10 @@ export class ReputationResponseDto { @ApiProperty({ enum: ReputationEntityType }) role: ReputationEntityType; + + @ApiProperty({ + enum: [JobType], + name: 'job_request_type', + }) + jobRequestType: JobRequestType; } diff --git a/packages/apps/reputation-oracle/server/src/modules/reputation/reputation.entity.ts b/packages/apps/reputation-oracle/server/src/modules/reputation/reputation.entity.ts index 4bae3b5a18..eeb2b5e1b0 100644 --- a/packages/apps/reputation-oracle/server/src/modules/reputation/reputation.entity.ts +++ b/packages/apps/reputation-oracle/server/src/modules/reputation/reputation.entity.ts @@ -1,12 +1,14 @@ import { Column, Entity, Index } from 'typeorm'; import { DATABASE_SCHEMA_NAME } from '@/common/constants'; +import { CvatJobType } from '@/common/enums'; +import type { JobRequestType } from '@/common/types'; import { BaseEntity } from '@/database'; import { ReputationEntityType } from './constants'; @Entity({ schema: DATABASE_SCHEMA_NAME, name: 'reputation' }) -@Index(['chainId', 'address', 'type'], { unique: true }) +@Index(['chainId', 'address', 'type', 'jobRequestType'], { unique: true }) export class ReputationEntity extends BaseEntity { @Column({ type: 'int' }) chainId: number; @@ -20,6 +22,9 @@ export class ReputationEntity extends BaseEntity { }) type: ReputationEntityType; + @Column({ type: 'varchar', default: CvatJobType.IMAGE_BOXES }) + jobRequestType: JobRequestType; + @Column({ type: 'int' }) reputationPoints: number; } diff --git a/packages/apps/reputation-oracle/server/src/modules/reputation/reputation.repository.ts b/packages/apps/reputation-oracle/server/src/modules/reputation/reputation.repository.ts index 8846cbca70..62efca678f 100644 --- a/packages/apps/reputation-oracle/server/src/modules/reputation/reputation.repository.ts +++ b/packages/apps/reputation-oracle/server/src/modules/reputation/reputation.repository.ts @@ -3,7 +3,9 @@ import { Injectable } from '@nestjs/common'; import { DataSource, FindManyOptions, In } from 'typeorm'; import { SortDirection } from '@/common/enums'; +import { JobRequestType } from '@/common/types'; import { BaseRepository } from '@/database'; +import { caseInsensitiveAddress } from '@/utils/database'; import { ReputationEntityType, ReputationOrderBy } from './constants'; import { ReputationEntity } from './reputation.entity'; @@ -19,12 +21,14 @@ export class ReputationRepository extends BaseRepository { chainId, address, type, + jobRequestType, }: ExclusiveReputationCriteria): Promise { return this.findOne({ where: { chainId, - address, + address: caseInsensitiveAddress(address), type, + jobRequestType, }, }); } @@ -33,6 +37,7 @@ export class ReputationRepository extends BaseRepository { filters: { address?: string; chainId?: ChainId; + jobRequestTypes?: JobRequestType[]; types?: ReputationEntityType[]; }, options?: { @@ -49,8 +54,11 @@ export class ReputationRepository extends BaseRepository { if (filters.types) { query.type = In(filters.types); } + if (filters.jobRequestTypes) { + query.jobRequestType = In(filters.jobRequestTypes); + } if (filters.address) { - query.address = filters.address; + query.address = caseInsensitiveAddress(filters.address); } return this.find({ diff --git a/packages/apps/reputation-oracle/server/src/modules/reputation/reputation.service.spec.ts b/packages/apps/reputation-oracle/server/src/modules/reputation/reputation.service.spec.ts index 5028487983..a65477911d 100644 --- a/packages/apps/reputation-oracle/server/src/modules/reputation/reputation.service.spec.ts +++ b/packages/apps/reputation-oracle/server/src/modules/reputation/reputation.service.spec.ts @@ -5,6 +5,7 @@ import { createMock } from '@golevelup/ts-jest'; import { EscrowClient } from '@human-protocol/sdk'; import { Test } from '@nestjs/testing'; +import { CvatJobType } from '@/common/enums'; import { ReputationConfigService, Web3ConfigService } from '@/config'; import { Web3Service } from '@/modules/web3'; import { @@ -90,18 +91,21 @@ describe('ReputationService', () => { chainId: withLowScore.chainId, address: withLowScore.address, role: withLowScore.type, + jobRequestType: withLowScore.jobRequestType, level: 'low', }); expect(reputations[1]).toEqual({ chainId: withMediumScore.chainId, address: withMediumScore.address, role: withMediumScore.type, + jobRequestType: withMediumScore.jobRequestType, level: 'medium', }); expect(reputations[2]).toEqual({ chainId: withHighScore.chainId, address: withHighScore.address, role: withHighScore.type, + jobRequestType: withHighScore.jobRequestType, level: 'high', }); }); @@ -129,6 +133,7 @@ describe('ReputationService', () => { chainId: generateTestnetChainId(), address: faker.finance.ethereumAddress(), type: generateReputationEntityType(), + jobRequestType: CvatJobType.IMAGE_BOXES, }, score, ), @@ -145,6 +150,7 @@ describe('ReputationService', () => { chainId: generateTestnetChainId(), address: faker.finance.ethereumAddress(), type: generateReputationEntityType(), + jobRequestType: CvatJobType.IMAGE_BOXES, }; const score = generateRandomScorePoints(); @@ -174,6 +180,7 @@ describe('ReputationService', () => { chainId: generateTestnetChainId(), address: mockWeb3ConfigService.operatorAddress, type: ReputationEntityType.REPUTATION_ORACLE, + jobRequestType: CvatJobType.IMAGE_BOXES, }; const score = generateRandomScorePoints(); @@ -206,6 +213,7 @@ describe('ReputationService', () => { chainId: reputationEntity.chainId, address: reputationEntity.address, type: reputationEntity.type, + jobRequestType: reputationEntity.jobRequestType, }; const score = generateRandomScorePoints(); const initialEntityScore = reputationEntity.reputationPoints; @@ -234,6 +242,7 @@ describe('ReputationService', () => { chainId: generateTestnetChainId(), address: faker.finance.ethereumAddress(), type: generateReputationEntityType(), + jobRequestType: CvatJobType.IMAGE_BOXES, }, score, ), @@ -253,6 +262,7 @@ describe('ReputationService', () => { chainId: generateTestnetChainId(), address: faker.finance.ethereumAddress(), type: generateReputationEntityType(), + jobRequestType: CvatJobType.IMAGE_BOXES, }; const score = generateRandomScorePoints(); @@ -285,6 +295,7 @@ describe('ReputationService', () => { chainId: reputationEntity.chainId, address: reputationEntity.address, type: reputationEntity.type, + jobRequestType: reputationEntity.jobRequestType, }; const score = generateRandomScorePoints(); const initialEntityScore = reputationEntity.reputationPoints; @@ -307,6 +318,7 @@ describe('ReputationService', () => { chainId: generateTestnetChainId(), address: mockWeb3ConfigService.operatorAddress, type: ReputationEntityType.REPUTATION_ORACLE, + jobRequestType: CvatJobType.IMAGE_BOXES, }; const score = generateRandomScorePoints(); @@ -356,6 +368,7 @@ describe('ReputationService', () => { jobLauncherAddress, exchangeOracleAddress, recordingOracleAddress, + CvatJobType.IMAGE_BOXES, ); expect(spyOnIncreaseReputation).toHaveBeenCalledTimes(4); @@ -364,6 +377,7 @@ describe('ReputationService', () => { chainId, address: jobLauncherAddress, type: ReputationEntityType.JOB_LAUNCHER, + jobRequestType: CvatJobType.IMAGE_BOXES, }, 1, ); @@ -372,6 +386,7 @@ describe('ReputationService', () => { chainId, address: exchangeOracleAddress, type: ReputationEntityType.EXCHANGE_ORACLE, + jobRequestType: CvatJobType.IMAGE_BOXES, }, 1, ); @@ -380,6 +395,7 @@ describe('ReputationService', () => { chainId, address: recordingOracleAddress, type: ReputationEntityType.RECORDING_ORACLE, + jobRequestType: CvatJobType.IMAGE_BOXES, }, 1, ); @@ -388,6 +404,7 @@ describe('ReputationService', () => { chainId, address: mockWeb3ConfigService.operatorAddress, type: ReputationEntityType.REPUTATION_ORACLE, + jobRequestType: CvatJobType.IMAGE_BOXES, }, 1, ); diff --git a/packages/apps/reputation-oracle/server/src/modules/reputation/reputation.service.ts b/packages/apps/reputation-oracle/server/src/modules/reputation/reputation.service.ts index 5da96b30b4..b598ff4521 100644 --- a/packages/apps/reputation-oracle/server/src/modules/reputation/reputation.service.ts +++ b/packages/apps/reputation-oracle/server/src/modules/reputation/reputation.service.ts @@ -2,6 +2,7 @@ import { ChainId } from '@human-protocol/sdk'; import { Injectable } from '@nestjs/common'; import { SortDirection } from '@/common/enums'; +import { JobRequestType } from '@/common/types'; import { ReputationConfigService, Web3ConfigService } from '@/config'; import { isDuplicatedError } from '@/database'; import { Web3Service } from '@/modules/web3'; @@ -53,12 +54,12 @@ export class ReputationService { * If the entity doesn't exist in the database - creates it first. */ async increaseReputation( - { chainId, address, type }: ExclusiveReputationCriteria, + { chainId, address, type, jobRequestType }: ExclusiveReputationCriteria, points: number, ): Promise { assertAdjustableReputationPoints(points); - const searchCriteria = { chainId, address, type }; + const searchCriteria = { chainId, address, type, jobRequestType }; let existingEntity = await this.reputationRepository.findExclusive(searchCriteria); @@ -76,6 +77,7 @@ export class ReputationService { reputationEntity.chainId = chainId; reputationEntity.address = address; reputationEntity.type = type; + reputationEntity.jobRequestType = jobRequestType; reputationEntity.reputationPoints = initialReputation; try { @@ -104,7 +106,7 @@ export class ReputationService { * If the entity doesn't exist in the database - creates it first. */ async decreaseReputation( - { chainId, address, type }: ExclusiveReputationCriteria, + { chainId, address, type, jobRequestType }: ExclusiveReputationCriteria, points: number, ): Promise { assertAdjustableReputationPoints(points); @@ -116,7 +118,7 @@ export class ReputationService { return; } - const searchCriteria = { chainId, address, type }; + const searchCriteria = { chainId, address, type, jobRequestType }; let existingEntity = await this.reputationRepository.findExclusive(searchCriteria); @@ -126,6 +128,7 @@ export class ReputationService { reputationEntity.chainId = chainId; reputationEntity.address = address; reputationEntity.type = type; + reputationEntity.jobRequestType = jobRequestType; reputationEntity.reputationPoints = INITIAL_REPUTATION; try { @@ -157,6 +160,7 @@ export class ReputationService { filter: { address?: string; chainId?: ChainId; + jobRequestTypes?: JobRequestType[]; types?: ReputationEntityType[]; }, options?: { @@ -175,6 +179,7 @@ export class ReputationService { chainId: reputation.chainId, address: reputation.address, role: reputation.type, + jobRequestType: reputation.jobRequestType, level: this.getReputationLevel(reputation.reputationPoints), })); } @@ -184,6 +189,7 @@ export class ReputationService { jobLauncherAddress: string, exchangeOracleAddress: string, recordingOracleAddress: string, + jobRequestType: JobRequestType, ): Promise { const reputationTypeToAddress = new Map([ [ReputationEntityType.JOB_LAUNCHER, jobLauncherAddress], @@ -200,7 +206,7 @@ export class ReputationService { address, ] of reputationTypeToAddress.entries()) { await this.increaseReputation( - { chainId, address, type: reputationEntityType }, + { chainId, address, type: reputationEntityType, jobRequestType }, 1, ); } diff --git a/packages/apps/reputation-oracle/server/src/modules/reputation/types.ts b/packages/apps/reputation-oracle/server/src/modules/reputation/types.ts index b40311a6e3..f8a12d7740 100644 --- a/packages/apps/reputation-oracle/server/src/modules/reputation/types.ts +++ b/packages/apps/reputation-oracle/server/src/modules/reputation/types.ts @@ -1,5 +1,7 @@ import { ChainId } from '@human-protocol/sdk'; +import { JobRequestType } from '@/common/types'; + import { ReputationEntityType, ReputationLevel } from './constants'; export type ReputationData = { @@ -7,10 +9,12 @@ export type ReputationData = { address: string; level: ReputationLevel; role: ReputationEntityType; + jobRequestType: JobRequestType; }; export type ExclusiveReputationCriteria = { chainId: number; address: string; type: ReputationEntityType; + jobRequestType: JobRequestType; }; diff --git a/packages/apps/reputation-oracle/server/src/utils/database.ts b/packages/apps/reputation-oracle/server/src/utils/database.ts new file mode 100644 index 0000000000..5edc9a59a6 --- /dev/null +++ b/packages/apps/reputation-oracle/server/src/utils/database.ts @@ -0,0 +1,7 @@ +import { Raw } from 'typeorm'; + +export function caseInsensitiveAddress(address: string) { + return Raw((addressAlias) => `LOWER(${addressAlias}) = LOWER(:address)`, { + address, + }); +}