diff --git a/packages/apps/reputation-oracle/server/src/modules/escrow-completion/escrow-completion.service.spec.ts b/packages/apps/reputation-oracle/server/src/modules/escrow-completion/escrow-completion.service.spec.ts index e63eece4b1..4b599be2bf 100644 --- a/packages/apps/reputation-oracle/server/src/modules/escrow-completion/escrow-completion.service.spec.ts +++ b/packages/apps/reputation-oracle/server/src/modules/escrow-completion/escrow-completion.service.spec.ts @@ -326,9 +326,10 @@ describe('EscrowCompletionService', () => { } as unknown as IEscrow); const fortuneManifest = generateFortuneManifest(); - mockStorageService.downloadJsonLikeData.mockResolvedValueOnce( - fortuneManifest, - ); + mockStorageService.downloadManifest.mockResolvedValueOnce({ + manifest: fortuneManifest, + encrypted: true, + }); const finalResultsUrl = faker.internet.url(); const finalResultsHash = faker.string.hexadecimal({ length: 42 }); mockFortuneResultsProcessor.storeResults.mockResolvedValueOnce({ @@ -351,7 +352,7 @@ describe('EscrowCompletionService', () => { pendingRecord.chainId, pendingRecord.escrowAddress, ); - expect(mockStorageService.downloadJsonLikeData).toHaveBeenCalledWith( + expect(mockStorageService.downloadManifest).toHaveBeenCalledWith( manifestUrl, ); expect(mockFortuneResultsProcessor.storeResults).toHaveBeenCalledTimes(1); @@ -359,6 +360,7 @@ describe('EscrowCompletionService', () => { pendingRecord.chainId, pendingRecord.escrowAddress, fortuneManifest, + true, ); expect(mockEscrowCompletionRepository.updateOne).toHaveBeenCalledWith( expect.objectContaining({ @@ -407,9 +409,10 @@ describe('EscrowCompletionService', () => { mockedEscrowUtils.getEscrow.mockResolvedValueOnce( {} as unknown as IEscrow, ); - mockStorageService.downloadJsonLikeData.mockResolvedValueOnce( - generateFortuneManifest(), - ); + mockStorageService.downloadManifest.mockResolvedValueOnce({ + manifest: generateFortuneManifest(), + encrypted: false, + }); const firstAddressPayout = { address: `0x1${faker.finance.ethereumAddress().slice(3)}`, diff --git a/packages/apps/reputation-oracle/server/src/modules/escrow-completion/escrow-completion.service.ts b/packages/apps/reputation-oracle/server/src/modules/escrow-completion/escrow-completion.service.ts index 46755dd3d1..61f0f5fbeb 100644 --- a/packages/apps/reputation-oracle/server/src/modules/escrow-completion/escrow-completion.service.ts +++ b/packages/apps/reputation-oracle/server/src/modules/escrow-completion/escrow-completion.service.ts @@ -15,7 +15,7 @@ import _ from 'lodash'; import { v4 as uuidv4 } from 'uuid'; import { BACKOFF_INTERVAL_SECONDS } from '@/common/constants'; -import { JobManifest, JobRequestType } from '@/common/types'; +import { JobRequestType } from '@/common/types'; import { ServerConfigService, Web3ConfigService } from '@/config'; import { isDuplicatedError } from '@/database'; import logger from '@/logger'; @@ -137,8 +137,8 @@ export class EscrowCompletionService { throw new Error('Escrow data is missing'); } - const manifest = - await this.storageService.downloadJsonLikeData( + const { manifest, encrypted: isManifestEncrypted } = + await this.storageService.downloadManifest( escrowData.manifest as string, ); const jobRequestType = manifestUtils.getJobRequestType(manifest); @@ -151,6 +151,7 @@ export class EscrowCompletionService { escrowCompletionEntity.chainId, escrowCompletionEntity.escrowAddress, manifest, + isManifestEncrypted, ); escrowCompletionEntity.finalResultsUrl = url; diff --git a/packages/apps/reputation-oracle/server/src/modules/escrow-completion/results-processing/escrow-results-processor.spec.ts b/packages/apps/reputation-oracle/server/src/modules/escrow-completion/results-processing/escrow-results-processor.spec.ts index dd0d332cc5..93c7fc701a 100644 --- a/packages/apps/reputation-oracle/server/src/modules/escrow-completion/results-processing/escrow-results-processor.spec.ts +++ b/packages/apps/reputation-oracle/server/src/modules/escrow-completion/results-processing/escrow-results-processor.spec.ts @@ -213,6 +213,60 @@ describe('BaseEscrowResultsProcessor', () => { 'text/plain', ); }); + + it('should store unencrypted results when encryption is disabled for results', async () => { + const chainId = generateTestnetChainId(); + const escrowAddress = faker.finance.ethereumAddress(); + + const baseResultsUrl = faker.internet.url(); + mockedGetIntermediateResultsUrl.mockResolvedValueOnce(baseResultsUrl); + + const resultsUrl = `${baseResultsUrl}/${faker.system.fileName()}`; + processor.constructIntermediateResultsUrl.mockReturnValueOnce(resultsUrl); + + const resultsFileContent = Buffer.from(faker.lorem.sentence()); + mockedStorageService.downloadFile.mockResolvedValueOnce( + resultsFileContent, + ); + + processor.assertResultsComplete.mockResolvedValueOnce(undefined); + + mockedEscrowUtils.getEscrow.mockResolvedValueOnce({ + launcher: faker.finance.ethereumAddress(), + status: EscrowStatus[EscrowStatus.Launched], + } as unknown as IEscrow); + + const resultsHash = crypto + .createHash('sha256') + .update(resultsFileContent) + .digest('hex'); + const storedResultsFileName = `${resultsHash}.${faker.system.fileExt()}`; + processor.getFinalResultsFileName.mockReturnValueOnce( + storedResultsFileName, + ); + + const storedResultsUrl = faker.internet.url(); + mockedStorageService.uploadData.mockResolvedValueOnce(storedResultsUrl); + + const manifest = {} as JobManifest; + const storedResultMeta = await processor.storeResults( + chainId, + escrowAddress, + manifest, + false, + ); + + expect(storedResultMeta.url).toBe(storedResultsUrl); + expect(storedResultMeta.hash).toBe(resultsHash); + + expect(mockedPgpEncryptionService.encrypt).not.toHaveBeenCalled(); + expect(mockedStorageService.uploadData).toHaveBeenCalledWith( + resultsFileContent, + storedResultsFileName, + 'text/plain', + ); + }); + it('should NOT call assertResultsComplete if status is ToCancel', async () => { /** ARRANGE */ const chainId = generateTestnetChainId(); diff --git a/packages/apps/reputation-oracle/server/src/modules/escrow-completion/results-processing/escrow-results-processor.ts b/packages/apps/reputation-oracle/server/src/modules/escrow-completion/results-processing/escrow-results-processor.ts index f0d5ece5dd..3d37f6eb46 100644 --- a/packages/apps/reputation-oracle/server/src/modules/escrow-completion/results-processing/escrow-results-processor.ts +++ b/packages/apps/reputation-oracle/server/src/modules/escrow-completion/results-processing/escrow-results-processor.ts @@ -24,6 +24,7 @@ export interface EscrowResultsProcessor { chainId: ChainId, escrowAddress: string, manifest: JobManifest, + encryptResults?: boolean, ): Promise; } @@ -41,6 +42,7 @@ export abstract class BaseEscrowResultsProcessor< chainId: ChainId, escrowAddress: string, manifest: TManifest, + encryptResults = true, ): Promise { const signer = this.web3Service.getSigner(chainId); const escrowClient = await EscrowClient.build(signer); @@ -66,21 +68,18 @@ export abstract class BaseEscrowResultsProcessor< await this.assertResultsComplete(fileContent, manifest); } - const encryptedResults = await this.pgpEncryptionService.encrypt( - fileContent, - chainId, - [escrowData.launcher as string], - ); + const finalResults = encryptResults + ? await this.pgpEncryptionService.encrypt(fileContent, chainId, [ + escrowData.launcher as string, + ]) + : fileContent; - const hash = crypto - .createHash('sha256') - .update(encryptedResults) - .digest('hex'); + const hash = crypto.createHash('sha256').update(finalResults).digest('hex'); const fileName = this.getFinalResultsFileName(hash); const url = await this.storageService.uploadData( - encryptedResults, + finalResults, fileName, ContentType.PLAIN_TEXT, ); diff --git a/packages/apps/reputation-oracle/server/src/modules/storage/storage.service.spec.ts b/packages/apps/reputation-oracle/server/src/modules/storage/storage.service.spec.ts index 275cd853d5..1a1a08d580 100644 --- a/packages/apps/reputation-oracle/server/src/modules/storage/storage.service.spec.ts +++ b/packages/apps/reputation-oracle/server/src/modules/storage/storage.service.spec.ts @@ -248,4 +248,51 @@ describe('StorageService', () => { expect(downloadedData).toEqual(data); }); }); + + describe('downloadManifest', () => { + const EXPECTED_DOWNLOAD_ERROR_MESSAGE = 'Error downloading manifest'; + let spyOnDownloadFile: jest.SpyInstance; + + beforeAll(() => { + spyOnDownloadFile = jest.spyOn(httpUtils, 'downloadFile'); + spyOnDownloadFile.mockImplementation(); + }); + + afterAll(() => { + spyOnDownloadFile.mockRestore(); + }); + + it('should throw custom error when fails to load manifest', async () => { + spyOnDownloadFile.mockRejectedValueOnce(new Error(faker.lorem.word())); + + await expect( + storageService.downloadManifest(faker.internet.url()), + ).rejects.toThrow(EXPECTED_DOWNLOAD_ERROR_MESSAGE); + }); + + it('should download manifest with encryption state', async () => { + const manifest = { + requestType: faker.string.sample(), + }; + + const fileUrl = faker.internet.url(); + spyOnDownloadFile.mockImplementation(async (url) => { + if (url === fileUrl) { + return Buffer.from(JSON.stringify(manifest)); + } + + throw new Error('File not found'); + }); + mockedPgpEncryptionService.maybeDecryptFile.mockImplementationOnce( + async (c) => c, + ); + + const downloadedManifest = await storageService.downloadManifest(fileUrl); + + expect(downloadedManifest).toEqual({ + manifest, + encrypted: false, + }); + }); + }); }); diff --git a/packages/apps/reputation-oracle/server/src/modules/storage/storage.service.ts b/packages/apps/reputation-oracle/server/src/modules/storage/storage.service.ts index 6103f45ee0..771dcdb08f 100644 --- a/packages/apps/reputation-oracle/server/src/modules/storage/storage.service.ts +++ b/packages/apps/reputation-oracle/server/src/modules/storage/storage.service.ts @@ -1,7 +1,9 @@ +import { EncryptionUtils } from '@human-protocol/sdk'; import { Injectable } from '@nestjs/common'; import * as Minio from 'minio'; import { ContentType } from '@/common/enums'; +import { JobManifest } from '@/common/types'; import { S3ConfigService } from '@/config'; import logger from '@/logger'; import { PgpEncryptionService } from '@/modules/encryption'; @@ -83,6 +85,30 @@ export class StorageService { } } + async downloadManifest( + url: string, + ): Promise<{ manifest: JobManifest; encrypted: boolean }> { + try { + let fileContent = await httpUtils.downloadFile(url); + const encrypted = EncryptionUtils.isEncrypted(fileContent.toString()); + + fileContent = + await this.pgpEncryptionService.maybeDecryptFile(fileContent); + + return { + manifest: JSON.parse(fileContent.toString()) as JobManifest, + encrypted, + }; + } catch (error) { + const errorMessage = 'Error downloading manifest'; + this.logger.error(errorMessage, { + error, + url, + }); + throw new Error(errorMessage); + } + } + async uploadData( content: string | Buffer, fileName: string,