diff --git a/README.md b/README.md index d338cf6..42ef866 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,6 @@ # deepevents.ai deepevents.ai main codebase + +## Scientific bounty modules + +- [Scientific bounty sponsor scorecard](scientific-bounty-sponsor-scorecard/README.md) - solver-facing sponsor reliability scoring for funding proof, review responsiveness, rubric clarity, payout history, dispute handling, amendment volatility, and IP/NDA readiness. diff --git a/scientific-bounty-sponsor-scorecard/README.md b/scientific-bounty-sponsor-scorecard/README.md new file mode 100644 index 0000000..080e8a8 --- /dev/null +++ b/scientific-bounty-sponsor-scorecard/README.md @@ -0,0 +1,54 @@ +# Scientific Bounty Sponsor Scorecard + +This module adds a solver-facing reliability layer for the Scientific Bounty System. It helps researchers decide whether a posted bounty is ready to accept before they invest time, while giving sponsors concrete actions to improve trust. + +The scorecard is intentionally different from intake, arbitration, appeals, escrow settlement, reproducibility audit, anti-collusion, workspace privacy, and amendment-control modules. It looks at sponsor reliability and challenge readiness before solver commitment. + +## What It Evaluates + +- Verified funding or escrow coverage for the current prize +- Review SLA and historical sponsor responsiveness +- Rubric completeness, measurable criteria, tie-break handling, and evidence expectations +- Amendment volatility after launch and whether solvers receive material-change protections +- Payout history, average payout lag, and paid-award evidence +- Dispute responsiveness and unresolved dispute exposure +- IP/NDA clarity, including whether unpaid work stays with the solver +- Public-safe audit digest that excludes private sponsor notes and payment/KYC fields + +## Run Locally + +```bash +npm run check +npm test +npm run demo +``` + +## API + +```js +import { buildSponsorScorecard } from "./src/sponsor-scorecard.js"; +import sample from "./data/sample-sponsor-input.json" with { type: "json" }; + +const report = buildSponsorScorecard(sample, { + generatedAt: "2026-05-16T15:30:00.000Z" +}); + +console.log(report.summary); +``` + +## Outputs + +- `summary`: portfolio counts and tier distribution +- `scorecards`: per-challenge score, tier, axis breakdown, findings, sponsor actions, solver guidance +- `auditDigest`: deterministic SHA-256 digest over public-safe report fields +- `sanitizedInputEcho`: redacted snapshot proving private notes and payment/KYC fields were not exported + +## Demo Artifacts + +- `docs/demo.svg` +- `docs/demo.mp4` +- `docs/demo.gif` + +## Design Notes + +The scoring axes follow established challenge-prize guidance: evaluation criteria should be measurable, judging protocols should be transparent, conflicts/NDA terms should be managed up front, and participants should understand award and eligibility conditions before doing work. See `docs/requirement-map.md` for the mapping to issue #18 and the public references reviewed. diff --git a/scientific-bounty-sponsor-scorecard/data/sample-sponsor-input.json b/scientific-bounty-sponsor-scorecard/data/sample-sponsor-input.json new file mode 100644 index 0000000..5a4485a --- /dev/null +++ b/scientific-bounty-sponsor-scorecard/data/sample-sponsor-input.json @@ -0,0 +1,195 @@ +{ + "portfolioId": "scientific-bounty-sponsor-scorecard-demo", + "asOf": "2026-05-16T15:30:00.000Z", + "sponsors": [ + { + "sponsorId": "helix-biofund", + "displayName": "Helix Biofund", + "sector": "biotech", + "privateNotes": "Finance lead phone number and internal budget notes are intentionally private.", + "bankAccount": "do-not-export", + "taxId": "do-not-export", + "history": { + "completedChallenges": 9, + "cancelledAfterStart": 0, + "paidAwards": 12, + "averagePayoutLagDays": 3, + "medianFirstResponseHours": 12, + "onTimeReviewRate": 0.94, + "materialAmendmentsAfterLaunch": 1, + "disputesOpened": 1, + "disputesResolvedWithinSla": 1, + "participantFeedbackAverage": 4.8 + }, + "challengePosts": [ + { + "challengeId": "single-cell-biomarker-2026", + "title": "Identify single-cell biomarker candidates", + "prizeUsd": 18000, + "visibility": "private", + "funding": { + "status": "escrow_verified", + "coverageRatio": 1, + "verifiedAt": "2026-05-12T09:00:00.000Z" + }, + "reviewSlaDays": 7, + "expectedFirstResponseHours": 24, + "rubric": { + "criteria": [ + { "name": "biological plausibility", "weight": 35, "measurable": true }, + { "name": "model reproducibility", "weight": 30, "measurable": true }, + { "name": "clinical literature support", "weight": 20, "measurable": true }, + { "name": "documentation quality", "weight": 15, "measurable": true } + ], + "evidenceRequirements": ["notebook", "dataset manifest", "model card"], + "tieBreakerDefined": true, + "judgingProtocolPublished": true + }, + "timeline": { + "milestones": 3, + "submissionWindowDays": 42, + "sponsorReviewWindowDays": 7 + }, + "amendmentPolicy": { + "materialChangeNoticeDays": 5, + "solverWithdrawalProtected": true, + "maxMaterialAmendments": 1 + }, + "ipTerms": { + "solverRetainsUntilPaid": true, + "licenseOnPayout": "exclusive field-limited license", + "ndaTemplatePublished": true + }, + "disputePolicy": { + "responseSlaDays": 5, + "neutralReviewerAvailable": true + } + } + ] + }, + { + "sponsorId": "aster-climate-lab", + "displayName": "Aster Climate Lab", + "sector": "climate", + "internalContactEmail": "private@example.invalid", + "identityDocuments": ["passport-redacted-placeholder"], + "history": { + "completedChallenges": 4, + "cancelledAfterStart": 1, + "paidAwards": 5, + "averagePayoutLagDays": 14, + "medianFirstResponseHours": 38, + "onTimeReviewRate": 0.72, + "materialAmendmentsAfterLaunch": 3, + "disputesOpened": 2, + "disputesResolvedWithinSla": 1, + "participantFeedbackAverage": 3.9 + }, + "challengePosts": [ + { + "challengeId": "regional-forecasting-ensemble", + "title": "Regional forecasting ensemble for flood alerts", + "prizeUsd": 9000, + "visibility": "public", + "funding": { + "status": "escrow_verified", + "coverageRatio": 0.85, + "verifiedAt": "2026-05-13T12:00:00.000Z" + }, + "reviewSlaDays": 14, + "expectedFirstResponseHours": 48, + "rubric": { + "criteria": [ + { "name": "forecast skill", "weight": 45, "measurable": true }, + { "name": "regional transferability", "weight": 30, "measurable": true }, + { "name": "operator handoff", "weight": 20, "measurable": true } + ], + "evidenceRequirements": ["validation report", "deployment notes"], + "tieBreakerDefined": false, + "judgingProtocolPublished": true + }, + "timeline": { + "milestones": 2, + "submissionWindowDays": 28, + "sponsorReviewWindowDays": 14 + }, + "amendmentPolicy": { + "materialChangeNoticeDays": 2, + "solverWithdrawalProtected": true, + "maxMaterialAmendments": 2 + }, + "ipTerms": { + "solverRetainsUntilPaid": true, + "licenseOnPayout": "non-exclusive implementation license", + "ndaTemplatePublished": false + }, + "disputePolicy": { + "responseSlaDays": 10, + "neutralReviewerAvailable": true + } + } + ] + }, + { + "sponsorId": "quantum-north", + "displayName": "Quantum North", + "sector": "quantum", + "privateNotes": "Do not show draft acquisition notes to solvers.", + "history": { + "completedChallenges": 1, + "cancelledAfterStart": 2, + "paidAwards": 1, + "averagePayoutLagDays": 35, + "medianFirstResponseHours": 96, + "onTimeReviewRate": 0.35, + "materialAmendmentsAfterLaunch": 6, + "disputesOpened": 3, + "disputesResolvedWithinSla": 0, + "participantFeedbackAverage": 2.7 + }, + "challengePosts": [ + { + "challengeId": "noise-reduction-kernel", + "title": "Improve quantum noise-reduction kernel", + "prizeUsd": 12000, + "visibility": "private", + "funding": { + "status": "missing", + "coverageRatio": 0, + "verifiedAt": null + }, + "reviewSlaDays": null, + "expectedFirstResponseHours": 120, + "rubric": { + "criteria": [ + { "name": "accuracy", "weight": 50, "measurable": false }, + { "name": "speed", "weight": 25, "measurable": true } + ], + "evidenceRequirements": [], + "tieBreakerDefined": false, + "judgingProtocolPublished": false + }, + "timeline": { + "milestones": 1, + "submissionWindowDays": 14, + "sponsorReviewWindowDays": null + }, + "amendmentPolicy": { + "materialChangeNoticeDays": 0, + "solverWithdrawalProtected": false, + "maxMaterialAmendments": 4 + }, + "ipTerms": { + "solverRetainsUntilPaid": false, + "licenseOnPayout": "assignment on submission", + "ndaTemplatePublished": false + }, + "disputePolicy": { + "responseSlaDays": null, + "neutralReviewerAvailable": false + } + } + ] + } + ] +} diff --git a/scientific-bounty-sponsor-scorecard/docs/demo.gif b/scientific-bounty-sponsor-scorecard/docs/demo.gif new file mode 100644 index 0000000..d9e6a3c Binary files /dev/null and b/scientific-bounty-sponsor-scorecard/docs/demo.gif differ diff --git a/scientific-bounty-sponsor-scorecard/docs/demo.mp4 b/scientific-bounty-sponsor-scorecard/docs/demo.mp4 new file mode 100644 index 0000000..3983565 Binary files /dev/null and b/scientific-bounty-sponsor-scorecard/docs/demo.mp4 differ diff --git a/scientific-bounty-sponsor-scorecard/docs/demo.svg b/scientific-bounty-sponsor-scorecard/docs/demo.svg new file mode 100644 index 0000000..5140f90 --- /dev/null +++ b/scientific-bounty-sponsor-scorecard/docs/demo.svg @@ -0,0 +1,45 @@ + + Scientific bounty sponsor scorecard demo + Dashboard preview with trusted, watch, and hold sponsor reliability tiers. + + + Sponsor Reliability Scorecard + Solver-facing readiness before research teams commit effort. + + + Helix Biofund + + 91 + TRUSTED + Escrow verified + 7 day review SLA + Clear IP and NDA terms + + + + + + Aster Climate Lab + + 67 + WATCH + Rubric weights need tie-break + Payout lag above target + Publish sponsor actions + + + + + + Quantum North + + 41 + HOLD + No funding proof + Unclear IP transfer trigger + Unresolved dispute exposure + + + + Public-safe digest: 31d5... - private notes, tax IDs, bank fields, and identity documents redacted. + diff --git a/scientific-bounty-sponsor-scorecard/docs/requirement-map.md b/scientific-bounty-sponsor-scorecard/docs/requirement-map.md new file mode 100644 index 0000000..a355b14 --- /dev/null +++ b/scientific-bounty-sponsor-scorecard/docs/requirement-map.md @@ -0,0 +1,33 @@ +# Requirement Map + +Issue #18 describes a global research marketplace that needs sponsor challenge posting, secure participation, evaluation, arbitration, reward distribution, and IP options. This module adds a pre-commitment trust layer for solvers. + +## Mapping To Issue #18 + +| Issue #18 area | Scorecard coverage | +| --- | --- | +| Challenge posting portal | Checks prize funding proof, timelines, rubric completeness, milestone schedule, NDA status, and IP term clarity before a challenge is marked solver-ready. | +| Evaluation criteria and scoring rubric | Scores whether criteria are measurable, weighted, evidence-backed, and include tie-break protocols. | +| Timeline and milestone deadlines | Flags missing review SLAs, unrealistic timelines, and absent sponsor-response windows. | +| Public vs private challenges and NDA support | Rates whether private challenges provide clear NDA scope and non-sensitive public summaries. | +| Submission engine trust | Gives solvers a readiness signal before creating a secure workspace or uploading private work. | +| Arbitration and reward distribution | Uses sponsor payout and dispute history as pre-commitment risk signals, without duplicating arbitration or escrow-settlement logic. | +| IP management options | Blocks or warns when IP transfer can occur before payment or when terms are missing. | + +## Non-Overlap + +This is not another intake gate, rubric scorer, arbitration ledger, appeals ledger, escrow settlement ledger, reproducibility audit, workspace privacy gate, anti-collusion module, or amendment-control engine. It sits before solver commitment and answers: "Is this sponsor and challenge reliable enough to spend research time on?" + +## References Reviewed + +- GSA Prize and Challenge Toolkit: emphasizes measurable evaluation criteria, transparent judging protocols, conflict/NDA management, and clear prize procedures. +- GSA prize competitions overview: distinguishes prize competitions from contracts/grants and describes the announce, submit, review, and award flow. +- Nesta Challenge Prizes practice guide: highlights careful prize construction, incentives, support, and attracting the right talent. + +## Acceptance Signals + +- Deterministic scorecards with no external service dependency. +- Clear trust tiers: `trusted`, `watch`, `hold`. +- Sponsor action list that can move a challenge toward solver readiness. +- Public-safe audit digest that redacts sensitive sponsor notes and payment/KYC fields. +- Tests for trusted, watch, and hold paths plus digest determinism and privacy redaction. diff --git a/scientific-bounty-sponsor-scorecard/package.json b/scientific-bounty-sponsor-scorecard/package.json new file mode 100644 index 0000000..2df8097 --- /dev/null +++ b/scientific-bounty-sponsor-scorecard/package.json @@ -0,0 +1,16 @@ +{ + "name": "scientific-bounty-sponsor-scorecard", + "version": "0.1.0", + "private": true, + "type": "module", + "description": "Solver-facing sponsor reliability scorecards for scientific bounty challenges.", + "scripts": { + "check": "node --check src/sponsor-scorecard.js && node --check scripts/demo.js && node --check test/sponsor-scorecard.test.js", + "test": "node --test", + "demo": "node scripts/demo.js" + }, + "engines": { + "node": ">=20" + }, + "license": "MIT" +} diff --git a/scientific-bounty-sponsor-scorecard/scripts/demo.js b/scientific-bounty-sponsor-scorecard/scripts/demo.js new file mode 100644 index 0000000..df08e21 --- /dev/null +++ b/scientific-bounty-sponsor-scorecard/scripts/demo.js @@ -0,0 +1,28 @@ +import { readFile } from "node:fs/promises"; +import { fileURLToPath } from "node:url"; +import { dirname, join } from "node:path"; +import { buildSponsorScorecard } from "../src/sponsor-scorecard.js"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const samplePath = join(__dirname, "..", "data", "sample-sponsor-input.json"); +const sample = JSON.parse(await readFile(samplePath, "utf8")); + +const report = buildSponsorScorecard(sample, { + generatedAt: sample.asOf +}); + +console.log("Scientific Bounty Sponsor Scorecard"); +console.log(`Portfolio: ${report.portfolioId}`); +console.log(`Average score: ${report.summary.averageScore}`); +console.log(`Tier distribution: ${JSON.stringify(report.summary.byTier)}`); +console.log(`Audit digest: ${report.auditDigest}`); + +for (const card of report.scorecards) { + console.log(""); + console.log(`${card.tier.toUpperCase()} ${card.score} - ${card.sponsorName}: ${card.challengeTitle}`); + console.log(`Solver guidance: ${card.solverGuidance}`); + console.log("Sponsor actions:"); + for (const item of card.sponsorActions.slice(0, 4)) { + console.log(`- ${item}`); + } +} diff --git a/scientific-bounty-sponsor-scorecard/src/sponsor-scorecard.js b/scientific-bounty-sponsor-scorecard/src/sponsor-scorecard.js new file mode 100644 index 0000000..09e6c16 --- /dev/null +++ b/scientific-bounty-sponsor-scorecard/src/sponsor-scorecard.js @@ -0,0 +1,502 @@ +import { createHash } from "node:crypto"; + +export const DEFAULT_WEIGHTS = Object.freeze({ + fundingProof: 18, + responsiveness: 14, + rubricClarity: 15, + timelineReadiness: 10, + amendmentStability: 12, + payoutHistory: 13, + disputeResponsiveness: 8, + ipNdaClarity: 10 +}); + +const SENSITIVE_KEYS = new Set([ + "bankAccount", + "identityDocuments", + "internalContactEmail", + "privateNotes", + "taxId" +]); + +const TIER_ORDER = ["trusted", "watch", "hold"]; + +export function buildSponsorScorecard(input, options = {}) { + const generatedAt = options.generatedAt ?? new Date().toISOString(); + const sponsors = Array.isArray(input?.sponsors) ? input.sponsors : []; + + const scorecards = sponsors.flatMap((sponsor) => { + const posts = Array.isArray(sponsor.challengePosts) ? sponsor.challengePosts : []; + return posts.map((challenge) => evaluateChallenge(sponsor, challenge)); + }); + + const summary = summarizeScorecards(scorecards); + const sanitizedInputEcho = redactSensitiveFields(input); + const digestPayload = { + portfolioId: input?.portfolioId ?? null, + asOf: input?.asOf ?? null, + summary, + scorecards: scorecards.map(toDigestableScorecard), + sanitizedInputEcho + }; + + return { + generatedAt, + portfolioId: input?.portfolioId ?? "unknown-portfolio", + summary, + scorecards, + sanitizedInputEcho, + auditDigest: sha256(stableStringify(digestPayload)) + }; +} + +export function evaluateChallenge(sponsor, challenge) { + const history = sponsor.history ?? {}; + const axisResults = { + fundingProof: scoreFunding(challenge.funding), + responsiveness: scoreResponsiveness(history, challenge), + rubricClarity: scoreRubric(challenge.rubric), + timelineReadiness: scoreTimeline(challenge.timeline, challenge.reviewSlaDays), + amendmentStability: scoreAmendments(history, challenge.amendmentPolicy), + payoutHistory: scorePayoutHistory(history), + disputeResponsiveness: scoreDisputes(history, challenge.disputePolicy), + ipNdaClarity: scoreIpNda(challenge.ipTerms, challenge.visibility) + }; + + const weightedScore = weightedAverage(axisResults, DEFAULT_WEIGHTS); + const score = Math.round(weightedScore); + const tier = tierForScore(score, axisResults); + const findings = buildFindings(axisResults); + const sponsorActions = buildSponsorActions(axisResults, tier); + + return { + sponsorId: sponsor.sponsorId, + sponsorName: sponsor.displayName ?? sponsor.sponsorId, + sector: sponsor.sector ?? "unspecified", + challengeId: challenge.challengeId, + challengeTitle: challenge.title, + prizeUsd: challenge.prizeUsd ?? null, + score, + tier, + axisResults, + findings, + sponsorActions, + solverGuidance: guidanceForTier(tier, findings), + publicEvidenceDigest: sha256(stableStringify({ + sponsorId: sponsor.sponsorId, + challengeId: challenge.challengeId, + score, + tier, + axisResults: compactAxisResults(axisResults) + })) + }; +} + +export function redactSensitiveFields(value) { + if (Array.isArray(value)) { + return value.map(redactSensitiveFields); + } + + if (!value || typeof value !== "object") { + return value; + } + + return Object.fromEntries( + Object.entries(value).map(([key, entry]) => [ + key, + SENSITIVE_KEYS.has(key) ? "[redacted]" : redactSensitiveFields(entry) + ]) + ); +} + +export function stableStringify(value) { + if (Array.isArray(value)) { + return `[${value.map(stableStringify).join(",")}]`; + } + + if (value && typeof value === "object") { + return `{${Object.keys(value) + .sort() + .map((key) => `${JSON.stringify(key)}:${stableStringify(value[key])}`) + .join(",")}}`; + } + + return JSON.stringify(value); +} + +function scoreFunding(funding = {}) { + const status = funding.status ?? "missing"; + const coverageRatio = clampNumber(funding.coverageRatio, 0, 1); + const verified = status === "escrow_verified" && coverageRatio >= 1; + + if (verified) { + return axis(100, "Funding proof", "Escrow or verified prize coverage is complete.", []); + } + + if (status === "escrow_verified" && coverageRatio >= 0.75) { + return axis(75, "Funding proof", "Funding is verified but coverage is below the full prize amount.", [ + action("warning", "Top up escrow or lower the published prize before accepting solvers.") + ]); + } + + if (status === "pledged") { + return axis(48 + Math.round(coverageRatio * 18), "Funding proof", "Prize is pledged but not yet verified.", [ + action("blocker", "Verify funding or escrow coverage before routing this to solvers.") + ]); + } + + return axis(15, "Funding proof", "No usable funding proof is attached to the challenge.", [ + action("blocker", "Add verified funding proof before solvers begin work.") + ]); +} + +function scoreResponsiveness(history = {}, challenge = {}) { + const medianFirstResponseHours = nullableNumber(history.medianFirstResponseHours); + const expectedFirstResponseHours = nullableNumber(challenge.expectedFirstResponseHours); + const onTimeReviewRate = clampNumber(history.onTimeReviewRate, 0, 1); + const reviewSlaDays = nullableNumber(challenge.reviewSlaDays); + + let score = 100; + if (medianFirstResponseHours == null || medianFirstResponseHours > 72) score -= 35; + else if (medianFirstResponseHours > 48) score -= 24; + else if (medianFirstResponseHours > 24) score -= 12; + + if (expectedFirstResponseHours == null || expectedFirstResponseHours > 72) score -= 20; + else if (expectedFirstResponseHours > 48) score -= 12; + + if (onTimeReviewRate < 0.5) score -= 30; + else if (onTimeReviewRate < 0.75) score -= 18; + else if (onTimeReviewRate < 0.9) score -= 8; + + if (reviewSlaDays == null || reviewSlaDays > 21) score -= 20; + else if (reviewSlaDays > 14) score -= 10; + else if (reviewSlaDays > 7) score -= 5; + + const actions = []; + if (reviewSlaDays == null || reviewSlaDays > 14) { + actions.push(action("warning", "Publish a review SLA of 14 days or less.")); + } + if (medianFirstResponseHours == null || medianFirstResponseHours > 48) { + actions.push(action("warning", "Assign a sponsor contact who can respond within two business days.")); + } + + return axis(clampScore(score), "Review responsiveness", "Historical and published response windows for sponsor feedback.", actions); +} + +function scoreRubric(rubric = {}) { + const criteria = Array.isArray(rubric.criteria) ? rubric.criteria : []; + const measurableCount = criteria.filter((criterion) => criterion.measurable).length; + const weightsTotal = criteria.reduce((total, criterion) => total + (Number(criterion.weight) || 0), 0); + const evidenceRequirements = Array.isArray(rubric.evidenceRequirements) ? rubric.evidenceRequirements : []; + + let score = 20; + if (criteria.length >= 3) score += 20; + else if (criteria.length >= 2) score += 10; + + if (measurableCount === criteria.length && criteria.length > 0) score += 20; + else if (measurableCount >= 2) score += 10; + + if (weightsTotal === 100) score += 15; + else if (weightsTotal >= 90 && weightsTotal <= 110) score += 8; + + if (evidenceRequirements.length >= 2) score += 15; + else if (evidenceRequirements.length === 1) score += 7; + + if (rubric.tieBreakerDefined) score += 10; + if (rubric.judgingProtocolPublished) score += 10; + + const actions = []; + if (criteria.length < 3) actions.push(action("blocker", "Define at least three evaluation criteria.")); + if (weightsTotal !== 100) actions.push(action("warning", "Normalize rubric weights to 100.")); + if (!rubric.tieBreakerDefined) actions.push(action("warning", "Publish a tie-break protocol before submissions open.")); + if (!rubric.judgingProtocolPublished) actions.push(action("warning", "Publish the judging protocol used for every submission.")); + + return axis(clampScore(score), "Rubric clarity", "Measurable criteria, evidence requirements, and judging protocol completeness.", actions); +} + +function scoreTimeline(timeline = {}, reviewSlaDays) { + const milestones = nullableNumber(timeline.milestones) ?? 0; + const submissionWindowDays = nullableNumber(timeline.submissionWindowDays); + const sponsorReviewWindowDays = nullableNumber(timeline.sponsorReviewWindowDays); + + let score = 35; + if (milestones >= 2) score += 20; + else if (milestones === 1) score += 8; + + if (submissionWindowDays != null && submissionWindowDays >= 21) score += 15; + else if (submissionWindowDays != null && submissionWindowDays >= 14) score += 8; + + if (sponsorReviewWindowDays != null && sponsorReviewWindowDays <= 14) score += 15; + else if (sponsorReviewWindowDays != null && sponsorReviewWindowDays <= 21) score += 7; + + if (reviewSlaDays != null && sponsorReviewWindowDays != null && reviewSlaDays <= sponsorReviewWindowDays) { + score += 15; + } + + const actions = []; + if (milestones < 2) actions.push(action("warning", "Split the challenge into at least two milestones for review and payout clarity.")); + if (submissionWindowDays == null || submissionWindowDays < 21) { + actions.push(action("warning", "Give solvers enough calendar time for scientific work and replication.")); + } + if (sponsorReviewWindowDays == null) { + actions.push(action("blocker", "Set a sponsor review window before launch.")); + } + + return axis(clampScore(score), "Timeline readiness", "Milestones, submission window, and review windows are clear and realistic.", actions); +} + +function scoreAmendments(history = {}, amendmentPolicy = {}) { + const historicalAmendments = nullableNumber(history.materialAmendmentsAfterLaunch) ?? 0; + const noticeDays = nullableNumber(amendmentPolicy.materialChangeNoticeDays); + const maxMaterialAmendments = nullableNumber(amendmentPolicy.maxMaterialAmendments); + + let score = 100; + if (historicalAmendments > 5) score -= 30; + else if (historicalAmendments > 2) score -= 18; + else if (historicalAmendments > 0) score -= 6; + + if (noticeDays == null || noticeDays < 2) score -= 20; + else if (noticeDays < 5) score -= 8; + + if (!amendmentPolicy.solverWithdrawalProtected) score -= 25; + if (maxMaterialAmendments == null || maxMaterialAmendments > 3) score -= 15; + + const actions = []; + if (!amendmentPolicy.solverWithdrawalProtected) { + actions.push(action("blocker", "Protect solver withdrawal or no-fault cancellation when material terms change.")); + } + if (noticeDays == null || noticeDays < 2) { + actions.push(action("warning", "Add advance notice for material challenge changes.")); + } + + return axis(clampScore(score), "Amendment stability", "Launch stability and solver protections for material changes.", actions); +} + +function scorePayoutHistory(history = {}) { + const paidAwards = nullableNumber(history.paidAwards) ?? 0; + const averagePayoutLagDays = nullableNumber(history.averagePayoutLagDays); + const cancelledAfterStart = nullableNumber(history.cancelledAfterStart) ?? 0; + const completedChallenges = nullableNumber(history.completedChallenges) ?? 0; + + let score = 25; + if (paidAwards >= 10) score += 30; + else if (paidAwards >= 5) score += 22; + else if (paidAwards >= 2) score += 14; + else if (paidAwards === 1) score += 8; + + if (averagePayoutLagDays != null && averagePayoutLagDays <= 5) score += 25; + else if (averagePayoutLagDays != null && averagePayoutLagDays <= 14) score += 16; + else if (averagePayoutLagDays != null && averagePayoutLagDays <= 30) score += 7; + + if (completedChallenges >= 5) score += 15; + else if (completedChallenges >= 2) score += 8; + + score -= Math.min(25, cancelledAfterStart * 10); + + const actions = []; + if (paidAwards < 2) actions.push(action("warning", "Attach prior paid-award evidence or mark the sponsor as new.")); + if (averagePayoutLagDays == null || averagePayoutLagDays > 14) { + actions.push(action("warning", "Publish a payout timeline and reduce average payout lag below 14 days.")); + } + if (cancelledAfterStart > 0) { + actions.push(action("warning", "Explain prior post-launch cancellations and add solver protections.")); + } + + return axis(clampScore(score), "Payout history", "Evidence of completed challenges and timely award settlement.", actions); +} + +function scoreDisputes(history = {}, disputePolicy = {}) { + const disputesOpened = nullableNumber(history.disputesOpened) ?? 0; + const disputesResolvedWithinSla = nullableNumber(history.disputesResolvedWithinSla) ?? 0; + const completedChallenges = Math.max(1, nullableNumber(history.completedChallenges) ?? 1); + const disputeRate = disputesOpened / completedChallenges; + + let score = 100; + if (disputeRate > 0.75) score -= 38; + else if (disputeRate > 0.35) score -= 24; + else if (disputeRate > 0.15) score -= 12; + + if (disputesOpened > 0) { + const resolvedRate = disputesResolvedWithinSla / disputesOpened; + if (resolvedRate < 0.35) score -= 28; + else if (resolvedRate < 0.75) score -= 14; + } + + if (disputePolicy.responseSlaDays == null || disputePolicy.responseSlaDays > 14) score -= 16; + if (!disputePolicy.neutralReviewerAvailable) score -= 14; + + const actions = []; + if (!disputePolicy.neutralReviewerAvailable) { + actions.push(action("warning", "Name a neutral reviewer or arbitration fallback for disputes.")); + } + if (disputePolicy.responseSlaDays == null || disputePolicy.responseSlaDays > 14) { + actions.push(action("warning", "Set a dispute response SLA of 14 days or less.")); + } + + return axis(clampScore(score), "Dispute responsiveness", "Dispute rate, resolution speed, and neutral-review availability.", actions); +} + +function scoreIpNda(ipTerms = {}, visibility = "public") { + let score = 35; + if (ipTerms.solverRetainsUntilPaid) score += 30; + if (typeof ipTerms.licenseOnPayout === "string" && ipTerms.licenseOnPayout.length >= 8) score += 20; + if (visibility === "public" || ipTerms.ndaTemplatePublished) score += 15; + + const actions = []; + if (!ipTerms.solverRetainsUntilPaid) { + actions.push(action("blocker", "State that solvers retain IP until payment is completed.")); + } + if (visibility !== "public" && !ipTerms.ndaTemplatePublished) { + actions.push(action("warning", "Publish the NDA template or provide a public summary of restricted terms.")); + } + if (!ipTerms.licenseOnPayout) { + actions.push(action("warning", "Define the license or transfer terms that apply after payout.")); + } + + return axis(clampScore(score), "IP and NDA clarity", "Solver rights, payout-triggered license terms, and private-challenge NDA clarity.", actions); +} + +function weightedAverage(axisResults, weights) { + const totalWeight = Object.values(weights).reduce((sum, weight) => sum + weight, 0); + return Object.entries(axisResults).reduce((sum, [key, result]) => { + return sum + result.score * (weights[key] / totalWeight); + }, 0); +} + +function buildFindings(axisResults) { + return Object.entries(axisResults) + .flatMap(([axisName, result]) => { + if (result.score >= 85 && result.actions.length === 0) { + return [{ + severity: "info", + axis: axisName, + message: `${result.label} is ready.`, + recommendedAction: "Keep current evidence visible to solvers." + }]; + } + + return result.actions.map((entry) => ({ + severity: entry.severity, + axis: axisName, + message: result.summary, + recommendedAction: entry.message + })); + }) + .sort((left, right) => severityRank(left.severity) - severityRank(right.severity)); +} + +function buildSponsorActions(axisResults, tier) { + const actions = Object.values(axisResults) + .flatMap((result) => result.actions) + .sort((left, right) => severityRank(left.severity) - severityRank(right.severity)) + .map((entry) => entry.message); + + if (actions.length > 0) { + return [...new Set(actions)]; + } + + if (tier === "trusted") { + return ["Keep funding proof, rubric, review SLA, and IP terms visible on the challenge page."]; + } + + return ["Publish additional sponsor evidence before soliciting more solver work."]; +} + +function guidanceForTier(tier, findings) { + if (tier === "trusted") { + return "Ready for solver participation. Evidence is strong enough to open or promote the challenge."; + } + + if (tier === "watch") { + const firstWarning = findings.find((finding) => finding.severity !== "info"); + return `Proceed only after sponsor follow-up: ${firstWarning?.recommendedAction ?? "clear the listed warnings."}`; + } + + const blocker = findings.find((finding) => finding.severity === "blocker"); + return `Hold solver onboarding until sponsor fixes: ${blocker?.recommendedAction ?? "the listed blockers."}`; +} + +function summarizeScorecards(scorecards) { + const byTier = Object.fromEntries(TIER_ORDER.map((tier) => [tier, 0])); + let totalScore = 0; + for (const card of scorecards) { + byTier[card.tier] += 1; + totalScore += card.score; + } + + return { + totalChallenges: scorecards.length, + averageScore: scorecards.length ? Math.round(totalScore / scorecards.length) : 0, + byTier + }; +} + +function tierForScore(score, axisResults) { + const hasBlocker = Object.values(axisResults).some((result) => { + return result.actions.some((entry) => entry.severity === "blocker"); + }); + if (hasBlocker || score < 55) return "hold"; + if (score < 80) return "watch"; + return "trusted"; +} + +function compactAxisResults(axisResults) { + return Object.fromEntries( + Object.entries(axisResults).map(([key, value]) => [ + key, + { score: value.score, label: value.label } + ]) + ); +} + +function toDigestableScorecard(card) { + return { + sponsorId: card.sponsorId, + challengeId: card.challengeId, + score: card.score, + tier: card.tier, + axisResults: compactAxisResults(card.axisResults), + sponsorActions: card.sponsorActions, + solverGuidance: card.solverGuidance, + publicEvidenceDigest: card.publicEvidenceDigest + }; +} + +function axis(score, label, summary, actions) { + return { + score: clampScore(score), + label, + summary, + actions + }; +} + +function action(severity, message) { + return { severity, message }; +} + +function severityRank(severity) { + if (severity === "blocker") return 0; + if (severity === "warning") return 1; + return 2; +} + +function sha256(payload) { + return createHash("sha256").update(payload).digest("hex"); +} + +function clampScore(value) { + return Math.max(0, Math.min(100, Math.round(value))); +} + +function clampNumber(value, minimum, maximum) { + const parsed = Number(value); + if (!Number.isFinite(parsed)) return minimum; + return Math.max(minimum, Math.min(maximum, parsed)); +} + +function nullableNumber(value) { + if (value === null || value === undefined) return null; + const parsed = Number(value); + return Number.isFinite(parsed) ? parsed : null; +} diff --git a/scientific-bounty-sponsor-scorecard/test/sponsor-scorecard.test.js b/scientific-bounty-sponsor-scorecard/test/sponsor-scorecard.test.js new file mode 100644 index 0000000..261d485 --- /dev/null +++ b/scientific-bounty-sponsor-scorecard/test/sponsor-scorecard.test.js @@ -0,0 +1,59 @@ +import assert from "node:assert/strict"; +import { describe, it } from "node:test"; +import sample from "../data/sample-sponsor-input.json" with { type: "json" }; +import { + buildSponsorScorecard, + evaluateChallenge, + redactSensitiveFields, + stableStringify +} from "../src/sponsor-scorecard.js"; + +describe("scientific bounty sponsor scorecard", () => { + it("marks verified, responsive sponsors as trusted", () => { + const trustedSponsor = sample.sponsors[0]; + const card = evaluateChallenge(trustedSponsor, trustedSponsor.challengePosts[0]); + + assert.equal(card.tier, "trusted"); + assert.ok(card.score >= 85); + assert.equal(card.findings.some((finding) => finding.severity === "blocker"), false); + assert.match(card.solverGuidance, /Ready for solver participation/); + }); + + it("holds challenges with missing funding and unsafe IP terms", () => { + const riskySponsor = sample.sponsors[2]; + const card = evaluateChallenge(riskySponsor, riskySponsor.challengePosts[0]); + + assert.equal(card.tier, "hold"); + assert.ok(card.score < 55); + assert.ok(card.sponsorActions.some((item) => item.includes("verified funding proof"))); + assert.ok(card.sponsorActions.some((item) => item.includes("retain IP until payment"))); + }); + + it("routes incomplete but fixable challenges into watch", () => { + const watchSponsor = sample.sponsors[1]; + const card = evaluateChallenge(watchSponsor, watchSponsor.challengePosts[0]); + + assert.equal(card.tier, "watch"); + assert.ok(card.score >= 55 && card.score < 80); + assert.ok(card.sponsorActions.some((item) => item.includes("tie-break protocol"))); + }); + + it("builds a deterministic public-safe digest", () => { + const first = buildSponsorScorecard(sample, { generatedAt: sample.asOf }); + const second = buildSponsorScorecard(sample, { generatedAt: "2030-01-01T00:00:00.000Z" }); + + assert.equal(first.auditDigest, second.auditDigest); + assert.equal(first.summary.totalChallenges, 3); + assert.deepEqual(first.summary.byTier, { trusted: 1, watch: 1, hold: 1 }); + }); + + it("redacts private notes and payment identity fields from public echoes", () => { + const redacted = redactSensitiveFields(sample); + const serialized = stableStringify(redacted); + + assert.equal(serialized.includes("do-not-export"), false); + assert.equal(serialized.includes("private@example.invalid"), false); + assert.equal(serialized.includes("Finance lead phone"), false); + assert.equal(serialized.includes("[redacted]"), true); + }); +});