Skip to content

Build multi-destination replicationInfo per backend #6172

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
maeldonn wants to merge 3 commits into
base: development/9.4
Choose a base branch
from
+1,523 −1,076
Open

Build multi-destination replicationInfo per backend #6172

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
a23e3ee
Reformat with prettier
maeldonn Jun 5, 2026
a946d61
Build multi-destination replicationInfo per backend
maeldonn Jun 5, 2026
56b72f4
Bump arsenal dependency
maeldonn May 29, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
176 changes: 78 additions & 98 deletions lib/api/apiUtils/object/getReplicationInfo.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,73 +1,61 @@
const { isServiceAccount, getServiceAccountProperties } =
require('../authorization/permissionChecks');
const { replicationBackends } = require('arsenal').constants;
const { isServiceAccount, getServiceAccountProperties } = require('../authorization/permissionChecks');
const { constants, models } = require('arsenal');

function _getBackend(objectMD, site) {
const backends = objectMD ? objectMD.replicationInfo.backends : [];
const backend = backends.find(o => o.site === site);
// If the backend already exists, just update the status.
if (backend) {
return Object.assign({}, backend, { status: 'PENDING' });
}
return {
site,
status: 'PENDING',
dataStoreVersionId: '',
};
}
const { replicationBackends } = constants;
const { ReplicationConfiguration } = models;

function _getStorageClasses(s3config, rule) {
if (rule.storageClass) {
return rule.storageClass.split(',');
}
const { replicationEndpoints } = s3config;
// If no storage class, use the given default endpoint or the sole endpoint
if (replicationEndpoints.length > 0) {
const endPoint =
replicationEndpoints.find(endpoint => endpoint.default) || replicationEndpoints[0];
return [endPoint.site];
}
return undefined;
/**
* Apply the default replication endpoint as a fallback storageClass
* for rules that don't specify one. Returns a new rules array; rules
* without a resolvable storageClass are dropped so they never reach
* the backend resolver.
*/
function _withDefaultStorageClass(rules, s3config) {
const { replicationEndpoints = [] } = s3config;
const fallback = replicationEndpoints.find(e => e.default)?.site ?? replicationEndpoints[0]?.site;
return rules
.map(rule => {
if (rule.storageClass) {
return rule;
}
if (!fallback) {
return null;
}
return { ...rule, storageClass: fallback };
})
.filter(Boolean);
}

function _getReplicationInfo(s3config, rule, replicationConfig, content, operationType,
objectMD, bucketMD) {
const storageTypes = [];
const backends = [];
const storageClasses = _getStorageClasses(s3config, rule);
if (!storageClasses) {
return undefined;
/**
* Check whether the authenticated user is allowed to trigger replication.
* Internal service accounts (e.g. Lifecycle) are not allowed unless their
* account properties explicitly permit it (e.g. MD ingestion).
* @param {AuthInfo} [authInfo] - authentication info of the request issuer
* @return {boolean} true if the user can trigger replication
*/
function _canUserReplicate(authInfo) {
if (!authInfo) {
return true;
}
storageClasses.forEach(storageClass => {
const storageClassName =
storageClass.endsWith(':preferred_read') ?
storageClass.split(':')[0] : storageClass;
// TODO CLDSRV-646: for consistency, should we look at replicationEndpoints instead, like
// `_getStorageClasses()` ?
const location = s3config.locationConstraints[storageClassName];
if (location && replicationBackends[location.type]) {
storageTypes.push(location.type);
}
backends.push(_getBackend(objectMD, storageClassName));
});
if (storageTypes.length > 0 && operationType) {
content.push(operationType);
const canonicalId = authInfo.getCanonicalID();
if (!isServiceAccount(canonicalId)) {
return true;
}
return {
status: 'PENDING',
backends,
content,
destination: replicationConfig.destination,
storageClass: storageClasses.join(','),
role: replicationConfig.role,
storageType: storageTypes.join(','),
isNFS: bucketMD.isNFS(),
};
const props = getServiceAccountProperties(canonicalId);
return !!props?.canReplicate;
}

/**
* Get the object replicationInfo to replicate data and metadata, or only
* metadata if the operation only changes metadata or the object is 0 bytes
* metadata if the operation only changes metadata or the object is 0 bytes.
*
* The rule-matching / dedup / per-backend stamping logic lives in
* arsenal's `ReplicationConfiguration.resolveBackends`. This function
* is the cloudserver-specific shell: it enforces the service-account
* gate, supplies a default storageClass from `replicationEndpoints`,
* decides the `content` array based on the operation kind, and
* stitches the result into a `replicationInfo` envelope.
*
* @param {object} s3config - Cloudserver configuration object
* @param {object} s3config.locationConstraints - Configured map of location constraints
* @param {object[]} s3config.replicationEndpoints - Configured replication endpoints
Expand All @@ -78,49 +66,41 @@ function _getReplicationInfo(s3config, rule, replicationConfig, content, operati
* @param {string} operationType - The type of operation to replicate
* @param {object} objectMD - The object metadata
* @param {AuthInfo} [authInfo] - authentication info of object owner
* @return {undefined}
* @return {object|undefined}
*/
function getReplicationInfo(
s3config, objKey, bucketMD, isMD, objSize, operationType, objectMD, authInfo) {
const content = isMD || objSize === 0 ? ['METADATA'] : ['DATA', 'METADATA'];
function getReplicationInfo(s3config, objKey, bucketMD, isMD, objSize, operationType, objectMD, authInfo) {
const config = bucketMD.getReplicationConfiguration();
if (!config || !_canUserReplicate(authInfo)) {
return undefined;
}

// Do not replicate object in the following cases:
//
// - bucket does not have a replication configuration
//
// - replication configuration does not apply to the object
// (i.e. no rule matches object prefix)
//
// - replication configuration applies to the object (i.e. a rule matches
// object prefix) but the status is disabled
//
// - object owner is an internal service account like Lifecycle,
// unless the account properties explicitly allow it to
// replicate like MD ingestion (because we do not want to
// replicate objects created from actions triggered by internal
// services, by design)
const isCloud = site => !!replicationBackends[s3config.locationConstraints[site]?.type];
const rules = _withDefaultStorageClass(config.rules || [], s3config);
const backends = ReplicationConfiguration.resolveBackends(
Comment thread
maeldonn marked this conversation as resolved.
{ ...config, rules },
objKey,
isCloud,
objectMD?.replicationInfo?.backends,
);

if (config) {
let doReplicate = false;
if (!authInfo || !isServiceAccount(authInfo.getCanonicalID())) {
doReplicate = true;
} else {
const serviceAccountProps = getServiceAccountProperties(
authInfo.getCanonicalID());
doReplicate = serviceAccountProps.canReplicate;
}
if (doReplicate) {
const rule = config.rules.find(
rule => (objKey.startsWith(rule.prefix) && rule.enabled));
if (rule) {
// TODO CLDSRV-646 : should "merge" the replicationInfo for different rules
return _getReplicationInfo(
s3config, rule, config, content, operationType, objectMD, bucketMD);
}
}
if (backends.length === 0) {
return undefined;
}
return undefined;

const hasCloudBackend = backends.some(b => isCloud(b.site));

const content = isMD || objSize === 0 ? ['METADATA'] : ['DATA', 'METADATA'];
if (hasCloudBackend && operationType) {
content.push(operationType);
}

return {
status: 'PENDING',
backends,
content,
role: ReplicationConfiguration.resolveSourceRole(config.role),
isNFS: bucketMD.isNFS(),
};
}

module.exports = getReplicationInfo;
115 changes: 64 additions & 51 deletions lib/metadata/acl.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,29 +31,46 @@ const acl = {
* contain the same number of elements, and all elements from one
* grant are incuded in the other grant
*/
return oldAcl[grant].length === newAcl[grant].length
&& oldAcl[grant].every(value => newAcl[grant].includes(value));
return (
oldAcl[grant].length === newAcl[grant].length && oldAcl[grant].every(value => newAcl[grant].includes(value))
);
},

addObjectACL(bucket, objectKey, objectMD, addACLParams, params, log, cb) {
log.trace('updating object acl in metadata');
const isAclUnchanged = Object.keys(objectMD.acl).length === Object.keys(addACLParams).length
&& Object.keys(objectMD.acl).every(grant => this._aclGrantDidNotChange(grant, objectMD.acl, addACLParams));
const isAclUnchanged =
Object.keys(objectMD.acl).length === Object.keys(addACLParams).length &&
Object.keys(objectMD.acl).every(grant => this._aclGrantDidNotChange(grant, objectMD.acl, addACLParams));
if (!isAclUnchanged) {
/* eslint-disable no-param-reassign */
objectMD.acl = addACLParams;
objectMD.originOp = 's3:ObjectAcl:Put';

// Use storageType to determine if replication update is needed, as it is set only for
// "cloud" locations. This ensures that we reset replication when CRR is used, but not
// when multi-backend replication (i.e. Zenko) is used.
// TODO: this should be refactored to properly update the replication info, accounting
// for multiple rules and resetting the status only if needed CLDSRV-646
// Rebuild replication info from the current bucket config to
// pick up any new destinations. A "role" on the entry means
// backbeat uses IAM role-based auth on the destination
// (CRR). No role means credentials live in the location
// configuration (cloud, including scality-to-scality via
// stored S3 creds), and ACL replication is not supported —
// preserve the existing status instead of resetting to
// PENDING.
const hasDestRole = b => !!b.role;

const replicationInfo = getReplicationInfo(config, objectKey, bucket, true);
if (replicationInfo && !replicationInfo.storageType) {
if (replicationInfo && replicationInfo.backends.some(hasDestRole)) {
const backends = replicationInfo.backends.map(b => {
if (hasDestRole(b)) {
return b;
}

const existing = objectMD.replicationInfo.backends.find(e => e.site === b.site);
return existing || b;

@benzekrimaha benzekrimaha Jun 10, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this still adds a newly configured cloud backend on an ACL-only update if there is also a CRR backend.
If cloud ACL replication is not supported, we should only preserve existing non-role backends, not add new ones as PENDING.
Maybe:

Suggested change
return existing || b;
const existing = objectMD.replicationInfo.backends.find(e => e.site === b.site);
return existing;

@maeldonn maeldonn Jun 10, 2026 •
edited
Loading

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good catch, switched to dropping newly configured cloud backends instead of adding them as PENDING, and matching existing entries by site + destination + role rather than site alone.

});

objectMD.replicationInfo = {
...objectMD.replicationInfo,
...replicationInfo,
backends,
};
}

Expand All @@ -77,14 +94,22 @@ const acl = {
};
let validCannedACL = [];
if (resourceType === 'bucket') {
validCannedACL =
['private', 'public-read', 'public-read-write',
'authenticated-read', 'log-delivery-write'];
validCannedACL = [
'private',
'public-read',
'public-read-write',
'authenticated-read',
'log-delivery-write',
];
} else if (resourceType === 'object') {
validCannedACL =
['private', 'public-read', 'public-read-write',
'authenticated-read', 'bucket-owner-read',
'bucket-owner-full-control'];
validCannedACL = [
'private',
'public-read',
'public-read-write',
'authenticated-read',
'bucket-owner-read',
'bucket-owner-full-control',
];
}

// parse canned acl
Expand All @@ -98,45 +123,34 @@ const acl = {
}

// parse grant headers
const grantReadHeader =
aclUtils.parseGrant(headers['x-amz-grant-read'], 'READ');
const grantReadHeader = aclUtils.parseGrant(headers['x-amz-grant-read'], 'READ');
let grantWriteHeader = [];
if (resourceType === 'bucket') {
grantWriteHeader = aclUtils
.parseGrant(headers['x-amz-grant-write'], 'WRITE');
grantWriteHeader = aclUtils.parseGrant(headers['x-amz-grant-write'], 'WRITE');
}
const grantReadACPHeader = aclUtils
.parseGrant(headers['x-amz-grant-read-acp'], 'READ_ACP');
const grantWriteACPHeader = aclUtils
.parseGrant(headers['x-amz-grant-write-acp'], 'WRITE_ACP');
const grantFullControlHeader = aclUtils
.parseGrant(headers['x-amz-grant-full-control'], 'FULL_CONTROL');
const allGrantHeaders =
[].concat(grantReadHeader, grantWriteHeader,
grantReadACPHeader, grantWriteACPHeader,
grantFullControlHeader).filter(item => item !== undefined);
const grantReadACPHeader = aclUtils.parseGrant(headers['x-amz-grant-read-acp'], 'READ_ACP');
const grantWriteACPHeader = aclUtils.parseGrant(headers['x-amz-grant-write-acp'], 'WRITE_ACP');
const grantFullControlHeader = aclUtils.parseGrant(headers['x-amz-grant-full-control'], 'FULL_CONTROL');
const allGrantHeaders = []
.concat(grantReadHeader, grantWriteHeader, grantReadACPHeader, grantWriteACPHeader, grantFullControlHeader)
.filter(item => item !== undefined);
if (allGrantHeaders.length === 0) {
return cb(null, currentResourceACL);
}

const usersIdentifiedByEmail = allGrantHeaders
.filter(it => it && it.userIDType.toLowerCase() === 'emailaddress');
const usersIdentifiedByGroup = allGrantHeaders
.filter(item => item && item.userIDType.toLowerCase() === 'uri');
const usersIdentifiedByEmail = allGrantHeaders.filter(
it => it && it.userIDType.toLowerCase() === 'emailaddress',
);
const usersIdentifiedByGroup = allGrantHeaders.filter(item => item && item.userIDType.toLowerCase() === 'uri');
const justEmails = usersIdentifiedByEmail.map(item => item.identifier);
const validGroups = [
constants.allAuthedUsersId,
constants.publicId,
constants.logId,
];
const validGroups = [constants.allAuthedUsersId, constants.publicId, constants.logId];

for (let i = 0; i < usersIdentifiedByGroup.length; i++) {
if (validGroups.indexOf(usersIdentifiedByGroup[i].identifier) < 0) {
return cb(errors.InvalidArgument);
}
}
const usersIdentifiedByID = allGrantHeaders
.filter(item => item && item.userIDType.toLowerCase() === 'id');
const usersIdentifiedByID = allGrantHeaders.filter(item => item && item.userIDType.toLowerCase() === 'id');
// TODO: Consider whether want to verify with Vault
// whether canonicalID is associated with existing
// account before adding to ACL
Expand All @@ -148,27 +162,26 @@ const acl = {
if (err) {
return cb(err);
}
const reconstructedUsersIdentifiedByEmail = aclUtils.
reconstructUsersIdentifiedByEmail(results,
usersIdentifiedByEmail);
const reconstructedUsersIdentifiedByEmail = aclUtils.reconstructUsersIdentifiedByEmail(
results,
usersIdentifiedByEmail,
);
const allUsers = [].concat(
reconstructedUsersIdentifiedByEmail,
usersIdentifiedByGroup,
usersIdentifiedByID);
const revisedACL =
aclUtils.sortHeaderGrants(allUsers, resourceACL);
usersIdentifiedByID,
);
const revisedACL = aclUtils.sortHeaderGrants(allUsers, resourceACL);
return cb(null, revisedACL);
});
} else {
// If don't have to look up canonicalID's just sort grants
// and add to bucket
const revisedACL = aclUtils
.sortHeaderGrants(allGrantHeaders, resourceACL);
const revisedACL = aclUtils.sortHeaderGrants(allGrantHeaders, resourceACL);
return cb(null, revisedACL);
}
return undefined;
},
};

module.exports = acl;

2 changes: 1 addition & 1 deletion package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@
"@opentelemetry/instrumentation-ioredis": "~0.64.0",
"@opentelemetry/instrumentation-mongodb": "~0.69.0",
"@smithy/node-http-handler": "^3.0.0",
"arsenal": "git+https://github.com/scality/Arsenal#8.4.5",
"arsenal": "git+https://github.com/scality/Arsenal#724d06289e11a429dfdf9e1236f98e904f748a90",
Comment thread
maeldonn marked this conversation as resolved.
Comment thread
maeldonn marked this conversation as resolved.
Comment thread
maeldonn marked this conversation as resolved.
"async": "2.6.4",
"bucketclient": "scality/bucketclient#8.2.7",
"bufferutil": "^4.0.8",
Expand Down
Loading
Loading