Skip to content

CDA-113 Adding csv support for timeseries #1780

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
rma-bryson wants to merge 2 commits into
base: develop
Choose a base branch
from
Open

CDA-113 Adding csv support for timeseries #1780

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
76e0901
CDA-113 Adding csv support for timeseries
rma-bryson Mar 6, 2026
deba238
CDA-113 - Refactored common ts query logic for re-use in csv stream. …
rma-bryson Jun 18, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 41 additions & 2 deletions cwms-data-api/src/main/java/cwms/cda/ApiServlet.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -186,13 +186,17 @@
import io.javalin.http.BadRequestResponse;
import io.javalin.http.Handler;
import io.javalin.http.JavalinServlet;
import cwms.cda.data.dto.csv.CwmsCsvDTO;
import cwms.cda.formatters.csv.CsvExampleGenerator;
import io.javalin.plugin.openapi.OpenApiOptions;
import io.javalin.plugin.openapi.OpenApiPlugin;
import io.swagger.v3.oas.models.Components;
import io.swagger.v3.oas.models.OpenAPI;
import io.swagger.v3.oas.models.Operation;
import io.swagger.v3.oas.models.PathItem;
import io.swagger.v3.oas.models.info.Info;
import io.swagger.v3.oas.models.media.MediaType;
import io.swagger.v3.oas.models.responses.ApiResponse;
import io.swagger.v3.oas.models.security.SecurityRequirement;
import io.swagger.v3.oas.models.servers.Server;
import java.io.IOException;
Expand Down Expand Up @@ -223,6 +227,8 @@
import org.jooq.exception.DataAccessException;
import org.owasp.html.HtmlPolicyBuilder;
import org.owasp.html.PolicyFactory;
import io.github.classgraph.ClassGraph;
import io.github.classgraph.ScanResult;


/**
Expand Down Expand Up @@ -932,8 +938,41 @@ private void getOpenApiOptions(JavalinConfig config) {
.addSecurityItem(new SecurityRequirement().addList(provider))
);
ops.path("/swagger-docs")
.responseModifier((ctx,api) -> {
api.getPaths().forEach((key,path) -> setSecurityRequirements(key,path,secReqs));
.responseModifier((ctx, api) -> {
api.getPaths().forEach((key, path) -> setSecurityRequirements(key, path, secReqs));

Map<String, Class<? extends CwmsCsvDTO>> schemaToClass = new HashMap<>();
try (ScanResult scanResult = new ClassGraph()
.acceptPackages("cwms.cda.data.dto")
.scan()) {
List<Class<CwmsCsvDTO>> csvDtoClasses = scanResult.getClassesImplementing(CwmsCsvDTO.class.getName())
.loadClasses(CwmsCsvDTO.class);
for (Class<? extends CwmsCsvDTO> clazz : csvDtoClasses) {
schemaToClass.put(clazz.getSimpleName(), clazz);
}
}
api.getPaths().values().forEach(pathItem -> {
for (Operation op : pathItem.readOperations()) {
if (op.getResponses() != null) {
for (ApiResponse resp : op.getResponses().values()) {
if (resp.getContent() != null && resp.getContent().containsKey(Formats.CSV)) {
MediaType csvMedia = resp.getContent().get(Formats.CSV);
if (csvMedia.getSchema() != null && csvMedia.getSchema().get$ref() != null) {
String ref = csvMedia.getSchema().get$ref();
String schemaName = ref.substring(ref.lastIndexOf('/') + 1);
@SuppressWarnings("unchecked")
Class<? extends CwmsCsvDTO<?>> dtoClass = (Class<? extends CwmsCsvDTO<?>>) schemaToClass.get(schemaName);

if (dtoClass != null) {
csvMedia.setExample(CsvExampleGenerator.getExample(dtoClass));
}
}
}
}
}
}
});

return api;
})
.defaultDocumentation(doc -> {
Expand Down
11 changes: 8 additions & 3 deletions cwms-data-api/src/main/java/cwms/cda/api/Controllers.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,9 @@
import io.javalin.core.validation.Validator;
import io.javalin.http.Context;
import java.time.Instant;
import java.time.ZoneId;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
Expand Down Expand Up @@ -127,6 +129,8 @@ public final class Controllers {
public static final String MATCH_NULL_PARENTS = "match-null-parents";
public static final String ENTITY_ID = "entity-id";
public static final String PARENT_ENTITY_ID = "parent-entity-id";
public static final String INCLUDE_METADATA_AS_CSV_COMMENTS = "include-metadata-as-comments";
public static final String INCLUDE_OPTIONAL_CSV_COLUMNS = "include-optional-csv-columns";

public static final String CREATE_AS_LRTS = "create-as-lrts";
public static final String STORE_RULE = "store-rule";
Expand Down Expand Up @@ -172,9 +176,8 @@ public final class Controllers {
public static final String TS_IDS = "ts-ids";

public static final String EXAMPLE_DATE = "2021-06-10T13:00:00-07:00";
public static final String DATE_FORMAT = "YYYY-MM-dd'T'hh:mm:ss[Z'['VV']']";
public static final String TIME_FORMAT_DESC = "The <a href=\"times.html\">format for this field</a> is ISO 8601 extended" +
", with optional offset and timezone, i.e., '" + DATE_FORMAT + "', e.g., '" + EXAMPLE_DATE + "'." ;
public static final String TIME_FORMAT_DESC = "The <a href=\"times.html\">format for this field</a> " +
"is ISO 8601 extended in UTC, e.g., 2026-06-18T19:42:00Z";

public static final String INCLUDE_ASSIGNED = "include-assigned";
public static final String ANY_MASK = "*";
Expand Down Expand Up @@ -214,6 +217,8 @@ public final class Controllers {
public static final String AREA_UNIT = "area-unit";
public static final String STATION_UNIT = "station-unit";
public static final String STAGE_UNIT = "stage-unit";
public static final String DATE_FORMAT = "date-format";
public static final String DATE_FORMAT_PATTERN = "date-format-pattern";
public static final String TRIM = "trim";
public static final String DESIGNATOR = "designator";
public static final String DESIGNATOR_MASK = "designator-mask";
Expand Down
99 changes: 92 additions & 7 deletions cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesController.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,10 @@
import cwms.cda.data.dao.VerticalDatum;
import cwms.cda.data.dto.TimeSeries;
import cwms.cda.formatters.ContentType;
import cwms.cda.formatters.DateFormatResolver;
import cwms.cda.formatters.DateFormat;
import cwms.cda.formatters.csv.CsvConfiguration;
import cwms.cda.data.dto.csv.TimeSeriesCsv;
import cwms.cda.formatters.Formats;
import cwms.cda.helpers.DateUtils;
import io.javalin.apibuilder.CrudHandler;
Expand Down Expand Up @@ -112,7 +116,7 @@ public class TimeSeriesController implements CrudHandler {
private final MetricRegistry metrics;

private final Histogram requestResultSize;
private static final int DEFAULT_PAGE_SIZE = 500;
static final int DEFAULT_PAGE_SIZE = 500;


public TimeSeriesController(MetricRegistry metrics) {
Expand Down Expand Up @@ -382,12 +386,26 @@ public void delete(@NotNull Context ctx, @NotNull String timeseries) {
+ "identifies where in the request you are. This is an opaque"
+ " value, and can be obtained from the 'next-page' value in "
+ "the response. Deprecated, use " + PAGE + " instead."),
@OpenApiParam(name = PAGE_SIZE,
type = Integer.class,
@OpenApiParam(name = PAGE_SIZE, type = Integer.class,
description = "How many entries per page returned. "
+ "Default " + DEFAULT_PAGE_SIZE + ". Use 0 to return an empty values array, "
+ "For JSON/XML paging, this controls page size. "
+ "For CSV, this controls the internal fetch batch size used while streaming a single response. "
+ "CSV clients do not request subsequent pages. "
+ "Default " + DEFAULT_PAGE_SIZE +". Use 0 to return an empty values array, "
+ "or -1 to return the entire window in one response without a next-page cursor. "
+ "Values less than -1 are invalid.")
+ "Values less than -1 are invalid."),
@OpenApiParam(name = INCLUDE_METADATA_AS_CSV_COMMENTS, type = Boolean.class,
description = "When true, include dataset metadata as csv header comments "
+ "prepended with # (default is false)."),
@OpenApiParam(name = INCLUDE_OPTIONAL_CSV_COLUMNS, type = Boolean.class,
description = "When true, include optional columns (quality-code, data-entry-date) "
+ "in the CSV response (default is false)."),
@OpenApiParam(name = DATE_FORMAT,
description = "Specifies the format of any dates in the response. "
+ "Default is ISO8601-Instant. Other possibilities are epoch-millis, ISO8601-Offset, "
+ "ISO8601-Local, date-only, and custom."),
@OpenApiParam(name = DATE_FORMAT_PATTERN,
description = "When date-format is set to 'custom', this parameter specifies the date format pattern.")
},
responses = {
@OpenApiResponse(status = STATUS_200,
Expand All @@ -397,6 +415,7 @@ public void delete(@NotNull Context ctx, @NotNull String timeseries) {
@OpenApiContent(from = TimeSeries.class, type = Formats.XMLV2),
@OpenApiContent(from = TimeSeries.class, type = Formats.XML),
@OpenApiContent(from = TimeSeries.class, type = Formats.JSON),
@OpenApiContent(from = TimeSeriesCsv.class, type= Formats.CSV),
@OpenApiContent(from = TimeSeries.class, type = ""),}),
@OpenApiResponse(status = STATUS_400, description = "Invalid parameter combination"),
@OpenApiResponse(status = STATUS_404, description = "The provided combination of "
Expand Down Expand Up @@ -444,8 +463,21 @@ public void getAll(@NotNull Context ctx) {
Integer.class, DEFAULT_PAGE_SIZE, metrics,
name(TimeSeriesController.class.getName(), GET_ALL)));

boolean includeMetadata = ctx.queryParamAsClass(INCLUDE_METADATA_AS_CSV_COMMENTS, Boolean.class)
.getOrDefault(false);
boolean includeOptionalColumns = ctx.queryParamAsClass(INCLUDE_OPTIONAL_CSV_COLUMNS, Boolean.class)
.getOrDefault(false);
String dateFormatParam = ctx.queryParam(DATE_FORMAT);
String dateFormatPattern = ctx.queryParam(DATE_FORMAT_PATTERN);

String acceptHeader = ctx.header(Header.ACCEPT);
ContentType contentType = Formats.parseHeaderAndQueryParm(acceptHeader, format, TimeSeries.class);
DateFormat dateFormat = DateFormatResolver.resolve(dateFormatParam, dateFormatPattern);
CsvConfiguration csvConfig = new CsvConfiguration.Builder()
.withMetadataIncluded(includeMetadata)
.withOptionalColumnsIncluded(includeOptionalColumns)
.withDateFormat(dateFormat)
.build();

String results;
String version = contentType.getParameters().get(VERSION);
Expand All @@ -471,6 +503,13 @@ public void getAll(@NotNull Context ctx) {
.withShouldTrim(trim.getOrDefault(true))
.withIncludeEntryDate(includeEntryDate)
.build();

// CSV: stream a single response; page-size is only internal batch size
if (Formats.CSV.equals(contentType.getType())) {
streamCsv(ctx, csvConfig, pageSize, dao, requestParameters);
return;
}

// Execute DAO call with a timeout so we can return a clearer message instead of a generic 500
int apiTimeoutMs = Integer.getInteger("cwms.cda.api.apiTimeoutMs", 45000);
CompletableFuture<TimeSeries> daoFuture = CompletableFuture.supplyAsync(
Expand All @@ -492,7 +531,7 @@ public void getAll(@NotNull Context ctx) {
throw unwrapExecutionException(ex);
}

if(datum != null) { //this will be null for non-elevation ts
if (datum != null) { //this will be null for non-elevation ts
// user has requested a specific vertical datum
VerticalDatum vd = VerticalDatum.valueOf(datum); // the users request
ts = TimeSeriesVerticalDatumConverter.convertToVerticalDatum(ts, vd);
Expand All @@ -506,6 +545,24 @@ public void getAll(@NotNull Context ctx) {

ctx.result(results).contentType(contentType.toString());
} else {
String office = ctx.queryParam(OFFICE);

// CSV: stream a single response; page-size is only internal batch size
if (Formats.CSV.equals(contentType.getType())) {
TimeSeriesRequestParameters requestParameters = new TimeSeriesRequestParameters.Builder()
.withNames(names)
.withOffice(office)
.withUnits(units)
.withBeginTime(beginZdt)
.withEndTime(endZdt)
.withShouldTrim(trim.getOrDefault(true))
.withIncludeEntryDate(includeEntryDate)
.withVersionDate(versionDate)
.build();
streamCsv(ctx, csvConfig, pageSize, dao, requestParameters);
return;
}

if (versionDate != null) {
throw new IllegalArgumentException(String.format("Version date is only supported for:%s and %s",
Formats.JSONV2, Formats.XMLV2));
Expand All @@ -520,11 +577,11 @@ public void getAll(@NotNull Context ctx) {
format = "json";
}

String office = ctx.queryParam(OFFICE);
results = dao.getTimeseries(format, names, office, units, datum, beginZdt, endZdt, tz);
ctx.status(HttpServletResponse.SC_OK);
ctx.result(results);
}

addDeprecatedContentTypeWarning(ctx, contentType);
requestResultSize.update(results.length());
} catch (NotFoundException e) {
Expand All @@ -540,6 +597,27 @@ public void getAll(@NotNull Context ctx) {
}
}

private void streamCsv(@NotNull Context ctx, CsvConfiguration csvConfig, int batchSize, TimeSeriesDao dao, TimeSeriesRequestParameters requestParameters) {
int csvBatchSize = validateCsvBatchSize(batchSize);
dao.streamRequestedTimeSeriesCsv(
requestParameters,
(stream, position, mediaType, totalLength) -> {
ctx.status(HttpServletResponse.SC_OK);
ctx.contentType(mediaType);
ctx.header(Header.CONTENT_TYPE, Formats.CSV + "; charset=UTF-8");
ctx.header("X-Stream-Batch-Size", String.valueOf(batchSize));
try (stream) {
IOUtils.copy(stream, ctx.res.getOutputStream());
} catch (IOException e) {
throw new RuntimeException(e);
}
},
csvConfig,
null,
csvBatchSize //page-size drives streaming chunk size
);
}

static RuntimeException unwrapExecutionException(java.util.concurrent.ExecutionException ex) {
Throwable cause = ex.getCause();
if (cause instanceof RuntimeException) {
Expand All @@ -552,6 +630,13 @@ static RuntimeException unwrapExecutionException(java.util.concurrent.ExecutionE
return new RuntimeException(cause);
}

private int validateCsvBatchSize(int requestedPageSize) {
if (requestedPageSize <= 0) {
throw new IllegalArgumentException("For CSV streaming, page-size must be greater than 0.");
}
return requestedPageSize;
}

private void addLinkHeader(@NotNull Context ctx, TimeSeries ts, ContentType contentType) {
try {
// Send back the link to the next page in the response header
Expand Down
Loading
Loading