1 package org.wikimedia.eventutilities.flink.stream;
2
3 import java.net.URI;
4 import java.time.Instant;
5 import java.util.Arrays;
6 import java.util.List;
7 import java.util.Map;
8 import java.util.Properties;
9 import java.util.function.Predicate;
10 import java.util.stream.Collectors;
11
12 import javax.annotation.Nonnull;
13 import javax.annotation.Nullable;
14
15 import org.apache.flink.api.common.eventtime.WatermarkStrategy;
16 import org.apache.flink.api.common.typeinfo.TypeInformation;
17 import org.apache.flink.connector.file.sink.FileSink;
18 import org.apache.flink.connector.file.src.FileSource;
19 import org.apache.flink.connector.kafka.sink.KafkaRecordSerializationSchema;
20 import org.apache.flink.connector.kafka.sink.KafkaSink;
21 import org.apache.flink.connector.kafka.sink.KafkaSinkBuilder;
22 import org.apache.flink.connector.kafka.source.KafkaSource;
23 import org.apache.flink.connector.kafka.source.KafkaSourceBuilder;
24 import org.apache.flink.connector.kafka.source.enumerator.initializer.OffsetsInitializer;
25 import org.apache.flink.core.fs.Path;
26 import org.apache.flink.streaming.api.datastream.DataStreamSource;
27 import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
28 import org.apache.flink.connector.kafka.sink.KafkaPartitioner;
29 import org.apache.flink.types.Row;
30 import org.apache.kafka.clients.consumer.OffsetResetStrategy;
31 import org.apache.kafka.clients.producer.ProducerConfig;
32 import org.slf4j.Logger;
33 import org.slf4j.LoggerFactory;
34 import org.wikimedia.eventutilities.core.SerializableClock;
35 import org.wikimedia.eventutilities.core.event.EventStream;
36 import org.wikimedia.eventutilities.core.event.EventStreamFactory;
37 import org.wikimedia.eventutilities.core.event.JsonEventGenerator;
38 import org.wikimedia.eventutilities.flink.EventRowTypeInfo;
39 import org.wikimedia.eventutilities.flink.formats.json.JsonRowDeserializationSchema;
40 import org.wikimedia.eventutilities.flink.formats.json.JsonRowSerializationSchema;
41 import org.wikimedia.eventutilities.flink.formats.json.JsonSchemaFlinkConverter;
42 import org.wikimedia.eventutilities.flink.formats.json.KafkaEventSerializationSchema;
43 import org.wikimedia.eventutilities.flink.formats.json.KafkaRecordTimestampStrategy;
44
45 import com.fasterxml.jackson.databind.node.ObjectNode;
46 import com.google.common.base.Preconditions;
47 import com.google.common.base.Verify;
48
49 import lombok.Setter;
50 import lombok.experimental.Accessors;
51
52 /**
53 * Wraps EventStreamFactory with Flink DataStream API to provide
54 * helper methods to instantiate DataStream of Row using
55 * EventStream JSONSchemas and other metadata.
56 *
57 * Examples:
58 *
59 * Instantiate a EventDataStreamFactory from URIs:
60 * <pre>{@code
61 * EventDataStreamFactory factory = EventDataStreamFactory.from(
62 * Arrays.asList(
63 * "https://schema.wikimedia.org/repositories/primary/jsonschema",
64 * "https://schema.wikimedia.org/repositories/secondary/jsonschema",
65 * ),
66 * "https://meta.wikimedia.org/w/api.php"
67 * );
68 * }</pre>
69 *
70 * Get a {@link KafkaSource} for a declared event stream:
71 * <pre>{@code
72 * KafkaSource<Row> eventStreamSource = factory.kafkaSourceBuilder(
73 * "test.event.example", // EventStream name
74 * "localhost:9092",
75 * "my_consumer_group"
76 * ).build();
77 * }</pre>
78 *
79 */
80 @SuppressWarnings("checkstyle:classfanoutcomplexity")
81 public class EventDataStreamFactory {
82
83 private static final Logger LOG = LoggerFactory.getLogger(EventDataStreamFactory.class);
84
85 private final EventStreamFactory eventStreamFactory;
86 private final JsonEventGenerator eventGenerator;
87 private final Predicate<String> topicFilter;
88
89
90 /** Helper method to enforce default configuration settings for
91 * Flink Kafka producers. These are meant to ensure that topics
92 * produced by Flink applications have configuration parity with
93 * EventGate.
94 *
95 * References:
96 * - https://phabricator.wikimedia.org/T345805
97 * */
98 private static Properties getKafkaProducerConfig() {
99 Properties properties = new Properties();
100 // Specify the final compression type for a given topic.
101 // We set snappy because it's the compression algorithm used
102 // by our MirrorMaker deployment.
103 properties.setProperty(ProducerConfig.COMPRESSION_TYPE_CONFIG, "snappy");
104
105 return properties;
106 }
107
108 /**
109 * @deprecated use {@link #builder()}
110 */
111 @Deprecated
112 public EventDataStreamFactory(
113 EventStreamFactory eventStreamFactory,
114 JsonEventGenerator eventGenerator
115 ) {
116 this(eventStreamFactory, eventGenerator, t -> true);
117 }
118
119 private EventDataStreamFactory(
120 EventStreamFactory eventStreamFactory,
121 JsonEventGenerator eventGenerator,
122 Predicate<String> topicFilter
123 ) {
124 this.eventStreamFactory = eventStreamFactory;
125 this.eventGenerator = eventGenerator;
126 this.topicFilter = topicFilter;
127 }
128
129 public static Builder builder() {
130 return new Builder();
131 }
132
133 @Setter
134 @Accessors(fluent = true, chain = true)
135 public static class Builder {
136 private List<String> schemaBaseUris;
137 private String eventStreamConfigUri;
138 private EventStreamFactory eventStreamFactory;
139 private JsonEventGenerator generator;
140 /**
141 * SerializableClock to use for setting ingestion time on serialized events.
142 * This is only used by the default JsonEventGenerator if generator is not provided.
143 * This really only exists to make unit testing easier.
144 */
145 private SerializableClock ingestionTimeClock;
146 private Map<String, String> httpClientRoutes;
147 /**
148 * The filter to apply when extracting the configured topics of a stream.
149 * Accept all topics by default.
150 */
151 @Nonnull
152 private Predicate<String> topicFilter = t -> true;
153
154 /**
155 * Filter source topics with a particular prefix (generally the datacenter name).
156 * This is based on the naming convention that topics are prefixed with datacenter name they're populated from.
157 * @param topicPrefix the prefix to use (usually the name of a datacenter)
158 */
159 public Builder topicPrefix(String topicPrefix) {
160 topicFilter = t -> t.startsWith(topicPrefix);
161 return this;
162 }
163
164 public EventDataStreamFactory build() {
165 if (eventStreamFactory == null) {
166 Preconditions.checkArgument(schemaBaseUris != null,
167 "eventSchemaBaseUris must be set if eventStreamFactory is not provided");
168 Preconditions.checkArgument(eventStreamConfigUri != null,
169 "eventStreamConfigUri must be set if eventStreamFactory is not provided");
170 eventStreamFactory = EventStreamFactory.from(
171 schemaBaseUris, eventStreamConfigUri, httpClientRoutes
172 );
173 }
174
175 if (generator == null) {
176 JsonEventGenerator.Builder builder = JsonEventGenerator.builder()
177 .eventStreamConfig(eventStreamFactory.getEventStreamConfig())
178 .schemaLoader(eventStreamFactory.getEventSchemaLoader());
179
180 if (ingestionTimeClock != null) {
181 builder.ingestionTimeClock(ingestionTimeClock);
182 }
183
184 generator = builder.build();
185 }
186
187 return new EventDataStreamFactory(
188 eventStreamFactory,
189 generator,
190 topicFilter
191 );
192 }
193 }
194 /**
195 * EventDataStreamFactory factory method.
196 *
197 * @param eventSchemaBaseUris
198 * URIs from which to fetch event JSONSchemas.
199 *
200 * @param eventStreamConfigUri
201 * URI from which to fetch event stream config.
202 */
203 public static EventDataStreamFactory from(
204 @Nonnull List<String> eventSchemaBaseUris,
205 @Nonnull String eventStreamConfigUri
206 ) {
207
208 return EventDataStreamFactory.from(
209 eventSchemaBaseUris, eventStreamConfigUri, null
210 );
211 }
212
213 /**
214 * EventDataStreamFactory factory method.
215 *
216 * @param eventSchemaBaseUris
217 * URIs from which to fetch event JSONSchemas.
218 *
219 * @param eventStreamConfigUri
220 * URI from which to fetch event stream config.
221 *
222 * @param httpClientRoutes
223 * Map of source to dest http client routes.
224 * E.g. "https://meta.wikimedia.org" to "https://api-ro.wikimedia.org"
225 * If null, no special routing will be configured.
226 */
227 public static EventDataStreamFactory from(
228 @Nonnull List<String> eventSchemaBaseUris,
229 @Nonnull String eventStreamConfigUri,
230 Map<String, String> httpClientRoutes
231 ) {
232 EventStreamFactory eventStreamFactory = EventStreamFactory.from(
233 eventSchemaBaseUris, eventStreamConfigUri, httpClientRoutes
234 );
235
236 JsonEventGenerator generator = JsonEventGenerator.builder()
237 .eventStreamConfig(eventStreamFactory.getEventStreamConfig())
238 .schemaLoader(eventStreamFactory.getEventSchemaLoader())
239 .build();
240
241 return EventDataStreamFactory.builder()
242 .eventStreamFactory(eventStreamFactory)
243 .generator(generator)
244 .build();
245 }
246
247 /**
248 * Convenience method to get the EventStreamFactory used
249 * by this EventDataStreamFactory.
250 */
251 public EventStreamFactory getEventStreamFactory() {
252 return eventStreamFactory;
253 }
254
255 /**
256 * Gets the {@link EventRowTypeInfo} (which is a TypeInformation of Row)
257 * for the streamName.
258 * The corresponding schema is the latest obtained from the EventStream configuration
259 * identified by streamName.
260 *
261 * @param streamName
262 * Name of the EventStream, must be declared in EventStreamConfig.
263 */
264 public EventRowTypeInfo rowTypeInfo(String streamName) {
265 EventStream eventStream = eventStreamFactory.createEventStream(streamName);
266 ObjectNode jsonSchema = (ObjectNode)eventStream.schema();
267 ObjectNode messageKeyFields = (ObjectNode) eventStream.messageKeyFields();
268 return JsonSchemaFlinkConverter.toRowTypeInfo(jsonSchema, messageKeyFields);
269 }
270
271 /**
272 * Gets the {@link EventRowTypeInfo} (which is a TypeInformation of Row)
273 * for the streamName and a particular schema version.
274 *
275 * @param streamName
276 * Name of the EventStream, must be declared in EventStreamConfig.
277 * @param version
278 * Version of the schema to use
279 */
280 public EventRowTypeInfo rowTypeInfo(String streamName, String version) {
281 EventStream eventStream = eventStreamFactory.createEventStream(streamName);
282 ObjectNode jsonSchema = (ObjectNode)eventStream.schema(version);
283 ObjectNode messageKeyFields = (ObjectNode)eventStream.messageKeyFields();
284 return JsonSchemaFlinkConverter.toRowTypeInfo(jsonSchema, messageKeyFields);
285 }
286
287 /**
288 * Gets a JSON to Row DeserializationSchema by streamName and its latest
289 * schema version.
290 *
291 * @param streamName
292 * Name of the EventStream, must be declared in EventStreamConfig.
293 *
294 */
295 public JsonRowDeserializationSchema deserializer(String streamName) {
296 return deserializer(streamName, null);
297 }
298
299 /**
300 * Gets a JSON to Row DeserializationSchema by streamName.
301 *
302 * @param streamName
303 * Name of the EventStream, must be declared in EventStreamConfig.
304 *
305 * @param version
306 * Version of the schema to use. If null, the latest version will be used.
307 */
308 public JsonRowDeserializationSchema deserializer(String streamName, @Nullable String version) {
309 EventStream eventStream = eventStreamFactory.createEventStream(streamName);
310
311 ObjectNode jsonSchema;
312 if (version != null) {
313 jsonSchema = (ObjectNode)eventStream.schema(version);
314 } else {
315 jsonSchema = (ObjectNode)eventStream.schema();
316 }
317 return JsonSchemaFlinkConverter.toDeserializationSchemaRow(jsonSchema);
318 }
319
320 /**
321 * Create a {@link org.apache.flink.api.common.serialization.SerializationSchema} suited for
322 * producing json to the stream identified by streamName using the provided schema version.
323 */
324 public JsonRowSerializationSchema serializer(String streamName, String version) {
325 EventStream stream = eventStreamFactory.createEventStream(streamName);
326 JsonEventGenerator.EventNormalizer generator = eventGenerator
327 .createEventStreamEventGenerator(streamName, stream.schemaUri(version).toString());
328
329 TypeInformation<Row> typeInformation = rowTypeInfo(streamName, version);
330 return JsonRowSerializationSchema.builder()
331 .withNormalizationFunction(generator)
332 .withObjectMapper(generator.getObjectMapper())
333 .withTypeInfo(typeInformation)
334 .build();
335 }
336
337 /**
338 * Get a {@link KafkaSourceBuilder} that is primed with settings needed to consume
339 * the streamName from Kafka.
340 *
341 * This sets the following:
342 * - bootstrapServers,
343 * - topics
344 * - consumer group id
345 * - value only deserializer that will deserialize to a Row conforming to streamName's JSONSchema at schemaVersion.
346 * - starting offsets will use committed offsets, resetting to LATEST if no offsets are committed.
347 *
348 * If you want to change any of these settings, then call the appropriate
349 * method on the returned KafkaSourceBuilder before calling build().
350 * Note that Flink's KafkaSourceBuilder does not allow you to call setTopics() more than once,
351 * so if you need to override the topics used, be sure to provide them here as the topics parameter.
352 *
353 * Example:
354 *
355 * <pre>{@code
356 * EventDataStreamFactory eventDataStreamFactory = EventDataStreamFactory.from(...)
357 * KafkaSource<Row> eventStreamSource = eventDataStreamFactory.kafkaSourceBuilder(
358 * "test.event.example", // EventStream name
359 * "1.1.0",
360 * "localhost:9092",
361 * "my_consumer_group",
362 * Arrays.asList("topic1", "topic2")
363 * ).build();
364 * }</pre>
365 *
366 * @param streamName
367 * Name of the EventStream, must be declared in EventStreamConfig.
368 *
369 * @param schemaVersion
370 * Version of the schema to use. If null, the latest schema version will be used.
371 *
372 * @param bootstrapServers
373 * Kafka bootstrap.servers property.
374 *
375 * @param consumerGroup
376 * Kafka consumer.group.id property.
377 *
378 * @param topics
379 * Optional List of Kafka topcis to subscribe to.
380 * If null, the topics from EventStreamConfig matching the topic filter or topic prefix for streamName will be used.
381 *
382 * @see Builder#topicPrefix(String)
383 * @see Builder#topicFilter(Predicate)
384 */
385 public KafkaSourceBuilder<Row> kafkaSourceBuilder(
386 String streamName,
387 @Nullable String schemaVersion,
388 String bootstrapServers,
389 String consumerGroup,
390 @Nullable List<String> topics
391 ) {
392 EventStream eventStream = eventStreamFactory.createEventStream(streamName);
393
394 if (topics == null) {
395 topics = eventStream.topics().stream().filter(topicFilter).collect(Collectors.toList());
396 Verify.verify(!topics.isEmpty(), "No topics matching the filter were found");
397 } else {
398 Preconditions.checkArgument(!topics.isEmpty(), "The list of provided topics must not be empty");
399 }
400
401 KafkaSourceBuilder<Row> builder = KafkaSource.builder();
402 builder
403 .setBootstrapServers(bootstrapServers)
404 .setGroupId(consumerGroup)
405 .setTopics(topics)
406 .setValueOnlyDeserializer(deserializer(eventStream.streamName(), schemaVersion))
407 .setStartingOffsets(
408 OffsetsInitializer.committedOffsets(OffsetResetStrategy.LATEST)
409 );
410
411 return builder;
412 }
413
414 /**
415 *
416 * Get a {@link KafkaSourceBuilder} that is primed with settings needed to consume
417 * the streamName from Kafka using the latest schema version.
418 *
419 * @see #kafkaSourceBuilder(String, String, String, String, List)
420 */
421 public KafkaSourceBuilder<Row> kafkaSourceBuilder(
422 String streamName,
423 String bootstrapServers,
424 String consumerGroup,
425 @Nullable List<String> topics
426 ) {
427 return kafkaSourceBuilder(streamName, null, bootstrapServers, consumerGroup, topics);
428 }
429
430 /**
431 * Get a {@link KafkaSourceBuilder} that is primed with settings needed to consume
432 * the streamName from Kafka from the stream's configured topics that matching the
433 * topic filter or topic prefix using the latest schema version.
434 *
435 * @see #kafkaSourceBuilder(String, String, String, String, List)
436 */
437 public KafkaSourceBuilder<Row> kafkaSourceBuilder(
438 String streamName,
439 String bootstrapServers,
440 String consumerGroup
441 ) {
442 return kafkaSourceBuilder(streamName, null, bootstrapServers, consumerGroup, null);
443 }
444
445 /**
446 * Prepare a KafkaSinkBuilder with all the required components to produce to kafka using a json format
447 * matching the provided schema.
448 *
449 * Produces events to the default kafka partition.
450 *
451 * @see #kafkaSinkBuilder(String, String, String, String, KafkaRecordTimestampStrategy, KafkaPartitioner)
452 */
453 public KafkaSinkBuilder<Row> kafkaSinkBuilder(
454 String streamName,
455 @Nullable String schemaVersion,
456 String bootstrapServers,
457 String topic,
458 KafkaRecordTimestampStrategy timestampStrategy
459 ) {
460 return kafkaSinkBuilder(streamName, schemaVersion, bootstrapServers, topic, timestampStrategy, null);
461 }
462
463 /**
464 * Prepare a KafkaSinkBuilder with all the required components to produce to kafka using a json format
465 * matching the provided schema. The instantiated KafkaSink will produce snappy compressed records.
466 *
467 * The produced messages match the WMF Event Platform Rules:
468 * - the meta.dt field is filled and will be used as the kafka timestamp
469 * - the topic to produce to is one of the topics defined in the EventStream configuration
470 * - the provided schema must match the one defined in the EventStream configuration
471 * - the resulting json is validated against this schema
472 * - the dt (event time) field remains optional and must be set beforehand in the pipeline
473 *
474 * @param streamName
475 * Name of the EventStream, must be declared in EventStreamConfig.
476 *
477 * @param schemaVersion
478 * Version of the schema to use.
479 *
480 * @param bootstrapServers
481 * Kafka bootstrap.servers property.
482 *
483 * @param topic
484 * The Kafka topic to write to.
485 * If not one of the topics referenced by the EventStreamConfig, a warning will be logged.
486 *
487 * @param timestampStrategy
488 * The strategy to use regarding producer records timestamps and event time.
489 *
490 * @param partitioner
491 * An optional partioner
492 *
493 * @see KafkaRecordTimestampStrategy
494 */
495 public KafkaSinkBuilder<Row> kafkaSinkBuilder(
496 String streamName,
497 @Nullable String schemaVersion,
498 String bootstrapServers,
499 String topic,
500 KafkaRecordTimestampStrategy timestampStrategy,
501 @Nullable KafkaPartitioner<Row> partitioner
502 ) {
503 EventStream stream = eventStreamFactory.createEventStream(streamName);
504
505 if (!stream.topics().contains(topic)) {
506 LOG.warn(
507 "The topic '{}' is not configured for stream '{}'. Configured topics are [{}]. Using provided topic anyway.",
508 topic,
509 stream,
510 String.join(",", stream.topics())
511 );
512 }
513
514 JsonRowSerializationSchema serializationSchema = serializer(streamName, schemaVersion);
515
516 KafkaRecordSerializationSchema<Row> kafkaRecordSerializationSchema = new KafkaEventSerializationSchema(
517 serializationSchema.getTypeInformation(),
518 serializationSchema,
519 Instant::now,
520 topic,
521 timestampStrategy,
522 partitioner
523 );
524 return KafkaSink.<Row>builder()
525 .setBootstrapServers(bootstrapServers)
526 .setRecordSerializer(kafkaRecordSerializationSchema)
527 .setKafkaProducerConfig(getKafkaProducerConfig());
528 }
529
530 /**
531 * Gets a {@link FileSource.FileSourceBuilder} that will generate Rows of events in JSON
532 * files. The Rows will be deserialized using the JSONSchema of streamName.
533 * This is a slightly lower level method than
534 * fileDataStream(String, StreamExecutionEnvironment, WatermarkStrategy, URI...)
535 * You'll probably want to use fileDataStream, especially if you are just using this FileSource
536 * for testing purposes.
537 *
538 * @param streamName
539 * Name of the EventStream, must be declared in EventStreamConfig.
540 *
541 * @param schemaVersion
542 * Version of the schema to use. If null, latest version will be used.
543 *
544 * @param paths
545 * Flink Paths from which to read JSON events.
546 */
547 public FileSource.FileSourceBuilder<Row> fileStreamSourceBuilder(
548 String streamName,
549 @Nullable String schemaVersion,
550 Path... paths
551 ) {
552
553 return FileSource.forRecordStreamFormat(
554 new LineStreamFormat<>(deserializer(streamName, schemaVersion)),
555 paths
556 );
557 }
558
559 /**
560 * Gets a {@link FileSource.FileSourceBuilder} that will generate Rows of events in JSON
561 * files.
562 *
563 * @see #fileStreamSourceBuilder(String, String, Path...)
564 */
565 public FileSource.FileSourceBuilder<Row> fileStreamSourceBuilder(
566 String streamName,
567 Path... paths
568 ) {
569 return fileStreamSourceBuilder(streamName, null, paths);
570 }
571
572 /**
573 * Gets a {@link DataStreamSource} of {@link Row} for streamName that reads JSON events from files.
574 *
575 * Example:
576 *
577 * <pre>{@code
578 * EventDataStreamFactory eventDataStreamFactory = EventDataStreamFactory.from(...)
579 * DataStreamSource<Row> eventFileSource = eventDataStreamFactory.dataStreamFromFiles(
580 * "test.event.example", // EventStream name
581 * env, // Flink StreamExecutionEnvironment
582 * WatermarkStrategy.noWatermarks(),
583 * new URI("file:///path/to/test.events.json"
584 * );
585 * }</pre>
586 *
587 * @param streamName
588 * Name of the EventStream, must be declared in EventStreamConfig.
589 *
590 * @param schemaVersion
591 * Version of the schema to use. If null, latest version will be used.
592 *
593 * @param env
594 * StreamExecutionEnvironment in which to call fromSource.
595 *
596 * @param watermarkStrategy
597 * For simple testing input from files, you'll likely want to use WatermarkStrategy.noWatermarks();
598 *
599 * @param files
600 * URIs to files of JSON events that conform to the JSONSchema of streamName.
601 * These will be converted to Flink {@link Path}s.
602 */
603 public DataStreamSource<Row> fileDataStream(
604 String streamName,
605 @Nullable String schemaVersion,
606 StreamExecutionEnvironment env,
607 WatermarkStrategy<Row> watermarkStrategy,
608 URI... files
609 ) {
610 // Convert file URIs into Flink Paths
611 Path[] paths = Arrays.stream(files).map(Path::new).toArray(Path[]::new);
612 EventStream eventStream = getEventStreamFactory().createEventStream(streamName);
613
614 // Make a nice DataStreamSource description including all the source files.
615 String dataSourceDescription = eventStream +
616 " from files [" +
617 Arrays.stream(paths).map(Path::toString).collect(Collectors.joining(",")) +
618 "]";
619
620 return env.fromSource(
621 fileStreamSourceBuilder(streamName, schemaVersion, paths).build(),
622 watermarkStrategy,
623 dataSourceDescription
624 );
625 }
626
627 /**
628 * Gets a {@link DataStreamSource} of {@link Row} for streamName that reads JSON events from files using
629 * the latest schema version to deserialize.
630 *
631 * @see #fileDataStream(String, String, StreamExecutionEnvironment, WatermarkStrategy, URI...)
632 */
633 public DataStreamSource<Row> fileDataStream(
634 String streamName,
635 StreamExecutionEnvironment env,
636 WatermarkStrategy<Row> watermarkStrategy,
637 URI... files
638 ) {
639 return fileDataStream(streamName, null, env, watermarkStrategy, files);
640 }
641
642
643 /**
644 * A Row to JSON string Encoder for use with FileSinks.
645 *
646 * @param streamName Name of the EventStream, must be declared in EventStreamConfig.
647 * @param schemaVersion Version of the schema to use when encoding.
648 */
649 public FileSinkJsonRowEncoder fileSinkEncoder(
650 String streamName,
651 String schemaVersion
652 ) {
653 return new FileSinkJsonRowEncoder(
654 rowTypeInfo(streamName, schemaVersion),
655 serializer(streamName, schemaVersion),
656 Instant::now
657 );
658 }
659
660
661 /**
662 * Shortcut helper for calling fileSinkBuilderRowFormat using a String path.
663 *
664 * @param streamName
665 * Name of the EventStream, must be declared in EventStreamConfig.
666 *
667 * @param schemaVersion
668 * Version of the schema to use when encoding.
669 *
670 * @param path
671 * Base directory path location in which to write partitioned directories and files.
672 * Must begin with a Flink supported file protocol, e.g. file://, hdfs:// etc.
673 *
674 */
675 public FileSink.DefaultRowFormatBuilder<Row> fileSinkBuilderRowFormat(
676 String streamName,
677 String schemaVersion,
678 String path
679 ) {
680 return fileSinkBuilderRowFormat(streamName, schemaVersion, new Path(path));
681 }
682
683
684 /**
685 * A {@link FileSink} {@link FileSink.RowFormatBuilder} used to output
686 * serialize event Rows to JSON. If desired, customize the returned Builder
687 * with custom bucket assigners and rolling policies, and then call
688 * .build() to get a FileSink.
689 *
690 * @param streamName
691 * Name of the EventStream, must be declared in EventStreamConfig.
692 *
693 * @param schemaVersion
694 * Version of the schema to use when encoding.
695 *
696 * @param path
697 * Base directory path location in which to write partitioned directories and files.
698 */
699 public FileSink.DefaultRowFormatBuilder<Row> fileSinkBuilderRowFormat(
700 String streamName,
701 String schemaVersion,
702 Path path
703 // TODO add support for custom bucketAssigner and other rollingPolicies
704 ) {
705 return FileSink.forRowFormat(
706 path,
707 fileSinkEncoder(streamName, schemaVersion)
708 );
709 }
710
711 }