View Javadoc
1   package org.wikimedia.eventutilities.flink.stream;
2   
3   import java.net.URI;
4   import java.time.Instant;
5   import java.util.Arrays;
6   import java.util.List;
7   import java.util.Map;
8   import java.util.Properties;
9   import java.util.function.Predicate;
10  import java.util.stream.Collectors;
11  
12  import javax.annotation.Nonnull;
13  import javax.annotation.Nullable;
14  
15  import org.apache.flink.api.common.eventtime.WatermarkStrategy;
16  import org.apache.flink.api.common.typeinfo.TypeInformation;
17  import org.apache.flink.connector.file.sink.FileSink;
18  import org.apache.flink.connector.file.src.FileSource;
19  import org.apache.flink.connector.kafka.sink.KafkaRecordSerializationSchema;
20  import org.apache.flink.connector.kafka.sink.KafkaSink;
21  import org.apache.flink.connector.kafka.sink.KafkaSinkBuilder;
22  import org.apache.flink.connector.kafka.source.KafkaSource;
23  import org.apache.flink.connector.kafka.source.KafkaSourceBuilder;
24  import org.apache.flink.connector.kafka.source.enumerator.initializer.OffsetsInitializer;
25  import org.apache.flink.core.fs.Path;
26  import org.apache.flink.streaming.api.datastream.DataStreamSource;
27  import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
28  import org.apache.flink.connector.kafka.sink.KafkaPartitioner;
29  import org.apache.flink.types.Row;
30  import org.apache.kafka.clients.consumer.OffsetResetStrategy;
31  import org.apache.kafka.clients.producer.ProducerConfig;
32  import org.slf4j.Logger;
33  import org.slf4j.LoggerFactory;
34  import org.wikimedia.eventutilities.core.SerializableClock;
35  import org.wikimedia.eventutilities.core.event.EventStream;
36  import org.wikimedia.eventutilities.core.event.EventStreamFactory;
37  import org.wikimedia.eventutilities.core.event.JsonEventGenerator;
38  import org.wikimedia.eventutilities.flink.EventRowTypeInfo;
39  import org.wikimedia.eventutilities.flink.formats.json.JsonRowDeserializationSchema;
40  import org.wikimedia.eventutilities.flink.formats.json.JsonRowSerializationSchema;
41  import org.wikimedia.eventutilities.flink.formats.json.JsonSchemaFlinkConverter;
42  import org.wikimedia.eventutilities.flink.formats.json.KafkaEventSerializationSchema;
43  import org.wikimedia.eventutilities.flink.formats.json.KafkaRecordTimestampStrategy;
44  
45  import com.fasterxml.jackson.databind.node.ObjectNode;
46  import com.google.common.base.Preconditions;
47  import com.google.common.base.Verify;
48  
49  import lombok.Setter;
50  import lombok.experimental.Accessors;
51  
52  /**
53   * Wraps EventStreamFactory with Flink DataStream API to provide
54   * helper methods to instantiate DataStream of Row using
55   * EventStream JSONSchemas and other metadata.
56   *
57   * Examples:
58   *
59   * Instantiate a EventDataStreamFactory from URIs:
60   * <pre>{@code
61   *     EventDataStreamFactory factory = EventDataStreamFactory.from(
62   *          Arrays.asList(
63   *              "https://schema.wikimedia.org/repositories/primary/jsonschema",
64   *              "https://schema.wikimedia.org/repositories/secondary/jsonschema",
65   *          ),
66   *          "https://meta.wikimedia.org/w/api.php"
67   *     );
68   * }</pre>
69   *
70   * Get a {@link KafkaSource} for a declared event stream:
71   * <pre>{@code
72   *     KafkaSource<Row> eventStreamSource = factory.kafkaSourceBuilder(
73   *          "test.event.example",  // EventStream name
74   *          "localhost:9092",
75   *          "my_consumer_group"
76   *     ).build();
77   * }</pre>
78   *
79   */
80  @SuppressWarnings("checkstyle:classfanoutcomplexity")
81  public class EventDataStreamFactory {
82  
83      private static final Logger LOG = LoggerFactory.getLogger(EventDataStreamFactory.class);
84  
85      private final EventStreamFactory eventStreamFactory;
86      private final JsonEventGenerator eventGenerator;
87      private final Predicate<String> topicFilter;
88  
89  
90      /** Helper method to enforce default configuration settings for
91       * Flink Kafka producers. These are meant to ensure that topics
92       * produced by Flink applications have configuration parity with
93       * EventGate.
94       *
95       * References:
96       *  - https://phabricator.wikimedia.org/T345805
97       * */
98      private static Properties getKafkaProducerConfig() {
99          Properties properties = new Properties();
100         // Specify the final compression type for a given topic.
101         // We set snappy because it's the compression algorithm used
102         // by our MirrorMaker deployment.
103         properties.setProperty(ProducerConfig.COMPRESSION_TYPE_CONFIG, "snappy");
104 
105         return properties;
106     }
107 
108     /**
109      * @deprecated use {@link #builder()}
110      */
111     @Deprecated
112     public EventDataStreamFactory(
113         EventStreamFactory eventStreamFactory,
114         JsonEventGenerator eventGenerator
115     ) {
116         this(eventStreamFactory, eventGenerator, t -> true);
117     }
118 
119     private EventDataStreamFactory(
120             EventStreamFactory eventStreamFactory,
121             JsonEventGenerator eventGenerator,
122             Predicate<String> topicFilter
123     ) {
124         this.eventStreamFactory = eventStreamFactory;
125         this.eventGenerator = eventGenerator;
126         this.topicFilter = topicFilter;
127     }
128 
129     public static Builder builder() {
130         return new Builder();
131     }
132 
133     @Setter
134     @Accessors(fluent = true, chain = true)
135     public static class Builder {
136         private List<String> schemaBaseUris;
137         private String eventStreamConfigUri;
138         private EventStreamFactory eventStreamFactory;
139         private JsonEventGenerator generator;
140         /**
141          * SerializableClock to use for setting ingestion time on serialized events.
142          * This is only used by the default JsonEventGenerator if generator is not provided.
143          * This really only exists to make unit testing easier.
144          */
145         private SerializableClock ingestionTimeClock;
146         private Map<String, String> httpClientRoutes;
147         /**
148          * The filter to apply when extracting the configured topics of a stream.
149          * Accept all topics by default.
150          */
151         @Nonnull
152         private Predicate<String> topicFilter = t -> true;
153 
154         /**
155          * Filter source topics with a particular prefix (generally the datacenter name).
156          * This is based on the naming convention that topics are prefixed with datacenter name they're populated from.
157          * @param topicPrefix the prefix to use (usually the name of a datacenter)
158          */
159         public Builder topicPrefix(String topicPrefix) {
160             topicFilter = t -> t.startsWith(topicPrefix);
161             return this;
162         }
163 
164         public EventDataStreamFactory build() {
165             if (eventStreamFactory == null) {
166                 Preconditions.checkArgument(schemaBaseUris != null,
167                         "eventSchemaBaseUris must be set if eventStreamFactory is not provided");
168                 Preconditions.checkArgument(eventStreamConfigUri != null,
169                         "eventStreamConfigUri must be set if eventStreamFactory is not provided");
170                 eventStreamFactory = EventStreamFactory.from(
171                         schemaBaseUris, eventStreamConfigUri, httpClientRoutes
172                 );
173             }
174 
175             if (generator == null) {
176                 JsonEventGenerator.Builder builder = JsonEventGenerator.builder()
177                     .eventStreamConfig(eventStreamFactory.getEventStreamConfig())
178                     .schemaLoader(eventStreamFactory.getEventSchemaLoader());
179 
180                 if (ingestionTimeClock != null) {
181                     builder.ingestionTimeClock(ingestionTimeClock);
182                 }
183 
184                 generator = builder.build();
185             }
186 
187             return new EventDataStreamFactory(
188                     eventStreamFactory,
189                     generator,
190                     topicFilter
191             );
192         }
193     }
194     /**
195      * EventDataStreamFactory factory method.
196      *
197      * @param eventSchemaBaseUris
198      *  URIs from which to fetch event JSONSchemas.
199      *
200      * @param eventStreamConfigUri
201      *  URI from which to fetch event stream config.
202      */
203     public static EventDataStreamFactory from(
204         @Nonnull List<String> eventSchemaBaseUris,
205         @Nonnull String eventStreamConfigUri
206     ) {
207 
208         return EventDataStreamFactory.from(
209             eventSchemaBaseUris, eventStreamConfigUri, null
210         );
211     }
212 
213     /**
214      * EventDataStreamFactory factory method.
215      *
216      * @param eventSchemaBaseUris
217      *  URIs from which to fetch event JSONSchemas.
218      *
219      * @param eventStreamConfigUri
220      *  URI from which to fetch event stream config.
221      *
222      * @param httpClientRoutes
223      *  Map of source to dest http client routes.
224      *  E.g. "https://meta.wikimedia.org" to "https://api-ro.wikimedia.org"
225      *  If null, no special routing will be configured.
226      */
227     public static EventDataStreamFactory from(
228         @Nonnull List<String> eventSchemaBaseUris,
229         @Nonnull String eventStreamConfigUri,
230         Map<String, String> httpClientRoutes
231     ) {
232         EventStreamFactory eventStreamFactory = EventStreamFactory.from(
233                 eventSchemaBaseUris, eventStreamConfigUri, httpClientRoutes
234         );
235 
236         JsonEventGenerator generator = JsonEventGenerator.builder()
237                 .eventStreamConfig(eventStreamFactory.getEventStreamConfig())
238                 .schemaLoader(eventStreamFactory.getEventSchemaLoader())
239                 .build();
240 
241         return EventDataStreamFactory.builder()
242                 .eventStreamFactory(eventStreamFactory)
243                 .generator(generator)
244                 .build();
245     }
246 
247     /**
248      * Convenience method to get the EventStreamFactory used
249      * by this EventDataStreamFactory.
250      */
251     public EventStreamFactory getEventStreamFactory() {
252         return eventStreamFactory;
253     }
254 
255     /**
256      * Gets the {@link EventRowTypeInfo} (which is a TypeInformation of Row)
257      * for the streamName.
258      * The corresponding schema is the latest obtained from the EventStream configuration
259      * identified by streamName.
260      *
261      * @param streamName
262      *  Name of the EventStream, must be declared in EventStreamConfig.
263      */
264     public EventRowTypeInfo rowTypeInfo(String streamName) {
265         EventStream eventStream = eventStreamFactory.createEventStream(streamName);
266         ObjectNode jsonSchema = (ObjectNode)eventStream.schema();
267         ObjectNode messageKeyFields = (ObjectNode) eventStream.messageKeyFields();
268         return JsonSchemaFlinkConverter.toRowTypeInfo(jsonSchema, messageKeyFields);
269     }
270 
271     /**
272      * Gets the {@link EventRowTypeInfo} (which is a TypeInformation of Row)
273      * for the streamName and a particular schema version.
274      *
275      * @param streamName
276      *  Name of the EventStream, must be declared in EventStreamConfig.
277      * @param version
278      *  Version of the schema to use
279      */
280     public EventRowTypeInfo rowTypeInfo(String streamName, String version) {
281         EventStream eventStream = eventStreamFactory.createEventStream(streamName);
282         ObjectNode jsonSchema = (ObjectNode)eventStream.schema(version);
283         ObjectNode messageKeyFields = (ObjectNode)eventStream.messageKeyFields();
284         return JsonSchemaFlinkConverter.toRowTypeInfo(jsonSchema, messageKeyFields);
285     }
286 
287     /**
288      * Gets a JSON to Row DeserializationSchema by streamName and its latest
289      * schema version.
290      *
291      * @param streamName
292      *  Name of the EventStream, must be declared in EventStreamConfig.
293      *
294      */
295     public JsonRowDeserializationSchema deserializer(String streamName) {
296         return deserializer(streamName, null);
297     }
298 
299     /**
300      * Gets a JSON to Row DeserializationSchema by streamName.
301      *
302      * @param streamName
303      *  Name of the EventStream, must be declared in EventStreamConfig.
304      *
305      * @param version
306      *  Version of the schema to use.  If null, the latest version will be used.
307      */
308     public JsonRowDeserializationSchema deserializer(String streamName, @Nullable String version) {
309         EventStream eventStream = eventStreamFactory.createEventStream(streamName);
310 
311         ObjectNode jsonSchema;
312         if (version != null) {
313             jsonSchema = (ObjectNode)eventStream.schema(version);
314         } else {
315             jsonSchema = (ObjectNode)eventStream.schema();
316         }
317         return JsonSchemaFlinkConverter.toDeserializationSchemaRow(jsonSchema);
318     }
319 
320     /**
321      * Create a {@link org.apache.flink.api.common.serialization.SerializationSchema} suited for
322      * producing json to the stream identified by streamName using the provided schema version.
323      */
324     public JsonRowSerializationSchema serializer(String streamName, String version) {
325         EventStream stream = eventStreamFactory.createEventStream(streamName);
326         JsonEventGenerator.EventNormalizer generator = eventGenerator
327                 .createEventStreamEventGenerator(streamName, stream.schemaUri(version).toString());
328 
329         TypeInformation<Row> typeInformation = rowTypeInfo(streamName, version);
330         return JsonRowSerializationSchema.builder()
331                 .withNormalizationFunction(generator)
332                 .withObjectMapper(generator.getObjectMapper())
333                 .withTypeInfo(typeInformation)
334                 .build();
335     }
336 
337     /**
338      * Get a {@link KafkaSourceBuilder} that is primed with settings needed to consume
339      * the streamName from Kafka.
340      *
341      * This sets the following:
342      * - bootstrapServers,
343      * - topics
344      * - consumer group id
345      * - value only deserializer that will deserialize to a Row conforming to streamName's JSONSchema at schemaVersion.
346      * - starting offsets will use committed offsets, resetting to LATEST if no offsets are committed.
347      *
348      * If you want to change any of these settings, then call the appropriate
349      * method on the returned KafkaSourceBuilder before calling build().
350      * Note that Flink's KafkaSourceBuilder does not allow you to call setTopics() more than once,
351      * so if you need to override the topics used, be sure to provide them here as the topics parameter.
352      *
353      * Example:
354      *
355      * <pre>{@code
356      *     EventDataStreamFactory eventDataStreamFactory = EventDataStreamFactory.from(...)
357      *     KafkaSource&lt;Row&gt; eventStreamSource = eventDataStreamFactory.kafkaSourceBuilder(
358      *          "test.event.example",  // EventStream name
359      *          "1.1.0",
360      *          "localhost:9092",
361      *          "my_consumer_group",
362      *          Arrays.asList("topic1", "topic2")
363      *     ).build();
364      * }</pre>
365      *
366      * @param streamName
367      *  Name of the EventStream, must be declared in EventStreamConfig.
368      *
369      * @param schemaVersion
370      *  Version of the schema to use. If null, the latest schema version will be used.
371      *
372      * @param bootstrapServers
373      *  Kafka bootstrap.servers property.
374      *
375      * @param consumerGroup
376      *  Kafka consumer.group.id property.
377      *
378      * @param topics
379      *  Optional List of Kafka topcis to subscribe to.
380      *  If null, the topics from EventStreamConfig matching the topic filter or topic prefix for streamName will be used.
381      *
382      * @see Builder#topicPrefix(String)
383      * @see Builder#topicFilter(Predicate)
384      */
385     public KafkaSourceBuilder<Row> kafkaSourceBuilder(
386         String streamName,
387         @Nullable String schemaVersion,
388         String bootstrapServers,
389         String consumerGroup,
390         @Nullable List<String> topics
391     ) {
392         EventStream eventStream = eventStreamFactory.createEventStream(streamName);
393 
394         if (topics == null) {
395             topics = eventStream.topics().stream().filter(topicFilter).collect(Collectors.toList());
396             Verify.verify(!topics.isEmpty(), "No topics matching the filter were found");
397         } else {
398             Preconditions.checkArgument(!topics.isEmpty(), "The list of provided topics must not be empty");
399         }
400 
401         KafkaSourceBuilder<Row> builder = KafkaSource.builder();
402         builder
403             .setBootstrapServers(bootstrapServers)
404             .setGroupId(consumerGroup)
405             .setTopics(topics)
406             .setValueOnlyDeserializer(deserializer(eventStream.streamName(), schemaVersion))
407             .setStartingOffsets(
408                 OffsetsInitializer.committedOffsets(OffsetResetStrategy.LATEST)
409             );
410 
411         return builder;
412     }
413 
414     /**
415      *
416      * Get a {@link KafkaSourceBuilder} that is primed with settings needed to consume
417      * the streamName from Kafka using the latest schema version.
418      *
419      * @see #kafkaSourceBuilder(String, String, String, String, List)
420      */
421     public KafkaSourceBuilder<Row> kafkaSourceBuilder(
422         String streamName,
423         String bootstrapServers,
424         String consumerGroup,
425         @Nullable List<String> topics
426     ) {
427         return kafkaSourceBuilder(streamName, null, bootstrapServers, consumerGroup, topics);
428     }
429 
430     /**
431      * Get a {@link KafkaSourceBuilder} that is primed with settings needed to consume
432      * the streamName from Kafka from the stream's configured topics that matching the
433      * topic filter or topic prefix using the latest schema version.
434      *
435      * @see #kafkaSourceBuilder(String, String, String, String, List)
436      */
437     public KafkaSourceBuilder<Row> kafkaSourceBuilder(
438         String streamName,
439         String bootstrapServers,
440         String consumerGroup
441     ) {
442         return kafkaSourceBuilder(streamName, null, bootstrapServers, consumerGroup, null);
443     }
444 
445     /**
446      * Prepare a KafkaSinkBuilder with all the required components to produce to kafka using a json format
447      * matching the provided schema.
448      *
449      * Produces events to the default kafka partition.
450      *
451      * @see #kafkaSinkBuilder(String, String, String, String, KafkaRecordTimestampStrategy, KafkaPartitioner)
452      */
453     public KafkaSinkBuilder<Row> kafkaSinkBuilder(
454             String streamName,
455             @Nullable String schemaVersion,
456             String bootstrapServers,
457             String topic,
458             KafkaRecordTimestampStrategy timestampStrategy
459     ) {
460         return kafkaSinkBuilder(streamName, schemaVersion, bootstrapServers, topic, timestampStrategy, null);
461     }
462 
463     /**
464      * Prepare a KafkaSinkBuilder with all the required components to produce to kafka using a json format
465      * matching the provided schema. The instantiated KafkaSink will produce snappy compressed records.
466      *
467      * The produced messages match the WMF Event Platform Rules:
468      * - the meta.dt field is filled and will be used as the kafka timestamp
469      * - the topic to produce to is one of the topics defined in the EventStream configuration
470      * - the provided schema must match the one defined in the EventStream configuration
471      * - the resulting json is validated against this schema
472      * - the dt (event time) field remains optional and must be set beforehand in the pipeline
473      *
474      * @param streamName
475      *  Name of the EventStream, must be declared in EventStreamConfig.
476      *
477      * @param schemaVersion
478      *  Version of the schema to use.
479      *
480      * @param bootstrapServers
481      *  Kafka bootstrap.servers property.
482      *
483      * @param topic
484      *  The Kafka topic to write to.
485      *  If not one of the topics referenced by the EventStreamConfig, a warning will be logged.
486      *
487      * @param timestampStrategy
488      *  The strategy to use regarding producer records timestamps and event time.
489      *
490      * @param partitioner
491      *  An optional partioner
492      *
493      * @see KafkaRecordTimestampStrategy
494      */
495     public KafkaSinkBuilder<Row> kafkaSinkBuilder(
496             String streamName,
497             @Nullable String schemaVersion,
498             String bootstrapServers,
499             String topic,
500             KafkaRecordTimestampStrategy timestampStrategy,
501             @Nullable KafkaPartitioner<Row> partitioner
502     ) {
503         EventStream stream = eventStreamFactory.createEventStream(streamName);
504 
505         if (!stream.topics().contains(topic)) {
506             LOG.warn(
507                 "The topic '{}' is not configured for  stream '{}'. Configured topics are [{}]. Using provided topic anyway.",
508                 topic,
509                 stream,
510                 String.join(",", stream.topics())
511             );
512         }
513 
514         JsonRowSerializationSchema serializationSchema = serializer(streamName, schemaVersion);
515 
516         KafkaRecordSerializationSchema<Row> kafkaRecordSerializationSchema = new KafkaEventSerializationSchema(
517                 serializationSchema.getTypeInformation(),
518                 serializationSchema,
519                 Instant::now,
520                 topic,
521                 timestampStrategy,
522                 partitioner
523         );
524         return KafkaSink.<Row>builder()
525                 .setBootstrapServers(bootstrapServers)
526                 .setRecordSerializer(kafkaRecordSerializationSchema)
527                 .setKafkaProducerConfig(getKafkaProducerConfig());
528     }
529 
530     /**
531      * Gets a {@link FileSource.FileSourceBuilder} that will generate Rows of events in JSON
532      * files.  The Rows will be deserialized using the JSONSchema of streamName.
533      * This is a slightly lower level method than
534      * fileDataStream(String, StreamExecutionEnvironment, WatermarkStrategy, URI...)
535      * You'll probably want to use fileDataStream, especially if you are just using this FileSource
536      * for testing purposes.
537      *
538      * @param streamName
539      *  Name of the EventStream, must be declared in EventStreamConfig.
540      *
541      * @param schemaVersion
542      *  Version of the schema to use. If null, latest version will be used.
543      *
544      * @param paths
545      *  Flink Paths from which to read JSON events.
546      */
547     public FileSource.FileSourceBuilder<Row> fileStreamSourceBuilder(
548         String streamName,
549         @Nullable String schemaVersion,
550         Path... paths
551     ) {
552 
553         return FileSource.forRecordStreamFormat(
554             new LineStreamFormat<>(deserializer(streamName, schemaVersion)),
555             paths
556         );
557     }
558 
559     /**
560      * Gets a {@link FileSource.FileSourceBuilder} that will generate Rows of events in JSON
561      * files.
562      *
563      * @see #fileStreamSourceBuilder(String, String, Path...)
564      */
565     public FileSource.FileSourceBuilder<Row> fileStreamSourceBuilder(
566         String streamName,
567         Path... paths
568     ) {
569         return fileStreamSourceBuilder(streamName, null, paths);
570     }
571 
572     /**
573      * Gets a {@link DataStreamSource} of {@link Row} for streamName that reads JSON events from files.
574      *
575      * Example:
576      *
577      * <pre>{@code
578      *     EventDataStreamFactory eventDataStreamFactory = EventDataStreamFactory.from(...)
579      *     DataStreamSource&lt;Row&gt; eventFileSource = eventDataStreamFactory.dataStreamFromFiles(
580      *          "test.event.example",       // EventStream name
581      *          env,                        // Flink StreamExecutionEnvironment
582      *          WatermarkStrategy.noWatermarks(),
583      *          new URI("file:///path/to/test.events.json"
584      *     );
585      * }</pre>
586      *
587      * @param streamName
588      *  Name of the EventStream, must be declared in EventStreamConfig.
589      *
590      * @param schemaVersion
591      *  Version of the schema to use. If null, latest version will be used.
592      *
593      * @param env
594      *  StreamExecutionEnvironment in which to call fromSource.
595      *
596      * @param watermarkStrategy
597      *  For simple testing input from files, you'll likely want to use WatermarkStrategy.noWatermarks();
598      *
599      * @param files
600      *  URIs to files of JSON events that conform to the JSONSchema of streamName.
601      *  These will be converted to Flink {@link Path}s.
602      */
603     public DataStreamSource<Row> fileDataStream(
604         String streamName,
605         @Nullable String schemaVersion,
606         StreamExecutionEnvironment env,
607         WatermarkStrategy<Row> watermarkStrategy,
608         URI... files
609     ) {
610         // Convert file URIs into Flink Paths
611         Path[] paths = Arrays.stream(files).map(Path::new).toArray(Path[]::new);
612         EventStream eventStream = getEventStreamFactory().createEventStream(streamName);
613 
614         // Make a nice DataStreamSource description including all the source files.
615         String dataSourceDescription = eventStream +
616             " from files [" +
617             Arrays.stream(paths).map(Path::toString).collect(Collectors.joining(",")) +
618             "]";
619 
620         return env.fromSource(
621             fileStreamSourceBuilder(streamName, schemaVersion, paths).build(),
622             watermarkStrategy,
623             dataSourceDescription
624         );
625     }
626 
627     /**
628      * Gets a {@link DataStreamSource} of {@link Row} for streamName that reads JSON events from files using
629      * the latest schema version to deserialize.
630      *
631      * @see #fileDataStream(String, String, StreamExecutionEnvironment, WatermarkStrategy, URI...)
632      */
633     public DataStreamSource<Row> fileDataStream(
634         String streamName,
635         StreamExecutionEnvironment env,
636         WatermarkStrategy<Row> watermarkStrategy,
637         URI... files
638     ) {
639         return fileDataStream(streamName, null, env, watermarkStrategy, files);
640     }
641 
642 
643     /**
644      * A Row to JSON string Encoder for use with FileSinks.
645      *
646      * @param streamName    Name of the EventStream, must be declared in EventStreamConfig.
647      * @param schemaVersion Version of the schema to use when encoding.
648      */
649     public FileSinkJsonRowEncoder fileSinkEncoder(
650         String streamName,
651         String schemaVersion
652     ) {
653         return new FileSinkJsonRowEncoder(
654             rowTypeInfo(streamName, schemaVersion),
655             serializer(streamName, schemaVersion),
656             Instant::now
657         );
658     }
659 
660 
661     /**
662      * Shortcut helper for calling fileSinkBuilderRowFormat using a String path.
663      *
664      * @param streamName
665      *  Name of the EventStream, must be declared in EventStreamConfig.
666      *
667      * @param schemaVersion
668      *  Version of the schema to use when encoding.
669      *
670      * @param path
671      *  Base directory path location in which to write partitioned directories and files.
672      *  Must begin with a Flink supported file protocol, e.g. file://, hdfs:// etc.
673      *
674      */
675     public FileSink.DefaultRowFormatBuilder<Row> fileSinkBuilderRowFormat(
676         String streamName,
677         String schemaVersion,
678         String path
679     ) {
680         return fileSinkBuilderRowFormat(streamName, schemaVersion, new Path(path));
681     }
682 
683 
684     /**
685      * A {@link FileSink} {@link FileSink.RowFormatBuilder} used to output
686      * serialize event Rows to JSON. If desired, customize the returned Builder
687      * with custom bucket assigners and rolling policies, and then call
688      * .build() to get a FileSink.
689      *
690      * @param streamName
691      *  Name of the EventStream, must be declared in EventStreamConfig.
692      *
693      * @param schemaVersion
694      *  Version of the schema to use when encoding.
695      *
696      * @param path
697      *  Base directory path location in which to write partitioned directories and files.
698      */
699     public FileSink.DefaultRowFormatBuilder<Row> fileSinkBuilderRowFormat(
700         String streamName,
701         String schemaVersion,
702         Path path
703         // TODO add support for custom bucketAssigner and other rollingPolicies
704     ) {
705         return FileSink.forRowFormat(
706             path,
707             fileSinkEncoder(streamName, schemaVersion)
708         );
709     }
710 
711 }