Class PslSparkUtils (1.0.0)

Version latest

• 1.0.0 (latest)
• 0.4.5
• 0.3.4

public class PslSparkUtils

Inheritance

java.lang.Object > PslSparkUtils

Inherited Members

Object.clone()

Object.equals(Object)

Object.finalize()

Object.getClass()

Object.hashCode()

Object.notify()

Object.notifyAll()

Object.toString()

Object.wait()

Object.wait(long)

Object.wait(long,int)

Static Methods

convertAttributesToSparkMap(ListMultimap<String,ByteString> attributeMap)

public static ArrayBasedMapData convertAttributesToSparkMap(ListMultimap<String,ByteString> attributeMap)

Parameter
Name	Description
attributeMap	com.google.common.collect.ListMultimap<String,ByteString>

Returns
Type	Description
org.apache.spark.sql.catalyst.util.ArrayBasedMapData

getSparkEndOffset(SparkSourceOffset headOffset, SparkSourceOffset startOffset, long maxMessagesPerBatch, long topicPartitionCount)

public static SparkSourceOffset getSparkEndOffset(SparkSourceOffset headOffset, SparkSourceOffset startOffset, long maxMessagesPerBatch, long topicPartitionCount)

Parameters
Name	Description
headOffset	SparkSourceOffset
startOffset	SparkSourceOffset
maxMessagesPerBatch	long
topicPartitionCount	long

Returns
Type	Description
SparkSourceOffset

toInternalRow(SequencedMessage msg, SubscriptionPath subscription, Partition partition)

public static InternalRow toInternalRow(SequencedMessage msg, SubscriptionPath subscription, Partition partition)

Parameters
Name	Description
msg	com.google.cloud.pubsublite.SequencedMessage
subscription	com.google.cloud.pubsublite.SubscriptionPath
partition	com.google.cloud.pubsublite.Partition

Returns
Type	Description
org.apache.spark.sql.catalyst.InternalRow

toPslPartitionOffset(SparkPartitionOffset sparkPartitionOffset)

public static PslPartitionOffset toPslPartitionOffset(SparkPartitionOffset sparkPartitionOffset)

Parameter
Name	Description
sparkPartitionOffset	com.google.cloud.pubsublite.spark.SparkPartitionOffset

Returns
Type	Description
PslPartitionOffset

toPslSourceOffset(SparkSourceOffset sparkSourceOffset)

public static PslSourceOffset toPslSourceOffset(SparkSourceOffset sparkSourceOffset)

Parameter
Name	Description
sparkSourceOffset	SparkSourceOffset

Returns
Type	Description
PslSourceOffset

toPubSubMessage(StructType inputSchema, InternalRow row)

public static Message toPubSubMessage(StructType inputSchema, InternalRow row)

Parameters
Name	Description
inputSchema	org.apache.spark.sql.types.StructType
row	org.apache.spark.sql.catalyst.InternalRow

Returns
Type	Description
com.google.cloud.pubsublite.Message

toSparkSourceOffset(PslSourceOffset pslSourceOffset)

public static SparkSourceOffset toSparkSourceOffset(PslSourceOffset pslSourceOffset)

Parameter
Name	Description
pslSourceOffset	PslSourceOffset

Returns
Type	Description
SparkSourceOffset

verifyWriteInputSchema(StructType inputSchema)

public static void verifyWriteInputSchema(StructType inputSchema)

Make sure data fields for publish have expected Spark DataType if they exist.

Parameter
Name	Description
inputSchema	org.apache.spark.sql.types.StructType input table schema to write to Pub/Sub Lite.

Constructors

PslSparkUtils()

public PslSparkUtils()