Untitled
unknown
plain_text
2 years ago
1.4 kB
11
Indexable
// Transformation function
Function<Tuple2<Row, Object>, Tuple2<Row, Object>> mapFunction = new Function<Tuple2<Row, Object>, Tuple2<Row, Object>>() {
@Override
public Tuple2<Row, Object> call(Tuple2<Row, Object> tuple) throws Exception {
Row r = tuple._1();
long recid = (Long) tuple._2();
long id1 = Long.parseLong(start_rid) + recid;
// Convert id1 to WrappedArray to include it in the row
WrappedArray<Long> wrappedArray = WrappedArray.make(new Long[]{id1});
// Create a new Row with id1 appended to the existing fields
Object[] rowData = new Object[r.size() + 1];
for (int i = 0; i < r.size(); i++) {
rowData[i] = r.get(i);
}
rowData[r.size()] = wrappedArray;
// Create a new Tuple2 with the updated Row and the original Object
Tuple2<Row, Object> updatedTuple = new Tuple2<>(RowFactory.create(rowData), tuple._2());
// Return the updated Tuple2
return updatedTuple;
}
};
// Apply the transformation using map transformation on JavaRDD
JavaRDD<Tuple2<Row, Object>> transformedTuplesJavaRDD = inputJavaRDD.map(mapFunction);
// Extract the Rows from the updated Tuple2s
JavaRDD<Row> transformedRowsJavaRDD = transformedTuplesJavaRDD.map(new Function<Tuple2<Row, Object>, Row>() {
Editor is loading...