Untitled

 avatar
unknown
plain_text
a year ago
1.4 kB
8
Indexable
// Transformation function
Function<Tuple2<Row, Object>, Tuple2<Row, Object>> mapFunction = new Function<Tuple2<Row, Object>, Tuple2<Row, Object>>() {
    @Override
    public Tuple2<Row, Object> call(Tuple2<Row, Object> tuple) throws Exception {
        Row r = tuple._1();
        long recid = (Long) tuple._2();
        long id1 = Long.parseLong(start_rid) + recid;

        // Convert id1 to WrappedArray to include it in the row
        WrappedArray<Long> wrappedArray = WrappedArray.make(new Long[]{id1});

        // Create a new Row with id1 appended to the existing fields
        Object[] rowData = new Object[r.size() + 1];
        for (int i = 0; i < r.size(); i++) {
            rowData[i] = r.get(i);
        }
        rowData[r.size()] = wrappedArray;

        // Create a new Tuple2 with the updated Row and the original Object
        Tuple2<Row, Object> updatedTuple = new Tuple2<>(RowFactory.create(rowData), tuple._2());

        // Return the updated Tuple2
        return updatedTuple;
    }
};

// Apply the transformation using map transformation on JavaRDD
JavaRDD<Tuple2<Row, Object>> transformedTuplesJavaRDD = inputJavaRDD.map(mapFunction);

// Extract the Rows from the updated Tuple2s
JavaRDD<Row> transformedRowsJavaRDD = transformedTuplesJavaRDD.map(new Function<Tuple2<Row, Object>, Row>() {
   
Editor is loading...