Untitled
unknown
plain_text
2 years ago
766 B
10
Indexable
val pairRegex = "\\((\\d+),(-?\\d+\\.\\d+E?-?\\d+)\\)".r
val processStart: DateTime = DateTime.now()
val localModelMaps = List
.range(0, 3)
.map(num => {
val modelFile = sc.textFile(args.model() + "/part-0000" + num)
modelFile
.aggregate(Map.empty[Int, Double])(
(acc, line) => {
pairRegex.findFirstMatchIn(line) match {
case Some(matchResult) =>
val key = matchResult.group(1).toInt
val value = matchResult.group(2).toDouble
acc + (key -> value)
case None => acc
}
},
(acc1, acc2) => acc1 ++ acc2
)
})
val modelMaps = sc.broadcast(localModelMaps)Editor is loading...
Leave a Comment