Untitled
unknown
plain_text
a year ago
766 B
6
Indexable
val pairRegex = "\\((\\d+),(-?\\d+\\.\\d+E?-?\\d+)\\)".r val processStart: DateTime = DateTime.now() val localModelMaps = List .range(0, 3) .map(num => { val modelFile = sc.textFile(args.model() + "/part-0000" + num) modelFile .aggregate(Map.empty[Int, Double])( (acc, line) => { pairRegex.findFirstMatchIn(line) match { case Some(matchResult) => val key = matchResult.group(1).toInt val value = matchResult.group(2).toDouble acc + (key -> value) case None => acc } }, (acc1, acc2) => acc1 ++ acc2 ) }) val modelMaps = sc.broadcast(localModelMaps)
Editor is loading...
Leave a Comment