Untitled

mail@pastecode.io avatar
unknown
plain_text
a month ago
766 B
2
Indexable
Never
val pairRegex = "\\((\\d+),(-?\\d+\\.\\d+E?-?\\d+)\\)".r
    val processStart: DateTime = DateTime.now()

    val localModelMaps = List
      .range(0, 3)
      .map(num => {
        val modelFile = sc.textFile(args.model() + "/part-0000" + num)

        modelFile
          .aggregate(Map.empty[Int, Double])(
            (acc, line) => {
              pairRegex.findFirstMatchIn(line) match {
                case Some(matchResult) =>
                  val key = matchResult.group(1).toInt
                  val value = matchResult.group(2).toDouble
                  acc + (key -> value)
                case None => acc
              }
            },
            (acc1, acc2) => acc1 ++ acc2
          )
      })

    val modelMaps = sc.broadcast(localModelMaps)
Leave a Comment