Question

The progress function below is my worker function. I need to give it access to some classes which are costly to create / acquire. Is there any standard machinery for thread-local-variables in the libraries for this ? Or will I have to write a object pool manager myself ?

object Start extends App {
    def progress {
        val current = counter.getAndIncrement
        if(current % 100 == 0) {
            val perc = current.toFloat * 100 / totalPosts
            print(f"\r$perc%4.2f%%")
        }
    }

    val lexicon = new Global()

    def processTopic(forumId: Int, topicId: Int) {
        val(topic, posts) = ProcessingQueries.getTopicAndPosts(forumId,topicId)

        progress
    }



    val (fid, tl) = ProcessingQueries.getAllTopics("hon")
    val totalPosts = tl.size
    val counter = new AtomicInteger(0)
    val par = tl.par

    par.foreach { topic_id =>
        processTopic(fid,topic_id)
    }
}
Was it helpful?

Solution

Replaced the previous answer. This does the trick nice and tidy

object MyAnnotator extends ThreadLocal[StanfordCoreNLP] {
    val props = new Properties()
    props.put("annotators", "tokenize,ssplit,pos,lemma,parse")
    props.put("ssplit.newlineIsSentenceBreak", "two")
    props.put("parse.maxlen", "40")

    override def initialValue = new StanfordCoreNLP(props)
  }
Licensed under: CC-BY-SA with attribution
Not affiliated with StackOverflow
scroll top