Solr(10)Async and Multiple Thread on SolrJ

Solr(10)Async and Multiple Thread on SolrJ

1. Async Operation
I try to provide async http client there, but it seems does not help a lot

import jp.sf.amateras.solr.scala.async.AsyncSolrClient
import scala.concurrent.ExecutionContext.Implicits.global

import scala.collection.mutable.ListBuffer
import scala.concurrent.duration.Duration
import scala.concurrent.{Future, Await}

def addJobs(jobs:List[Job]):Unit = {
val futures = new ListBuffer[Future[Unit]]
for ( i<- 0 to jobs.size - 1){
val future = solrAsyncClient.add(jobs(i))
futures+= future
}
Await.ready(Future.sequence(futures),Duration.Inf)
}

The test class is as follow, it helps, but not much.
val num = 10000
val batch = 350
val jobs = ListBuffer[Job]()
val start = System.currentTimeMillis()
for ( i<- 1 to num){
val job = Job("id" + i, "title" + i, "desc" + i, "industry" + i)
jobs += job
if(i % batch == 0){
SolrClientDAO.addJobs(jobs.toList)
jobs.clear()
}
}
val end = System.currentTimeMillis()

println("total time for " + num + " is " + (end-start))
println("it is " + num / ((end-start)/1000) + " jobs/second")

2. SolrJ with Latest Version
package com.sillycat.jobsconsumer.persistence

import com.sillycat.jobsconsumer.models.Job
import com.sillycat.jobsconsumer.utilities.{IncludeConfig, IncludeLogger}
import org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient
import org.apache.solr.common.SolrInputDocument

import scala.collection.mutable.ListBuffer
import scala.collection.JavaConverters._

/**
* Created by carl on 8/7/15.
*/
object SolrJDAO extends IncludeLogger with IncludeConfig{


private val solrClient = {
try {
logger.info("Init the SOLR Client ---------------")
val solrURL = config.getString(envStr("solr.url.jobs"))
logger.info("SOLR URL = " + solrURL)
val client = new ConcurrentUpdateSolrClient(solrURL,100000,100)
client
} catch {
case x: Throwable =>
logger.error("Couldn't connect to SOLR: " + x)
null
}
}


def releaseResource = {
if(solrClient != null){
solrClient.close()
}
}

def addJobs(jobs:List[Job]):Unit = {
val docs = new ListBuffer[SolrInputDocument]
for( i<- 0 to (jobs.size - 1)){
val job = jobs(i)
val doc = new SolrInputDocument()
doc.addField("title", job.title)
doc.addField("id", job.id)
doc.addField("desc", job.desc)
doc.addField("industry", job.industry)
docs += doc
}
solrClient.add(docs.toList.asJava)
}


def commit = {
solrClient.commit()
}


}

The test class is as follow:
package com.sillycat.jobsconsumer.persistence

import com.sillycat.jobsconsumer.models.Job
import com.sillycat.jobsconsumer.utilities.IncludeConfig
import org.scalatest.{BeforeAndAfterAll, Matchers, FunSpec}

import scala.collection.mutable.ListBuffer

/**
* Created by carl on 8/7/15.
*/
class SolrJDAOSpec extends FunSpec with Matchers with BeforeAndAfterAll with IncludeConfig{

override def beforeAll() {
if(config.getString("build.env").equals("test")){

}
}

override def afterAll() {

}

describe("SolrDAO") {
describe("#add and query"){
it("Add one single job to Solr") {
val expect = Job("id1","title1","desc1","industry1")

val num = 1000000
val batch = 300
val jobs = ListBuffer[Job]()

val start = System.currentTimeMillis()
for ( i<- 1 to num ){

val job = Job("id" + i, "title" + i, "desc" + i, "industry" + i)
jobs += job
if(i % batch == 0){
SolrJDAO.addJobs(jobs.toList)
jobs.clear()
}
}
val end = System.currentTimeMillis()

println("total time for " + num + " is " + (end-start))
val duration = (end - start) / 1000
println("it is " + num / duration + " jobs/second")

SolrJDAO.commit

}
}
}

}

The result is amazing….
INFO [pool-4-thread-2-ScalaTest-running-SolrJDAOSpec] 2015-08-07 16:04:10,009 SolrJDAO.scala (line 24) Init the SOLR Client ---------------
INFO [pool-4-thread-2-ScalaTest-running-SolrJDAOSpec] 2015-08-07 16:04:10,012 SolrJDAO.scala (line 26) SOLR URL = http://ubuntu-master:8983/solr/jobs
total time for 1000000 is 8502
it is 125000 jobs/second

3. Try to Build the Driver myself


Tips
Add the open file limit on ubuntu and mac os
sudo sh -c "ulimit -n 65535 && exec su carl"
sudo ulimit -n 10000


References:
http://sillycat.iteye.com/blog/2233708

http://www.cnblogs.com/wgp13x/p/3742653.html
http://blog.csdn.net/john_f_lau/article/details/8780013