data science team / dsquality

Authored by Joanne about 6 years ago

Exists in master

Showing 2 changed files with 8 additions and 8 deletions Side-by-side Diff

...	...	@@ -44,20 +44,20 @@
44	44
45	45	val conScores = Array(conClick,conDwell,conPostAction,conReject,conEfficiency)
46	46
47		- val contentScore: RDD[(String, Int, Int)] = sc.union(conScores).reduceByKey((a, b) => {
	47	+ val contentScore: RDD[(String, Double)] = sc.union(conScores).reduceByKey((a, b) => {
48	48	a++b
49	49	}).map(x=>{
50	50	(x._1.toString,
51		- x._2.getOrElse("click",0)+
52		- x._2.getOrElse("dwell",0)+
53		- x._2.getOrElse("postAction",0)+
54		- x._2.getOrElse("reject",0),
55		- x._2.getOrElse("efficiency",0))
	51	+ 0.15*x._2.getOrElse("click",10)+
	52	+ 0.15*x._2.getOrElse("dwell",10)+
	53	+ 0.25*x._2.getOrElse("postAction",10)+
	54	+ 0.20*x._2.getOrElse("reject",10)+
	55	+ 0.25*x._2.getOrElse("efficiency",10))
56	56	})
57	57	val cidOfInterest: RDD[(Long, (Int, Long))] = getCidByStatus(sqlContext, Array("ACTV","HOLD"))
58	58
59	59	val cidAndScoreFormatted = make_0to1(contentScore.map(x=> (x._1,x._2.toDouble))).map(x=>(x._1, (x._2*1000).toLong))
60		- TempScoreSaveLoad.scoreSave(doi,"content","all_ver2",cidAndScoreFormatted,1)
	60	+ TempScoreSaveLoad.scoreSave(doi,"content","all_ver6",cidAndScoreFormatted,1)
61	61	}
62	62
63	63	}

...	...	@@ -68,7 +68,7 @@
68	68	val nowTS: Long = System.currentTimeMillis
69	69	val doi = getDateKey(nowTS)
70	70	//val doi = "20160508"
71		- val content: RDD[(Int, Int)] = scoreLoad(sc,doi,"content","all")
	71	+ val content: RDD[(Int, Int)] = scoreLoad(sc,doi,"content","all_ver6")
72	72	val comment: RDD[(Int, Int)] = scoreLoad(sc,doi,"comment","").groupByKey().map(x=>(x._1,0))
73	73	val editor: RDD[(Int, Int)] = scoreLoad(sc,doi,"editor","")
74	74