Commit c9e21b9c5afabaf24573a9676b49f34f831df259

Authored by Noah ago
1 parent a047e6666e
Exists in master

대시보드 프로젝트에서 분리

크롤링 범위 지정가능 메소드 분리
기타등등

Showing 4 changed files with 151 additions and 14 deletions Side-by-side Diff

insight/context.py View file @ c9e21b9
... ... @@ -27,7 +27,7 @@
27 27  
28 28 def __init__(self, token):
29 29 print "context"
30   - self.token = token
  30 + self.token = token.getToken()
31 31  
32 32 def setOrderList(self, list):
33 33 self.orderlist = list
insight/datadb.py View file @ c9e21b9
  1 +# -*- coding: utf-8 -*-
  2 +import MySQLdb
  3 +
  4 +class DataDB():
  5 + def __init__(self):
  6 + self.init_database()
  7 +
  8 + def init_database(self):
  9 + #mysqlHost = '192.168.20.61'
  10 + mysqlHost = '127.0.0.1'
  11 + UserName = 'root'
  12 + mysqlPassword = 'zmfflr2715'
  13 + DBName = 'piki_data_dashboard'
  14 + self.db = MySQLdb.connect(host=mysqlHost, user=UserName, passwd=mysqlPassword, db=DBName)
  15 + self.db.set_character_set('utf8')
  16 + charset = self.db.character_set_name()
  17 + self.cursor = self.db.cursor(MySQLdb.cursors.DictCursor)
  18 +
  19 + def fb_insert(self,sql):
  20 + print sql
  21 + self.cursor.execute(sql)
  22 + try :
  23 + self.db.commit()
  24 + except Exception, e:
  25 + print e
  26 + return -1
  27 + return
insight/insight.py View file @ c9e21b9
... ... @@ -11,7 +11,7 @@
11 11 insight_map = {}
12 12  
13 13 def __init__(self,token):
14   - self.token = token
  14 + self.token = token.getToken()
15 15 print "Insight"
16 16  
17 17 def setContentInsight(self,content_id):
18 18  
19 19  
20 20  
... ... @@ -20,19 +20,32 @@
20 20 txt = requests.get(url + self.token).text
21 21 self.data = json.loads(txt)
22 22 self.setContentType(self.data)
23   -
  23 + print txt
24 24 for i in self.data['insights']['data']:
25 25 name = i['id'].split("/")[2]
26 26 values = i['values'][0]
27 27 self.insight_map[name]= values
28 28  
29   -
30   -
31 29 def getContentInsightByKey(self, key):
  30 + #subkey is double only
32 31  
33 32 sub_key_list = key.split(',')
34   - print sub_key_list
35   - return self.insight_map[key]
  33 +
  34 + try:
  35 + if len(sub_key_list) == 2:
  36 + ret = self.insight_map[sub_key_list[0]]['value'][sub_key_list[1]]
  37 +
  38 + else :
  39 + ret = self.insight_map[sub_key_list[0]]['value']
  40 +
  41 + if ret == {} :
  42 + return 0
  43 +
  44 + return ret
  45 +
  46 + except:
  47 + return 0
  48 +
36 49  
37 50 def getContentInsightAll(self):
38 51 return self.insight_map
... ... @@ -5,28 +5,125 @@
5 5 from insight.token import Token
6 6 from insight.page import Page
7 7 from insight.insight import Insight
  8 +from insight.datadb import DataDB
8 9 import time
9 10  
10 11  
11 12 if __name__=='__main__':
12 13  
13   - main_token = "CAAUTLd5JgaoBABnBUUgOjgaUkuviOZCC1otVIJYwYapCRTZBU8ZAbTOd1uo1SBiqHr1eBhVdzoYDoBjPzqO0ImpxGZAXtlXeVtp4r2muUDt3O53NLYqFgrJqKAowcU7fWMsmMTZAAtKT8MXeF4Lb0rHQ2I1gHmbr6nlJWBHzk6dsNPmYHguMp"
  14 + token_str = "CAAUTLd5JgaoBABnBUUgOjgaUkuviOZCC1otVIJYwYapCRTZBU8ZAbTOd1uo1SBiqHr1eBhVdzoYDoBjPzqO0ImpxGZAXtlXeVtp4r2muUDt3O53NLYqFgrJqKAowcU7fWMsmMTZAAtKT8MXeF4Lb0rHQ2I1gHmbr6nlJWBHzk6dsNPmYHguMp"
  15 + token = Token(token_str)
14 16  
15   - token = Token(main_token)
16   -
17   - contexts = Context(token.getToken())
  17 + contexts = Context(token)
18 18 contexts_list = contexts.getContextsFeedAndTimestemp(3600 * 24 * 10)
19 19  
20 20 page = Page(contexts_list)
21 21 contents_list = page.getContentsList()
22 22  
23   - for content in contents_list[0:3]:
  23 +
  24 + for content in contents_list:
  25 +
24 26 print content['id']
25 27 print content['created_time'].replace('T'," ").replace('+0000',"")
26 28 print content['message'].replace('\n',"").replace('\r',"").replace('\'',"")
27 29  
28   - insight = Insight(token.getToken())
  30 + insight = Insight(token)
29 31 insight.setContentInsight(content['id'])
  32 + lists =["id",
  33 + "message",
  34 + "created_time",
  35 + "loging_time",
  36 + "post_story_adds_unique",
  37 + "post_story_adds",
  38 + "post_story_adds_by_action_type_unique,comment",
  39 + "post_story_adds_by_action_type_unique,like",
  40 + "post_story_adds_by_action_type_unique,share",
  41 + "post_story_adds_by_action_type,comment",
  42 + "post_story_adds_by_action_type,like",
  43 + "post_story_adds_by_action_type,share",
  44 + "post_impressions_unique",
  45 + "post_impressions",
  46 + "post_impressions_paid_unique",
  47 + "post_impressions_paid",
  48 + "post_impressions_organic_unique",
  49 + "post_impressions_organic",
  50 + "post_impressions_viral_unique",
  51 + "post_impressions_viral",
  52 + "post_impressions_by_story_type_unique,other",
  53 + "post_impressions_by_story_type,other",
  54 + "post_impressions_fan_unique",
  55 + "post_impressions_fan",
  56 + "post_impressions_fan_paid_unique",
  57 + "post_impressions_fan_paid",
  58 + "post_impressions_by_paid_non_paid_unique,unpaid",
  59 + "post_impressions_by_paid_non_paid_unique,total",
  60 + "post_impressions_by_paid_non_paid_unique,paid",
  61 + "post_consumptions_by_type_unique,other clicks",
  62 + "post_consumptions_by_type_unique,photo view",
  63 + "post_consumptions_by_type_unique,video play",
  64 + "post_consumptions_by_type_unique,link clicks",
  65 + "post_consumptions_by_type,other clicks",
  66 + "post_consumptions_by_type,photo view",
  67 + "post_consumptions_by_type,video play",
  68 + "post_consumptions_by_type,link clicks",
  69 + "post_negative_feedback_unique",
  70 + "post_negative_feedback_by_type_unique,hide_click",
  71 + "post_negative_feedback_by_type_unique,hide_all_clicks",
  72 + "post_negative_feedback_by_type,hide_click",
  73 + "post_negative_feedback_by_type,hide_all_clicks",
  74 + "post_engaged_fan",
  75 + "post_fan_reach",
  76 + "post_storytellers",
  77 + "post_storytellers_by_action_type,comment",
  78 + "post_storytellers_by_action_type,like",
  79 + "post_engaged_users",
  80 + "post_stories",
  81 + "post_stories_by_action_type,comment",
  82 + "post_stories_by_action_type,like",
  83 + "post_interests_impressions_unique",
  84 + "post_interests_impressions",
  85 + "post_interests_consumptions_unique",
  86 + "post_interests_consumptions",
  87 + "post_interests_consumptions_by_type_unique,link clicks",
  88 + "post_interests_consumptions_by_type_unique,other clicks",
  89 + "post_interests_consumptions_by_type_unique,photo view",
  90 + "post_interests_consumptions_by_type_unique,video play",
  91 + "post_interests_consumptions_by_type,link clicks",
  92 + "post_interests_consumptions_by_type,other clicks",
  93 + "post_interests_consumptions_by_type,photo view",
  94 + "post_interests_consumptions_by_type,video play",
  95 + "post_interests_action_by_type_unique,comment",
  96 + "post_interests_action_by_type_unique,share",
  97 + "post_interests_action_by_type_unique,like",
  98 + "post_interests_action_by_type,comment",
  99 + "post_interests_action_by_type,share",
  100 + "post_interests_action_by_type,like",
  101 + "post_video_views_10s",
  102 + "post_video_views_10s_unique",
  103 + "post_video_views_10s_paid",
  104 + "post_video_views_10s_organic",
  105 + "post_video_views_10s_clicked_to_play",
  106 + "post_video_views_10s_autoplayed",
  107 + "post_video_views_10s_sound_on",
  108 + "post_video_views_sound_on",
  109 + "post_video_view_time",
  110 + "post_video_complete_views_organic",
  111 + "post_video_complete_views_paid"]
30 112  
31   - print "post_story_adds_by_action_type_unique", insight.getContentInsightByKey("post_story_adds_by_action_type_unique")
  113 + sqlprefix = "insert into facebook_insights ("
  114 + sqlvalues = " values ("
  115 + for list in lists:
  116 + print list, insight.getContentInsightByKey(list)
  117 + sqlprefix = sqlprefix + "`" + list.replace(',','_') + "`, "
  118 + sqlvalues = sqlvalues + str(insight.getContentInsightByKey(list)) + ", "
  119 + sqlprefix = sqlprefix[:len(sqlprefix)-2] + ")"
  120 + sqlvalues = sqlvalues[:len(sqlvalues)-2] + ");"
  121 +
  122 + #datadb = DataDB()
  123 + #datadb.fb_insert(sqlprefix + sqlvalues)
  124 + print sqlprefix + sqlvalues
  125 +
  126 +
  127 + del(insight)
  128 + #del(datadb)