-
Notifications
You must be signed in to change notification settings - Fork 4
/
test.py
54 lines (46 loc) · 1.34 KB
/
test.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
from bs4 import BeautifulSoup
import urllib2
import shlex
import os
import re
import sys
import wget
import json
from subprocess import Popen, PIPE
def mysoup(link):
url = urllib2.Request(link, headers={ 'User-Agent': 'Mozilla/5.0 (X11; Linux i686; rv:33.0) Gecko/20100101 Firefox/33.0' })
page = urllib2.urlopen(url)
soup = BeautifulSoup(page.read(), "html5lib")
return soup
def save(filename, data):
with open(filename, 'w') as f:
f.write(','.join(set(data)))
def read(filename):
with open(filename) as f:
data = f.read().split(',')
return data
print sys.argv[-1]
url = sys.argv[-1]
soup = mysoup(url)
article = soup.find("article")
print article
tags_p =[]
img = article.find('img').get('src')
print img
for p in article.find_all('p'):
print type(p)
tags_p.append(p)
print tags_p
with open('videos.json') as f:
video = json.loads(f.read())
print json.dumps(video, indent=1, ensure_ascii=False)
#update video json file with img + data
video['items'][0]['snippet']['img'] = img
video['items'][0]['snippet']['data'] = []
print tags_p
for i in tags_p:
video['items'][0]['snippet']['data'].append(unicode(i))
print json.dumps(video, indent=1, ensure_ascii=False)
with open('/home/hadn' + sys.argv[-1], 'wb') as f:
f.write(json.dumps(video,indent=1, ensure_ascii=False).encode('utf-8'))
#print tags_p'''